@starci/hfs 5.1.0 → 5.2.0

package/scaffold/add-module.mjs

@@ -0,0 +1,232 @@
+// starci app add module <name> / starci app remove module <name> - a module package of the product's agent SDK and its registry.
+//
+// A module is a workspace package `packages/module-<name>` (`@<project>/module-<name>`) built on the product's agent SDK (`packages/agent-sdk`): its
+// template and workbench declaration (React-free, for the back end), and its workbench (drawn with Grammar alone). `add` writes the package from
+// templates/app/module, registers it (root `workspaces`, the app dependencies on the registry and the module) and re-renders the managed registry
+// package `packages/modules` (module-registry.mts, the same render `starci app sync` checks); `remove` reverses every one of those steps.
+import fs from 'node:fs';
+import path from 'node:path';
+import { TEMPLATES_DIR } from '../sync/index.mjs';
+import { readRepoDeclaration, loadSlotManifest } from '../runtime/scripts/hfs/slots.mjs';
+import { byCodeUnit } from '../report/order.mjs';
+import { ScaffoldError, pascalOf, camelOf } from './service.mjs';
+import { packageJsonText } from './app.mjs';
+import { MODULE_PREFIX, REGISTRY_DIR, SDK_DIR, moduleNames, registryFiles, upperOf } from './module-registry.mjs';
+const NAME = /^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$/;
+const TEMPLATE_DIR = path.join(TEMPLATES_DIR, 'app', 'module');
+const WORKSPACE_GLOB = 'packages/*';
+const read = (file) => JSON.parse(fs.readFileSync(file, 'utf8'));
+const sortedDependencies = (dependencies) => Object.fromEntries(Object.entries(dependencies).sort(([left], [right]) => byCodeUnit(left, right)));
+const listFiles = (dir, base = dir) => fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
+    const full = path.join(dir, entry.name);
+    return entry.isDirectory() ? listFiles(full, base) : [path.relative(base, full).split(path.sep).join('/')];
+});
+/** A template's text with `{{project}}`, `{{name}}`, `{{pascal}}`, `{{camel}}` and `{{upper}}` filled; any other placeholder is a defect of the template. */
+function fill(text, values, file) {
+    return text.replace(/\{\{([A-Za-z]\w*)\}\}/g, (whole, key) => {
+        const value = values[key];
+        if (value === undefined)
+            throw new ScaffoldError('HFS_ADD_PLACEHOLDER_UNKNOWN', `module template ${file} uses ${whole}, which add module does not provide`);
+        return value;
+    }).replace(/\r\n/g, '\n');
+}
+/** The package files written from code: the manifests are never template files. */
+function manifests(project, name, versions) {
+    const sdk = `@${project}/agent-sdk`;
+    const exports = {
+        '.': { types: './dist/index.d.ts', default: './dist/index.js' },
+        './template': { types: './dist/template/index.d.ts', default: './dist/template/index.js' },
+        './manifest': { types: './dist/manifest/index.d.ts', default: './dist/manifest/index.js' },
+        './workbench': { types: './dist/esm/workbench/index.d.ts', default: './dist/esm/workbench/index.js' },
+        './public': { types: './dist/esm/public/index.d.ts', default: './dist/esm/public/index.js' },
+    };
+    const packageJson = {
+        name: `@${project}/${MODULE_PREFIX}${name}`,
+        version: '0.0.0',
+        private: true,
+        description: `The ${pascalOf(name)} module on the ${project} agent SDK: its template (./template) and workbench declaration (./manifest), both React-free for the back end, its Grammar-only workbench (./workbench) and its public face (./public) for the front end.`,
+        exports,
+        scripts: { build: 'tsc -p tsconfig.build.json && tsc -p tsconfig.build.workbench.json', typecheck: 'tsc --noEmit --pretty false', test: 'jest --coverage' },
+        dependencies: { [sdk]: '*' },
+        peerDependencies: { '@starci/grammar': versions['@starci/grammar'], react: versions['react'] },
+        devDependencies: sortedDependencies({ '@types/jest': versions['@types/jest'] ?? '', '@types/react': versions['@types/react'] ?? '', jest: versions['jest'] ?? '', 'ts-jest': versions['ts-jest'] ?? '', typescript: versions['typescript'] ?? '' }),
+    };
+    const build = { rootDir: 'src', noEmit: false, declaration: true, types: [] };
+    return [
+        { relative: 'package.json', body: packageJsonText(packageJson) },
+        { relative: 'tsconfig.json', body: packageJsonText({ extends: '@starci/tsconfig/next.json', compilerOptions: { incremental: false, types: ['jest', 'node'] }, include: ['src/**/*.ts', 'src/**/*.tsx', 'src/**/*.json'], exclude: ['node_modules', 'dist'] }) },
+        { relative: 'tsconfig.build.json', body: packageJsonText({ extends: './tsconfig.json', compilerOptions: { ...build, outDir: 'dist', module: 'node16', moduleResolution: 'node16' }, exclude: ['node_modules', 'dist', 'src/**/*.spec.ts', 'src/workbench/**', 'src/public/**'] }) },
+        { relative: 'tsconfig.build.workbench.json', body: packageJsonText({ extends: './tsconfig.json', compilerOptions: { ...build, outDir: 'dist/esm' }, include: ['src/**/*.ts', 'src/**/*.tsx', 'src/**/*.json'], exclude: ['node_modules', 'dist', 'src/**/*.spec.ts'] }) },
+    ];
+}
+/** The versions a module package declares, read from the app's own manifests so it installs the one version of each. */
+function versionsOf(root, feAppFiles) {
+    const rootPackage = read(path.join(root, 'package.json'));
+    const all = { ...rootPackage.devDependencies, ...rootPackage.dependencies };
+    for (const file of feAppFiles)
+        Object.assign(all, read(file).dependencies);
+    const wanted = ['@starci/grammar', 'react', '@types/jest', '@types/react', 'jest', 'ts-jest', 'typescript'];
+    const missing = wanted.filter((name) => all[name] === undefined);
+    if (missing.length > 0)
+        throw new ScaffoldError('HFS_ADD_MODULE_PINS', `add module reads the versions of ${missing.join(', ')} from the app's manifests, which declare none of them`);
+    return Object.fromEntries(wanted.map((name) => [name, all[name]]));
+}
+/** The package.json of every front-end Next app of the app. */
+const feManifests = (root) => {
+    const apps = path.join(root, 'fe', 'apps');
+    if (!fs.existsSync(apps))
+        return [];
+    return fs.readdirSync(apps, { withFileTypes: true }).filter((entry) => entry.isDirectory() && fs.existsSync(path.join(apps, entry.name, 'package.json'))).map((entry) => path.join(apps, entry.name, 'package.json')).sort(byCodeUnit);
+};
+/** Everything a verb changes, snapshotted first so a failure leaves the tree as it was. */
+class Transaction {
+    before = new Map();
+    root;
+    constructor(root) { this.root = root; }
+    write(relative, body) {
+        const file = path.join(this.root, ...relative.split('/'));
+        if (!this.before.has(file))
+            this.before.set(file, fs.existsSync(file) ? fs.readFileSync(file, 'utf8') : null);
+        fs.mkdirSync(path.dirname(file), { recursive: true });
+        fs.writeFileSync(file, body);
+    }
+    remove(relative) {
+        const file = path.join(this.root, ...relative.split('/'));
+        if (!fs.existsSync(file))
+            return;
+        for (const inner of fs.statSync(file).isDirectory() ? listFiles(file).map((name) => path.join(file, ...name.split('/'))) : [file])
+            this.before.set(inner, fs.readFileSync(inner, 'utf8'));
+        fs.rmSync(file, { recursive: true, force: true });
+    }
+    rollback() {
+        for (const [file, text] of this.before) {
+            if (text === null)
+                fs.rmSync(file, { force: true });
+            else {
+                fs.mkdirSync(path.dirname(file), { recursive: true });
+                fs.writeFileSync(file, text);
+            }
+        }
+    }
+}
+function context(root, name) {
+    if (!NAME.test(String(name)))
+        throw new ScaffoldError('HFS_ADD_NAME_INVALID', `module name ${name} must be kebab-case (letters, digits and single dashes)`);
+    const declarationFile = path.join(root, 'hfs.json');
+    const packageFile = path.join(root, 'package.json');
+    if (!fs.existsSync(declarationFile) || !fs.existsSync(packageFile))
+        throw new ScaffoldError('HFS_ADD_NOT_AN_APP', 'a module verb runs at the app root containing hfs.json and package.json');
+    const repo = readRepoDeclaration(loadSlotManifest(), root);
+    return { project: repo.project, declarationFile, packageFile };
+}
+/** Sets or deletes one dependency of a manifest file, keeping the section sorted; answers whether the text changed. */
+function setDependency(tx, root, file, name, version) {
+    const manifest = read(file);
+    const current = manifest.dependencies ?? {};
+    if ((version === null && current[name] === undefined) || current[name] === version)
+        return false;
+    const next = { ...current };
+    if (version === null)
+        Reflect.deleteProperty(next, name);
+    else
+        next[name] = version;
+    manifest.dependencies = sortedDependencies(next);
+    tx.write(path.relative(root, file).split(path.sep).join('/'), packageJsonText(manifest));
+    return true;
+}
+/** Registers the modules on the app: the workspace glob, and the registry and module dependencies of the back end (the root manifest) and every front-end app. */
+function register(tx, root, project, packageFile, names, changed) {
+    const rootPackage = read(packageFile);
+    const workspaces = new Set(rootPackage.workspaces ?? []);
+    const others = fs.existsSync(path.join(root, 'packages')) && fs.readdirSync(path.join(root, 'packages')).some((entry) => !entry.startsWith(MODULE_PREFIX) && entry !== 'modules');
+    if (names.length > 0)
+        workspaces.add(WORKSPACE_GLOB);
+    else if (!others)
+        workspaces.delete(WORKSPACE_GLOB);
+    if ([...workspaces].join('|') !== (rootPackage.workspaces ?? []).join('|')) {
+        rootPackage.workspaces = [...workspaces];
+        tx.write('package.json', packageJsonText(rootPackage));
+        changed.push('package.json');
+    }
+    const registry = `@${project}/modules`;
+    if (setDependency(tx, root, packageFile, registry, names.length > 0 ? '*' : null))
+        changed.push('package.json');
+    for (const file of feManifests(root)) {
+        let touched = setDependency(tx, root, file, registry, names.length > 0 ? '*' : null);
+        const present = Object.keys(read(file).dependencies ?? {}).filter((dependency) => dependency.startsWith(`@${project}/${MODULE_PREFIX}`));
+        for (const dependency of present)
+            if (!names.some((name) => `@${project}/${MODULE_PREFIX}${name}` === dependency))
+                touched = setDependency(tx, root, file, dependency, null) || touched;
+        for (const name of names)
+            touched = setDependency(tx, root, file, `@${project}/${MODULE_PREFIX}${name}`, '*') || touched;
+        if (touched)
+            changed.push(path.relative(root, file).split(path.sep).join('/'));
+    }
+}
+/** Replaces the registry package by the render of `names`: files written, and a registry of no module deleted. */
+function writeRegistry(tx, root, project, names, result) {
+    const files = registryFiles(project, names);
+    if (files.length === 0) {
+        if (fs.existsSync(path.join(root, REGISTRY_DIR))) {
+            tx.remove(REGISTRY_DIR);
+            result.removed.push(REGISTRY_DIR);
+        }
+        return;
+    }
+    for (const file of files) {
+        (fs.existsSync(path.join(root, file.path)) ? result.updated : result.created).push(file.path);
+        tx.write(file.path, file.content);
+    }
+}
+/** Adds the module package `name`, registers it and re-renders the registry. Refused when the app has no agent SDK package or the module exists. */
+export function addModule({ root, name }) {
+    const { project, packageFile } = context(root, name);
+    if (!fs.existsSync(path.join(root, SDK_DIR, 'package.json')))
+        throw new ScaffoldError('HFS_ADD_MODULE_SDK_MISSING', `a module is built on the product's agent SDK, which must exist as ${SDK_DIR}/ (a workspace package named @${project}/agent-sdk)`);
+    const folder = `packages/${MODULE_PREFIX}${name}`;
+    if (fs.existsSync(path.join(root, folder)))
+        throw new ScaffoldError('HFS_ADD_EXISTS', `module ${name} already exists: ${folder}`);
+    const values = { project, name, pascal: pascalOf(name), camel: camelOf(name), upper: upperOf(name) };
+    const planned = [
+        ...manifests(project, name, versionsOf(root, feManifests(root))),
+        ...listFiles(TEMPLATE_DIR).map((file) => ({
+            relative: file.replace(/\.tpl$/, '').split('/').map((segment) => segment.replace('__name__', name).replace('__pascal__', values.pascal)).join('/'),
+            body: fill(fs.readFileSync(path.join(TEMPLATE_DIR, ...file.split('/')), 'utf8'), values, file),
+        })),
+    ];
+    const result = { created: planned.map((file) => `${folder}/${file.relative}`), updated: [], removed: [] };
+    const tx = new Transaction(root);
+    try {
+        for (const file of planned)
+            tx.write(`${folder}/${file.relative}`, file.body);
+        const names = moduleNames(root);
+        register(tx, root, project, packageFile, names, result.updated);
+        writeRegistry(tx, root, project, names, result);
+    }
+    catch (error) {
+        tx.rollback();
+        fs.rmSync(path.join(root, folder), { recursive: true, force: true });
+        throw error;
+    }
+    return result;
+}
+/** Removes the module package `name`, its registrations and its entry in the registry; the last module takes the registry package with it. */
+export function removeModule({ root, name }) {
+    const { project, packageFile } = context(root, name);
+    const folder = `packages/${MODULE_PREFIX}${name}`;
+    if (!fs.existsSync(path.join(root, folder)))
+        throw new ScaffoldError('HFS_REMOVE_NOT_FOUND', `module ${name} does not exist: ${folder}`);
+    const result = { created: [], updated: [], removed: [folder] };
+    const tx = new Transaction(root);
+    try {
+        tx.remove(folder);
+        const names = moduleNames(root);
+        register(tx, root, project, packageFile, names, result.updated);
+        writeRegistry(tx, root, project, names, result);
+    }
+    catch (error) {
+        tx.rollback();
+        throw error;
+    }
+    return result;
+}

package/scaffold/add-queue-lite.mjs

@@ -300,6 +300,54 @@ function keysPatch(file) {
     lines.splice(last + 1, 0, ...KEY_ROWS);
     return { file, text: lines.join('\n') };
 }
+const DEV_REDIS_PORT = 56379;
+/** The redis service a dev compose file gains: loopback only, append-only, on the host port the environment declares. */
+const devRedisService = `  redis:
+    image: ${REDIS_IMAGE}
+    command: ["redis-server", "--appendonly", "yes"]
+    ports:
+      - "127.0.0.1:\${STACK_PORT_REDIS}:6379"
+    volumes:
+      - redis-data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 20
+    restart: unless-stopped
+`;
+/**
+ * A docker-compose dev environment (runtime docker-compose, a `local` block) runs the declared Redis from its compose file, on its own loopback
+ * port, and the queue processes reach it there: the compose service, the port, the service entry and the api and worker processes' queue settings.
+ */
+function devQueuePatches(repoRoot, manifest, edit, consumers) {
+    const patches = [];
+    const dev = Object.entries(manifest?.environments ?? {}).filter(([, environment]) => environment?.runtime === 'docker-compose' && environment.local);
+    for (const [name, environment] of dev) {
+        const composeRelative = `.starcistacks/${name}/${environment.composeFiles?.[0] ?? 'infra/compose/compose.yaml'}`;
+        const composeFile = path.join(repoRoot, ...composeRelative.split('/'));
+        const compose = read(composeFile).replace(/\r\n/g, '\n');
+        if (/^ {2}redis:/m.test(compose))
+            continue;
+        const base = ['environments', name];
+        const value = edit.value.environments[name];
+        edit.append([...base, 'components'], `redis: {port: ${DEV_REDIS_PORT}}\n`);
+        value.components.redis = { port: DEV_REDIS_PORT };
+        edit.append([...base, 'local', 'ports'], `redis: ${DEV_REDIS_PORT}\n`);
+        value.local.ports.redis = DEV_REDIS_PORT;
+        edit.append([...base, 'local', 'services'], '- {name: redis, wait: healthy}\n');
+        value.local.services.push({ name: 'redis', wait: 'healthy' });
+        for (const run of value.local.processes ?? []) {
+            if (!consumers.includes(run.name))
+                continue;
+            edit.append([...base, 'local', 'processes', { item: 'name', is: run.name }, 'env'], 'QUEUE_ENABLED: "true"\nQUEUE_REDIS_HOST: 127.0.0.1\nQUEUE_REDIS_PORT: "${port.redis}"\n');
+            Object.assign(run.env, { QUEUE_ENABLED: 'true', QUEUE_REDIS_HOST: '127.0.0.1', QUEUE_REDIS_PORT: '${port.redis}' });
+        }
+        const head = compose.includes('\nvolumes:\n') ? compose.replace('\nvolumes:\n', `\n${devRedisService}volumes:\n`) : `${compose.replace(/\n*$/, '\n')}${devRedisService}volumes:\n`;
+        patches.push({ file: composeFile, text: `${head.replace(/\n*$/, '\n')}  redis-data: {}\n` });
+    }
+    return patches;
+}
 /** Every stack file the declared Redis adds to or changes, planned before anything is written. */
 function stackFiles(repoRoot, { project, consumers }) {
     const manifestRelative = '.starcistacks/application-stacks.yaml';
@@ -330,6 +378,7 @@ function stackFiles(repoRoot, { project, consumers }) {
         if (!fs.existsSync(path.join(repoRoot, ...recovery.split('/'))))
             created.push({ relative: recovery, text: recoveryRecord });
     }
+    patches.push(...devQueuePatches(repoRoot, manifest, edit, consumers));
     patches.unshift({ file: manifestFile, text: edit.text() });
     return { patches: patches.filter((patch) => patch !== null), created, environments: swarm.map(([name]) => name) };
 }

package/scaffold/add.mjs

@@ -23,6 +23,7 @@ import { ScaffoldError, pascalOf } from './service.mjs';
 import { addTable, graphqlPlatformFiles } from './add-table.mjs';
 import { addModuleImport, compositionCode, readComposeGroup, writeComposeGroup } from './add-compose.mjs';
 import { addApp } from './add-app.mjs';
+import { addModule } from './add-module.mjs';
 import { FE_UNIT_LAYERS, addFeUnit } from './add-fe-unit.mjs';
 import { addFePage } from './add-fe-page.mjs';
 import { addFeOverlay } from './add-fe-overlay.mjs';
@@ -57,6 +58,10 @@ export function addKind({ repoRoot, noun, name, options = {}, now = Date.now })
         const result = addApp({ root: repoRoot, name });
         return { ...result, registered: { patterns: [], kinds: [] } };
     }
+    if (noun === 'module') {
+        const result = addModule({ root: repoRoot, name });
+        return { created: [...result.created, ...result.updated], registered: { patterns: [], kinds: [] } };
+    }
     const feOwner = FE_OWNERS[noun];
     if (feOwner !== undefined || Object.hasOwn(FE_UNIT_LAYERS, noun)) {
         const result = feOwner === undefined ? addFeUnit({ repoRoot, noun, name, options }) : feOwner({ repoRoot, name, options });

package/scaffold/app.mjs

@@ -18,7 +18,7 @@ import { spawnSync } from 'node:child_process';
 import { loadSlotManifest, resolveAppDeclaration } from '../runtime/scripts/hfs/slots.mjs';
 import { parseYaml } from '../runtime/engine/yaml.mjs';
 import { HARNESS_PACKAGE, canonPins } from '../runtime/scripts/hfs/canon-pins.mjs';
-import { TEMPLATES_DIR, appSource, imageFiles, render, renderTargets, writeTargets } from '../sync/index.mjs';
+import { TEMPLATES_DIR, appSource, imageFiles, render, renderTargets, settleSource, writeTargets } from '../sync/index.mjs';
 import { ScaffoldError } from './service.mjs';
 import { testWorldFiles } from './test-world.mjs';
 import { FE_APP_SCRIPTS, PACKAGE_MANAGER, WORKSPACES, feAppPackageName } from '../runtime/scripts/hfs/rules/monorepo.mjs';
@@ -158,15 +158,14 @@ const STARTER_DEPENDENCIES = Object.freeze({
 const LITE_STARTER_DEPENDENCIES = Object.freeze({
     dependencies: {
         '@apollo/server': null, '@nestjs/apollo': null, '@nestjs/common': null, '@nestjs/core': null, '@nestjs/cqrs': null, '@nestjs/graphql': null,
-        '@nestjs/platform-express': null, '@nestjs/typeorm': null, '@nestjs/passport': null, passport: null, 'passport-google-oauth20': null, 'passport-local': null,
-        '@aws-sdk/client-s3': null, '@aws-sdk/s3-request-presigner': null, bcryptjs: null,
-        'class-transformer': null, 'class-validator': null, graphql: null, jose: null, pg: null,
+        '@nestjs/platform-express': null, '@nestjs/typeorm': null, '@starci/auth': null, '@starci/object-storage': null,
+        'class-transformer': null, 'class-validator': null, graphql: null, pg: null,
         'reflect-metadata': null, rxjs: null, tslib: null, typeorm: null, undici: null, zod: null,
     },
     devDependencies: {
         turbo: null, '@nestjs/testing': null, '@starci/eslint-canon-be': null, '@starci/eslint-canon-fe': null,
         '@starci/jest-preset': null, '@starci/prettier-config': null, '@starci/stylelint-canon': null, '@starci/tsconfig': null, 'starci-harness': null, '@tailwindcss/postcss': null,
-        '@types/express': null, '@types/jest': null, '@types/node': null, '@types/passport-google-oauth20': null, '@types/passport-local': null, '@types/react': null, '@types/react-dom': null,
+        '@types/express': null, '@types/jest': null, '@types/node': null, '@types/react': null, '@types/react-dom': null,
         eslint: null, 'eslint-plugin-react-hooks': null, husky: null, jest: null, 'postcss-value-parser': null, prettier: null,
         stylelint: null, tailwindcss: null, 'ts-jest': null, 'ts-node-dev': null, 'tsc-alias': null,
         'tsconfig-paths': null, typescript: null,
@@ -187,7 +186,7 @@ const FE_APP_DEPENDENCIES = Object.freeze({
  * holds no identity SDK: modules/auth talks only to the app's own back-end auth doors.
  */
 const LITE_FE_APP_DEPENDENCIES = Object.freeze({
-    '@heroicons/react': null, '@heroui/react': null, '@heroui/styles': null, '@starci/grammar': null, '@starci/heroicons': null, '@xyflow/react': null, 'react-aria-components': null,
+    '@heroicons/react': null, '@heroui/react': null, '@heroui/styles': null, '@starci/auth': null, '@starci/grammar': null, '@starci/heroicons': null, '@xyflow/react': null, 'react-aria-components': null,
     next: null, 'next-intl': null, react: null, 'react-dom': null, 'server-only': null,
 });
 /**
@@ -656,7 +655,8 @@ export function scaffoldApp({ name, into, presets, harness, edition = 'full', re
     try {
         for (const file of files)
             writes.write(file.path, file.content);
-        const targets = renderTargets(declaration, presets, { manifest, source: appSource(root) });
+        const source = appSource(root);
+        const targets = renderTargets(declaration, presets, { manifest, source: settleSource(source, renderTargets(declaration, presets, { manifest, source })) });
         for (const target of targets)
             writes.claim(target.path);
         writeTargets(root, targets);

package/scaffold/module-registry.d.mts

@@ -0,0 +1,19 @@
+/** The folder name prefix of a module package. */
+export declare const MODULE_PREFIX = "module-";
+/** The package that lists every module package. */
+export declare const REGISTRY_DIR = "packages/modules";
+/** The product's agent SDK package folder every module builds on. */
+export declare const SDK_DIR = "packages/agent-sdk";
+/** A file of the registry package: its app-relative path and its content. */
+export interface RegistryFile {
+    readonly path: string;
+    readonly content: string;
+}
+/** `loyalty-points` as the UPPER_SNAKE prefix of the constants a module exports. */
+export declare const upperOf: (name: string) => string;
+/** The module names under `packages/` of the app at `root` (a folder module-<name> holding a package.json), sorted. */
+export declare function moduleNames(root: string): string[];
+/** The files of the registry package for `names` (empty when no module exists: the package is then absent). */
+export declare function registryFiles(project: string, names: readonly string[]): RegistryFile[];
+/** The PascalCase label of a module, for the messages that name it. */
+export declare const labelOf: (name: string) => string;

package/scaffold/module-registry.mjs

@@ -0,0 +1,84 @@
+// module-registry.mts - the module packages of an app (packages/module-<name>) and the registry package (packages/modules) that lists them.
+//
+// The registry is managed: its manifests and its three source files are rendered from the module list, by `starci app add|remove module` and by
+// `starci app sync` (the same function), so a hand edit is drift. A module package is written once by `add module` and is the product's from then on.
+import fs from 'node:fs';
+import path from 'node:path';
+import { byCodeUnit } from '../report/order.mjs';
+import { pascalOf } from './service.mjs';
+/** The folder name prefix of a module package. */
+export const MODULE_PREFIX = 'module-';
+/** The package that lists every module package. */
+export const REGISTRY_DIR = 'packages/modules';
+/** The product's agent SDK package folder every module builds on. */
+export const SDK_DIR = 'packages/agent-sdk';
+const HEADER = '// Generated by starci app sync (module registry). Do not edit: run "npx starci app sync --execute".';
+const json = (value) => `${JSON.stringify(value, null, 2)}\n`;
+/** `loyalty-points` as the UPPER_SNAKE prefix of the constants a module exports. */
+export const upperOf = (name) => name.replace(/-/g, '_').toUpperCase();
+/** The module names under `packages/` of the app at `root` (a folder module-<name> holding a package.json), sorted. */
+export function moduleNames(root) {
+    const folder = path.join(root, 'packages');
+    if (!fs.existsSync(folder))
+        return [];
+    return fs.readdirSync(folder, { withFileTypes: true })
+        .filter((entry) => entry.isDirectory() && entry.name.startsWith(MODULE_PREFIX) && fs.existsSync(path.join(folder, entry.name, 'package.json')))
+        .map((entry) => entry.name.slice(MODULE_PREFIX.length))
+        .sort(byCodeUnit);
+}
+/** The exports map of the registry and of a module: the React-free halves and, for a module, its Grammar workbench. */
+const registryExports = () => ({
+    '.': { types: './dist/index.d.ts', default: './dist/index.js' },
+    './template': { types: './dist/template/index.d.ts', default: './dist/template/index.js' },
+    './manifest': { types: './dist/manifest/index.d.ts', default: './dist/manifest/index.js' },
+});
+/** The files of the registry package for `names` (empty when no module exists: the package is then absent). */
+export function registryFiles(project, names) {
+    if (names.length === 0)
+        return [];
+    const sorted = [...names].sort(byCodeUnit);
+    const sdk = `@${project}/agent-sdk`;
+    const module = (name) => `@${project}/${MODULE_PREFIX}${name}`;
+    const serverImports = sorted.map((name) => `import { ${upperOf(name)}_SERVER } from "${module(name)}/template"`).join('\n');
+    const workbenchImports = sorted.map((name) => `import { ${upperOf(name)}_WORKBENCH } from "${module(name)}/manifest"`).join('\n');
+    const dependencies = Object.fromEntries([[sdk, '*'], ...sorted.map((name) => [module(name), '*'])].sort(([a], [b]) => byCodeUnit(a, b)));
+    return [
+        {
+            path: `${REGISTRY_DIR}/package.json`,
+            content: json({
+                name: `@${project}/modules`,
+                version: '0.0.0',
+                private: true,
+                description: 'The one list of the product modules built on the agent SDK, all React-free: ./template (every module\'s server half, for the back end) and ./manifest (every workbench declaration and the spec registry they assemble).',
+                exports: registryExports(),
+                scripts: { build: 'tsc -p tsconfig.build.json', typecheck: 'tsc --noEmit --pretty false' },
+                dependencies,
+            }),
+        },
+        {
+            path: `${REGISTRY_DIR}/tsconfig.json`,
+            content: json({ extends: '@starci/tsconfig/next.json', compilerOptions: { incremental: false, types: [] }, include: ['src/**/*.ts'], exclude: ['node_modules', 'dist'] }),
+        },
+        {
+            path: `${REGISTRY_DIR}/tsconfig.build.json`,
+            content: json({
+                extends: './tsconfig.json',
+                compilerOptions: { rootDir: 'src', outDir: 'dist', noEmit: false, declaration: true, module: 'node16', moduleResolution: 'node16' },
+            }),
+        },
+        {
+            path: `${REGISTRY_DIR}/src/index.ts`,
+            content: `${HEADER}\nimport { MODULE_SERVERS } from "./template"\n\n/** The keys of every module this registry lists, for a host that checks its own wiring against it. */\nexport const REGISTERED_MODULE_KEYS: ReadonlyArray<string> = MODULE_SERVERS.map((module) => module.key)\n`,
+        },
+        {
+            path: `${REGISTRY_DIR}/src/template/index.ts`,
+            content: `${HEADER}\nimport type { ModuleServer } from "${sdk}/module"\n${serverImports}\n\n/** The server half of every module built on the SDK; the back end reads nothing else of a module. */\nexport const MODULE_SERVERS: ReadonlyArray<ModuleServer> = [${sorted.map((name) => `${upperOf(name)}_SERVER`).join(', ')}]\n`,
+        },
+        {
+            path: `${REGISTRY_DIR}/src/manifest/index.ts`,
+            content: `${HEADER}\nimport { specRegistryFrom } from "${sdk}/workbench/contract"\nimport type { WorkbenchDeclaration, WorkbenchSpecRegistry } from "${sdk}/workbench/contract"\n${workbenchImports}\n\n/** The workbench declaration of every module built on the SDK; the front end pairs each with how it draws. */\nexport const MODULE_WORKBENCHES: ReadonlyArray<WorkbenchDeclaration> = [${sorted.map((name) => `${upperOf(name)}_WORKBENCH`).join(', ')}]\n\n/** Everything a stored spec may name: the SDK's blocks and each module's exposed blocks, reads and actions. */\nexport const SPEC_REGISTRY: WorkbenchSpecRegistry = specRegistryFrom(MODULE_WORKBENCHES)\n`,
+        },
+    ];
+}
+/** The PascalCase label of a module, for the messages that name it. */
+export const labelOf = (name) => pascalOf(name);

package/src/main.mjs

@@ -101,6 +101,7 @@ import { emitContractsAsync } from '../emit/contracts.mjs';
 import { ScaffoldError, newService, newSpec } from '../scaffold/service.mjs';
 import { EditionRefusal } from '../scaffold/edition-gate.mjs';
 import { addKind } from '../scaffold/add.mjs';
+import { removeModule } from '../scaffold/add-module.mjs';
 import { scaffoldApp } from '../scaffold/app.mjs';
 import { newImages } from '../scaffold/image.mjs';
 import { cachedLinters, eslintRepository, runLinter as nativeLinter } from '../lint/run.mjs';
@@ -140,7 +141,7 @@ const listFlag = (opts, flag) => { const given = opts[flag]; return Array.isArra
 /** An argument the parser refuses (an unknown flag, a missing value, a form a verb does not take): the one error `main` follows with the usage. */
 export class UsageError extends Error {
 }
-export const VERBS = Object.freeze(['scaffold', 'add', 'sync', 'check', 'upgrade', 'stack', 'world', 'release', 'slot', 'emit', 'secret', 'ci', 'db', 'coverage', 'codemod']);
+export const VERBS = Object.freeze(['scaffold', 'add', 'remove', 'sync', 'check', 'upgrade', 'stack', 'world', 'release', 'slot', 'emit', 'secret', 'ci', 'db', 'coverage', 'codemod']);
 const DATABASE_VERBS = new Map([['baseline', dbBaselineMain], ['restamp', dbRestampMain], ['migrate', dbMigrateMain], ['post-cutover', dbPostCutoverMain], ['dump', dbDumpMain], ['new', dbNewMain], ['drift', dbDriftMain]]);
 /**
  * The verbs parsed here, each with the only flags it takes ({ boolean, value, list }); a flag of another verb is refused, never ignored.
@@ -158,12 +159,13 @@ const PARSED = {
     add: {
         run: addMain, ...ADD_FLAGS, list: ['--inject'],
         forms: [
-            { operands: '<api|webhook|realtime|saga|job|reactor|queue|projection|cli|table|app|block|frame|composite|branch|leaf|page|overlay|layout> <name>', only: ADD_FLAGS },
+            { operands: '<api|webhook|realtime|saga|job|reactor|queue|projection|cli|table|app|module|block|frame|composite|branch|leaf|page|overlay|layout> <name>', only: ADD_FLAGS },
             { operands: 'service <dir> <name>', only: { list: ['--inject'] } },
             { operands: 'spec <file>.service.ts', flags: [] },
             { operands: 'image', flags: [] },
         ],
     },
+    remove: { run: removeMain, forms: [{ operands: 'module <name>', flags: [] }] },
     check: { run: checkMain, value: ['--coverage-from', '--shard', '--side', '--out'], list: GATE_FLAGS, boolean: ['--json', '--staged', '--no-cache', '--fix'], edition: true, forms: [{ only: { boolean: ['--json', '--no-cache', '--fix'], list: GATE_FLAGS, value: ['--coverage-from', '--shard', '--side', '--out'] } }, { only: { boolean: ['--staged'] }, edition: false }] },
     slot: { run: slotMain, boolean: ['--json'], forms: [{ operands: '<path>' }] },
     emit: { run: emitMain, forms: [{}] },
@@ -204,6 +206,11 @@ const VERB_USAGE = {
     world: ['starci app world <up|down|status> [--stack <dir>] [--services <list>] [--k3d] [--force] [--json] [--edition full|lite]'],
     stack: [
         ...STACK_ACTIONS.map((action) => line(`starci app stack ${action}`, STACK_OPERANDS[action], STACK_READ_ONLY.includes(action) ? flagUsage('--env', { value: true }) : '--env <env>', STACK_FLAGS[action].map((flag) => flagUsage(flag, { value: STACK_VALUE.has(flag), repeat: STACK_REPEAT.has(flag) })), '[--json]')),
+        // The dev actions of a docker-compose environment (stack/local-cli.mts); prepare, status and verify above answer them too.
+        'starci app stack doctor --env <env> [--json]',
+        'starci app stack up --env <env> [--execute] [--timeout <seconds>] [--json]',
+        'starci app stack down --env <env> [--execute] [--json]',
+        'starci app stack logs --env <env> --service <name> [--lines <n>] [--json]',
     ],
     release: Object.entries(RELEASE_FLAGS).map(([action, spec]) => line(`starci app release ${action}`, specUsage(spec))),
     secret: SECRET_VERBS.map((verb) => line(`starci app secret ${verb}`, secretUsage(verb))),
@@ -680,6 +687,20 @@ function addMain(opts, { cwd: repoRoot, stdout: out }) {
         out(`starci app add ${noun}: registered patterns ${registered.patterns.join(', ')}${kinds}\n`);
     return 0;
 }
+/** `remove module <name>`: deletes the module package and its registrations and re-renders the registry. */
+function removeMain(opts, { cwd: repoRoot, stdout: out }) {
+    const [noun, name, ...extra] = opts.positional;
+    if (noun !== 'module' || !name || extra.length)
+        throw new UsageError('starci app remove takes `module <name>`');
+    const { removed, updated, created } = removeModule({ root: repoRoot, name });
+    for (const file of removed)
+        out(`removed ${file}
+`);
+    for (const file of [...updated, ...created])
+        out(`updated ${file}
+`);
+    return 0;
+}
 async function scaffoldMain(opts, { cwd, harness, edition = 'full', presets, lock, stdout: out }) {
     const [name, ...extra] = opts.positional;
     const registryVisibility = valueFlag(opts, 'registry-visibility');

package/stack/cli.mjs

@@ -18,6 +18,7 @@ import { withDockerTransport } from './docker-transport.mjs';
 import { StackError, EXIT, refusal } from './errors.mjs';
 import { parseTargetBinding, bindStackTarget, nativeToolEnvironment } from './target-binding.mjs';
 import { oneCommand, prepareUrgentRelease } from './urgent-release.mjs';
+import { isLocalEnvironment, isLocalOnly, isLocalShared, runLocal } from './local-cli.mjs';
 /** The actions that drive the docker CLI of the operator machine (pointed at the manager by docker-transport.mjs when the key is sealed). */
 const DOCKER_ACTIONS = new Set(['deploy', 'accept', 'converge', 'verify', 'rollback', 'prune', 'status']);
 export const ACTIONS = Object.freeze(['prepare', 'connect', 'disconnect', 'build-args', 'render', 'deploy', 'converge', 'verify', 'rollback', 'prune', 'backup', 'jobs', 'task', 'ingress', 'retire', 'build', 'accept', 'status', 'ci-key']);
@@ -185,8 +186,12 @@ export async function stackMain(argv, io = {}) {
     const ctx = stackContext(io);
     try {
         const action = argv[0];
+        if (isLocalOnly(action))
+            return await runLocal(ctx, action, argv.slice(1), io.localDeps);
         if (!isAction(action))
-            throw refusal('STACK_USAGE', `starci app stack takes ${[...ACTIONS, 'up', 'down', 'status'].join(', ')}`);
+            throw refusal('STACK_USAGE', `starci app stack takes ${[...ACTIONS, 'doctor', 'up', 'down', 'logs'].join(', ')}`);
+        if (isLocalShared(action) && argv.includes('--env') && isLocalEnvironment(ctx.cwd, String(argv[argv.indexOf('--env') + 1])))
+            return await runLocal(ctx, action, argv.slice(1), io.localDeps);
         const { flags, positional } = parseFlags(action, argv.slice(1));
         if (flags.env === undefined && !READ_ONLY.includes(action))
             throw refusal('STACK_USAGE', `stack ${action} has an effect: name its environment with --env <env> (only ${READ_ONLY.join(', ')} default to production)`);

package/stack/context.d.mts

@@ -47,6 +47,16 @@ export interface PipeResult {
     sink: number;
     stderr: string;
 }
+/** One heavy command the dev verbs run (`npm ci`, a compose build or up): admitted by the harness's heavy-command governor when it hands one in. */
+export interface HeavyRequest {
+    kind: 'install' | 'build';
+    label: string;
+    cwd: string;
+    argv: string[];
+    env: NodeJS.ProcessEnv;
+}
+/** The governed runner: waits for a slot of the host-wide semaphore, runs the argv without a shell and returns its result. */
+export type HeavyRun = (request: HeavyRequest) => Promise<RunResult>;
 /** The secret seam: decrypts the app's sops files. */
 export interface Sops {
     decrypt(file: string, format?: string): Record<string, string>;
@@ -82,6 +92,9 @@ export interface StackIo {
         role?: string | undefined;
     } | undefined;
     importer?: ((file: string) => Promise<Record<string, unknown>>) | undefined;
+    heavy?: HeavyRun | undefined;
+    /** The seams of the dev verbs a spec replaces (stack/local-run.mjs LocalDeps). */
+    localDeps?: import('./local-run.mjs').LocalDeps | undefined;
 }
 /** The context of one stack invocation (see stackContext). */
 export interface StackContext {
@@ -98,6 +111,8 @@ export interface StackContext {
     readonly fetch: typeof fetch;
     readonly pipe: (source: PipeEnd, sink: PipeEnd) => Promise<PipeResult>;
     readonly sops: Sops;
+    /** The governed runner of heavy commands (the harness CLI hands it in); absent, a dev verb runs them directly. */
+    readonly heavy?: HeavyRun | undefined;
     /** The command-owned SSH connection, set by the Docker transport (docker-transport.mjs). */
     readonly stackSsh?: SshConnection | undefined;
 }

package/stack/context.mjs

@@ -74,6 +74,7 @@ export function stackContext(io = {}) {
         stderr: io.stderr ?? ((text) => process.stderr.write(text)),
         fetch: io.fetch ?? globalThis.fetch,
         pipe: io.pipe ?? realPipe,
+        heavy: io.heavy,
         /** The secret seam: the app's own sops/age tools, created on first use so verbs that open no custody never need them. */
         get sops() {
             const opened = sops ?? io.openSops?.() ?? defaultSops(env, cwd, io.platform ?? process.platform, undefined, io.hostIdentity ?? null);

package/stack/local-cli.d.mts

@@ -0,0 +1,25 @@
+import { isLocalEnvironment } from './local-config.mjs';
+import type { StackContext } from './context.mjs';
+import type { LocalDeps } from './local-run.mjs';
+/** The actions that exist only for a dev environment. */
+export declare const LOCAL_ONLY: readonly ["doctor", "up", "down", "logs"];
+/** The actions a swarm environment already has, which a docker-compose environment answers with its dev implementation. */
+export declare const LOCAL_SHARED: readonly ["prepare", "status", "verify"];
+/** True when `value` is an action only a dev environment has. */
+export declare const isLocalOnly: (value: unknown) => boolean;
+/** True when `value` is an action both kinds have. */
+export declare const isLocalShared: (value: unknown) => boolean;
+/** The parsed flags of a dev action. */
+export interface LocalFlags {
+    env?: string;
+    json: boolean;
+    execute: boolean;
+    timeout?: string;
+    service?: string;
+    lines?: string;
+}
+/** Parses the flags of one dev action; a flag of another action, an unknown flag and a missing value are refusals. */
+export declare function parseLocalFlags(action: string, argv: readonly string[]): LocalFlags;
+/** Runs one dev action and returns the exit code. */
+export declare function runLocal(ctx: StackContext, action: string, argv: readonly string[], deps?: LocalDeps): Promise<number>;
+export { isLocalEnvironment };