enablement-build-template-generate 1.0.16 → 1.0.17

package/README.md

@@ -12,9 +12,9 @@ npx enablement-build-template-generate@latest [flags]
 
 ## How it works
 
-1. **Read config** — Loads a JSON config file, auto-detected from `.github/templates/templated.json` or `.cicd/templates/templated.json` based on the presence of a `.github/` directory.
-2. **Discover packages** — Globs for `package.json`, `version.json`, and `pyproject.toml` under the configured `rootPath`. When multiple manifest files exist in the same folder, the highest-priority one is used (`package.json` → `version.json` → `pyproject.toml`). Results are sorted alphabetically by folder name.
-3. **Expand job template** — For each discovered package, reads the package name from its manifest and fills in placeholders in the job snippet template.
+1. **Detect platform** — Checks for `manifest.json` (preferred) or `templated.json` (legacy) in `.cicd/` (ADO) or `.github/` (GitHub). Defaults to `ado` if none are found, or use `--platform` to override.
+2. **Read config** — If `manifest.json` is present, reads the `templated` section (each package maps to its own job template file). If only `templated.json` is present, reads the legacy `{ rootPath, template }` format and globs for packages automatically.
+3. **Expand job templates** — For each package, reads its job snippet template and fills in placeholders. When using `manifest.json`, metadata (`folderName`, `packageFolder`) comes from `manifest.cicd`; when using `templated.json`, package names are read from each folder's manifest file.
 4. **Stitch** — Joins all expanded job snippets and substitutes them into the document template at the named placeholder.
 5. **Write output** — Writes the final file one level above the template directory (`.github/templates/` → `.github/` or `.cicd/templates/` → `.cicd/`), wrapped in `DO NOT EDIT` banners.
 
@@ -24,55 +24,55 @@ npx enablement-build-template-generate@latest [flags]
 
 | Flag | Default | Description |
 |---|---|---|
-| `--config <path>` | auto-detect | Path to the JSON config file. Defaults to `.github/templates/templated.json` if a `.github/` directory exists, otherwise `.cicd/templates/templated.json`. Resolved relative to the package first, then relative to the working directory. |
-| `--platform <name>` | auto-detect | Force the platform: `ado` (Azure DevOps) or `github` (GitHub Actions). Auto-detected from the presence of a `.github/` directory at the working directory root. |
+| `--platform <name>` | auto-detect | Force the platform: `ado` (Azure DevOps) or `github` (GitHub Actions). Auto-detected from `manifest.json` or `templated.json` presence in `.cicd/` or `.github/`. |
 | `--exclude <list>` | — | Comma- or space-separated list of package folder names to skip during generation. |
 | `--debug` | off | Print verbose output. |
 
 ---
 
-## Config file format
+## Config source
 
-The config file is a JSON object. Each key is the **output filename** (relative to the directory above the template folder). Each value maps placeholder variable names to job configurations.
+The generator supports two config formats, checked in this order:
+
+### 1. `manifest.json` (preferred)
+
+Reads the `templated` section of `{configDir}/manifest.json` (produced by `enablement-build-manifest`). Each key is the **output filename**; each value maps a placeholder variable name to a per-package template mapping.
 
 ```json
 {
-  "ci-build.yaml": {
-    "jobs": {
-      "template": "job-template.yaml",
-      "rootPath": "./packages",
-      "exclude": ["internal-tool"]
+  "templated": {
+    "build.yml": {
+      "PACKAGES": {
+        "my-scope/component-a": "build_node_component.yml",
+        "my-scope/component-b": ["build_node_component.yml", "build_scan_node.yml"],
+        "my-scope/python-lib":  ["build_python_package.yml", "build_scan.yml"]
+      }
     }
   }
 }
 ```
 
-| Field | Description |
-|---|---|
-| `template` | Path to the job snippet template, relative to the config file's directory. |
-| `rootPath` | Root directory to scan for `package.json` files, relative to the config file's directory. |
-| `exclude` | _(optional)_ Array of folder names to skip. |
+Each package name maps to a **single template filename** or an **array of template filenames** (all expanded in order for that package). Template lookup tries the active platform templates directory first (`.cicd/templates/` or `.github/templates/`) and then falls back to the other platform directory if a file is missing.
+
+Package metadata (`folderName`, `packageFolder`) is read from `manifest.cicd[packageName]` and used to fill template placeholders.
+
+### 2. `templated.json` (legacy fallback)
 
-Multiple output files and multiple variables per output file are supported:
+When `manifest.json` is absent, reads `{configDir}/templates/templated.json`. Each entry provides a `rootPath` to glob for packages and a single `template` applied to all discovered packages. Template lookup still uses active-platform-first, then falls back to the other platform templates folder if needed.
 
 ```json
 {
-  "ci-components.yaml": {
-    "componentJobs": {
-      "template": "component-job.yaml",
-      "rootPath": "./components"
-    }
-  },
-  "ci-saas.yaml": {
-    "saasJobs": {
-      "template": "saas-job.yaml",
-      "rootPath": "./saas",
-      "exclude": ["saas-legacy"]
+  "build.yml": {
+    "PACKAGES": {
+      "rootPath": "../../packages",
+      "template": "build_node_component.yml"
     }
   }
 }
 ```
 
+Package names are read from each folder's `package.json`, `version.json`, or `pyproject.toml`.
+
 ---
 
 ## Template placeholders
@@ -82,71 +82,52 @@ Multiple output files and multiple variables per output file are supported:
 The document template (the top-level YAML file) uses `{variableName}` placeholders that are replaced with the fully expanded set of job snippets:
 
 ```yaml
-# ci-build.yaml (document template)
+# build.yml (document template)
 stages:
   - stage: Build
     jobs:
-      {jobs}
+      {PACKAGES}
 ```
 
-### Supported manifest files
-
-Each subfolder is scanned for a version manifest in this priority order:
-
-| Priority | File | Format |
-|---|---|---|
-| 1 | `package.json` | JSON — `"name"` field |
-| 2 | `version.json` | JSON — same shape as `package.json` |
-| 3 | `pyproject.toml` | TOML — `name` field under `[project]` or `[tool.poetry]` |
-
-When multiple manifest files exist in the same folder the highest-priority one is used. If none contains a `name` field the folder name is used as a fallback.
-
 ### Job template
 
 Each job snippet template is expanded once per package. The following placeholders are available:
 
 | Placeholder | Value |
 |---|---|
-| `{name}` | Package folder name (e.g. `my-component`) |
+| `{name}` | Package folder name (from `cicd[pkg].folderName`, e.g. `my-component`) |
 | `{safe}` | Folder name with `-`, `_`, and `.` removed — safe for use as a YAML identifier (e.g. `mycomponent`) |
 | `{package}` | npm package name without scope prefix (e.g. `my-component` from `@scope/my-component`) |
 | `{full}` | Full npm package name including scope (e.g. `@scope/my-component`) |
-| `{path}` | The `rootPath` value with any leading `./` stripped |
-| `{other}` | Expands to a comma-separated list of paths to every **other** package in the same scan — useful for pipeline dependency declarations |
-
-Example job template:
-
-```yaml
-      - job: build_{safe}
-        displayName: Build {name}
-        steps:
-          - script: pnpm --filter {full} build
-```
+| `{path}` | The package folder root (from `cicd[pkg].packageFolder`, e.g. `packages`) |
+| `{other}` | Expands to a comma-separated list of paths to every **other** package in the same variable — useful for pipeline dependency declarations |
 
 ---
 
 ## Directory layout
 
-Templates and config live inside a subfolder; generated files are written one level up.
+Job template files live in the templates subfolder; document templates and generated output files are one level up.
 
-**Azure DevOps** (default when no `.github/` directory):
+**Azure DevOps** (when `.cicd/manifest.json` exists):
 ```
 .cicd/
-├── ci-build.yaml          ← generated output
+├── manifest.json          ← source of truth for templated config (preferred)
+├── build.yml              ← generated output
 └── templates/
-    ├── templated.json     ← config file
-    ├── document.yaml      ← document template
-    └── job.yaml           ← job snippet template
+    ├── build.yml          ← document template
+    ├── templated.json     ← legacy config (used when manifest.json is absent)
+    └── build_node_component.yml  ← job snippet template
 ```
 
-**GitHub Actions** (when `.github/` directory is present):
+**GitHub Actions** (when `.github/manifest.json` exists):
 ```
 .github/
-├── ci-build.yaml          ← generated output
+├── manifest.json          ← source of truth for templated config (preferred)
+├── build.yml              ← generated output
 └── templates/
-    ├── templated.json     ← config file
-    ├── document.yaml      ← document template
-    └── job.yaml           ← job snippet template
+    ├── build.yml          ← document template
+    ├── templated.json     ← legacy config (used when manifest.json is absent)
+    └── build_node_component.yml  ← job snippet template
 ```
 
 ---
@@ -160,12 +141,6 @@ Run the generator as a pipeline step before the build stage:
   displayName: Generate pipeline YAML
 ```
 
-With a custom config location:
-
-```bash
-npx enablement-build-template-generate@latest --config .azuredevops/templates/templated.json
-```
-
 Excluding specific packages:
 
 ```bash
@@ -176,7 +151,7 @@ npx enablement-build-template-generate@latest --exclude legacy-pkg,test-harness
 
 ## Output format
 
-Generated files are wrapped in `DO NOT EDIT` banners. Edit the templates, not the output:
+Generated files are wrapped in `DO NOT EDIT` banners. Edit the templates or re-generate via `manifest`, not the output directly:
 
 ```yaml
 #######################################

package/package.json

@@ -1,17 +1,16 @@
 {
   "name": "enablement-build-template-generate",
-  "version": "1.0.16",
+  "version": "1.0.17",
   "description": "Generates build yaml for mono repos",
   "type": "module",
   "module": "./src/index.mjs",
   "main": "./src/index.mjs",
-  "bin": "./src/index.mjs",
   "dependencies": {
-    "globby": "^14.0.1",
+    "globby": "^16.2.0",
     "user": "^0.0.0"
   },
   "devDependencies": {
-    "jest": "^29.0.0"
+    "jest": "^30.4.2"
   },
   "jest": {
     "testEnvironment": "node",
@@ -25,5 +24,8 @@
     "start": "node src/index.mjs",
     "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
     "test:watch": "node --experimental-vm-modules node_modules/jest/bin/jest.js --watch"
+  },
+  "bin": {
+    "enablement-build-template-generate": "./src/index.mjs"
   }
 }

package/src/index.mjs

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import * as url from 'url';
 import path from 'path';
-import { globby } from 'globby';
 import { readFileSync, writeFileSync, existsSync } from 'fs';
+import { globby } from 'globby';
 
 import {
   stripWhitespace,
@@ -19,87 +19,181 @@ import {
 
 const __dirname = url.fileURLToPath(new URL('.', import.meta.url));
 
-function readManifestName(filePath) {
-  const content = readFileSync(filePath, 'utf8');
-  const fileType = filePath.endsWith('.toml') ? 'toml' : 'json';
-  return parseManifestName(content, fileType);
+// ---------------------------------------------------------------------------
+// Read a job template file, falling back to the alternate platform folder
+// ---------------------------------------------------------------------------
+function readJobTemplate(templateFile, templatePath, fallbackTemplatePath) {
+  const primary = path.join(templatePath, templateFile);
+  if (existsSync(primary)) return readFileSync(primary, 'utf8');
+  if (fallbackTemplatePath) {
+    const fallback = path.join(fallbackTemplatePath, templateFile);
+    if (existsSync(fallback)) return readFileSync(fallback, 'utf8');
+  }
+  throw new Error(`Template file not found: ${templateFile} (looked in ${templatePath}${fallbackTemplatePath ? ' and ' + fallbackTemplatePath : ''})`);
+}
+
+// ---------------------------------------------------------------------------
+// Expand using manifest.json "templated" section (new format)
+// ---------------------------------------------------------------------------
+function expandVarFromManifest(varname, packageMap, cicd, templatePath, fallbackTemplatePath, options) {
+  return new Promise((resolve, reject) => {
+    try {
+      const packageNames = Object.keys(packageMap).filter(pkg => {
+        if (packageMap[pkg] === false) return false;
+        const folderName = (cicd[pkg] || {}).folderName || pkg;
+        return !options.exclude.includes(folderName);
+      });
+
+      const allFolderNames = packageNames.map(pkg => (cicd[pkg] || {}).folderName || pkg);
+
+      const folderCounts = {};
+      packageNames.forEach(pkg => {
+        const f = (cicd[pkg] || {}).packageFolder || 'packages';
+        folderCounts[f] = (folderCounts[f] || 0) + 1;
+      });
+      const commonPackageFolder = Object.keys(folderCounts).sort((a, b) => folderCounts[b] - folderCounts[a])[0] || 'packages';
+      const cwd = path.join(process.cwd(), commonPackageFolder);
+
+      const expandedJobs = packageNames.map(pkgName => {
+        const templateFiles = Array.isArray(packageMap[pkgName])
+          ? packageMap[pkgName]
+          : [packageMap[pkgName]];
+
+        const info = cicd[pkgName] || {};
+        const name = info.folderName || pkgName;
+        const jobRoot = info.packageFolder || 'packages';
+        const safe = makeSafe(name);
+        const packageName = resolvePackageName(pkgName);
+        const metadata = info.metadata || {};
+
+        return templateFiles.map(templateFile => {
+          const jobTemplate = readJobTemplate(templateFile, templatePath, fallbackTemplatePath);
+          return expandJobTemplate(jobTemplate, { name, safe, packageName, fullName: pkgName, jobRoot, metadata });
+        }).join('\n');
+      });
+
+      const jobs = resolveOtherPlaceholders(expandedJobs.join('\n'), allFolderNames, cwd);
+      resolve({ name: varname, text: jobs });
+    } catch (e) {
+      reject(e);
+    }
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Expand using templated.json (legacy format: rootPath + template glob)
+// ---------------------------------------------------------------------------
+function expandVarFromTemplatedJson(varname, jobConfig, templatePath, fallbackTemplatePath, options) {
+  return new Promise((resolve, reject) => {
+    const jobTemplate = readJobTemplate(jobConfig.template, templatePath, fallbackTemplatePath);
+    const cwd = path.join(templatePath, jobConfig.rootPath);
+    const search = buildSearchGlobs(options.exclude);
+
+    globby(search, { cwd }).then((files) => {
+      const folderBest = groupByFolder(files);
+      const allNames = Object.keys(folderBest).sort().filter(n => !options.exclude.includes(n));
+
+      const expandedJobs = allNames.map(name => {
+        const content = readFileSync(path.join(cwd, folderBest[name]), 'utf8');
+        const ext = path.extname(folderBest[name]).substring(1).toLowerCase();
+        const fullName = parseManifestName(content, ext) || name;
+        const safe = makeSafe(name);
+        const packageName = resolvePackageName(fullName);
+        const jobRoot = path.relative(process.cwd(), cwd);
+        return expandJobTemplate(jobTemplate, { name, safe, packageName, fullName, jobRoot, metadata: {} });
+      });
+
+      const jobs = resolveOtherPlaceholders(expandedJobs.join('\n'), allNames, cwd);
+      resolve({ name: varname, text: jobs });
+    }, reject);
+  });
 }
 
+// ---------------------------------------------------------------------------
+// main
+// ---------------------------------------------------------------------------
 async function main(options) {
-  let configFile = path.join(__dirname, options.config);
-  const try1 = configFile;
+  const configDir = options.platform === 'github' ? '.github' : '.cicd';
+  const altConfigDir = options.platform === 'github' ? '.cicd' : '.github';
+  const templatePath = path.join(process.cwd(), `${configDir}/templates`);
+  const fallbackTemplatePath = path.join(process.cwd(), `${altConfigDir}/templates`);
 
-  if (!existsSync(configFile)) {
-    configFile = path.join(options.config);
+  // Try manifest.json first, then fall back to templated.json
+  const manifestPath = path.join(process.cwd(), `${configDir}/manifest.json`);
+  const templatedJsonPath = path.join(process.cwd(), `${configDir}/templates/templated.json`);
+
+  const useManifest = existsSync(manifestPath);
+  const useTemplatedJson = !useManifest && existsSync(templatedJsonPath);
+
+  if (!useManifest && !useTemplatedJson) {
+    throw `enablement-build-template-generate: could not find manifest.json at ${configDir}/manifest.json or templated.json at ${configDir}/templates/templated.json`;
   }
 
-  if (!existsSync(configFile)) {
-    throw `enablement-build-template-generate: could not load config file ${configFile}, also tried ${try1}`;
+  let config;
+  let cicd = {};
+
+  if (useManifest) {
+    const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
+    if (!manifest.templated) {
+      throw `enablement-build-template-generate: manifest.json has no "templated" section`;
+    }
+    config = manifest.templated;
+    cicd = manifest.cicd || {};
+  } else {
+    config = JSON.parse(readFileSync(templatedJsonPath, 'utf8'));
   }
 
-  const templatePath = path.dirname(configFile);
-  const config = JSON.parse(readFileSync(configFile, 'utf8'));
-  const search = buildSearchGlobs(options.exclude);
   const plist = [];
 
   Object.keys(config).forEach((templateFilename) => {
-    plist.push(new Promise(async (primaryResolve, reject) => {
-      const docTemplate = readFileSync(path.join(templatePath, templateFilename), 'utf8');
-      const jlist = [];
-
-      Object.keys(config[templateFilename]).forEach((varname) => {
-        const jobConfig = config[templateFilename][varname];
-        const job = new Promise(async (resolve, reject) => {
-          const jobTemplate = readFileSync(path.join(templatePath, jobConfig.template), 'utf8');
-          const cwd = path.join(templatePath, jobConfig.rootPath);
-          const jobRoot = jobConfig.rootPath.replace(/^[\.\/\\]+/, '');
-          const exclude = jobConfig.exclude || [];
-
-          globby(search, { cwd }).then((images) => {
-            images.sort();
-            const folderBest = groupByFolder(images);
-            const allNames = Object.keys(folderBest).sort()
-              .filter(name => !name.startsWith('_') && !exclude.includes(name));
-
-            const expandedJobs = allNames.map(name => {
-              const fullName = readManifestName(path.join(cwd, folderBest[name].filePath)) || name;
-              const packageName = resolvePackageName(fullName);
-              const safe = makeSafe(name);
-              return expandJobTemplate(jobTemplate, { name, safe, packageName, fullName, jobRoot });
-            });
-
-            let jobs = resolveOtherPlaceholders(expandedJobs.join('\n'), allNames, cwd);
-            resolve({ name: varname, text: jobs });
-          }, reject);
+    plist.push(new Promise((primaryResolve, reject) => {
+      try {
+        const docTemplate = readFileSync(path.join(templatePath, templateFilename), 'utf8');
+        const jlist = [];
+
+        Object.keys(config[templateFilename]).forEach((varname) => {
+          const varConfig = config[templateFilename][varname];
+
+          if (useManifest) {
+            // New format: varConfig is { pkgName: "template.yml" | [...] }
+            jlist.push(expandVarFromManifest(varname, varConfig, cicd, templatePath, fallbackTemplatePath, options));
+          } else {
+            // Legacy format: varConfig is { rootPath, template, exclude? }
+            jlist.push(expandVarFromTemplatedJson(varname, varConfig, templatePath, fallbackTemplatePath, options));
+          }
         });
-        jlist.push(job);
-      });
-
-      Promise.all(jlist).then((results) => {
-        const output = applyVarsToDocTemplate(docTemplate, results);
-        const wrapped = wrapWithBanner(output);
-
-        if (stripWhitespace(wrapped) !== docTemplate) {
-          writeFileSync(path.join(templatePath, `../${templateFilename}`), wrapped, 'utf8');
-        }
 
-        primaryResolve(`generated template ${templateFilename}`);
-      }, reject);
+        Promise.all(jlist).then((results) => {
+          const output = applyVarsToDocTemplate(docTemplate, results);
+          const wrapped = wrapWithBanner(output);
+
+          const outPath = path.join(templatePath, `../${templateFilename}`);
+          const existing = existsSync(outPath) ? readFileSync(outPath, 'utf8') : '';
+          if (stripWhitespace(wrapped) !== stripWhitespace(existing)) {
+            writeFileSync(outPath, wrapped, 'utf8');
+          }
+
+          primaryResolve(`generated template ${templateFilename}`);
+        }, reject);
+      } catch (e) {
+        reject(e);
+      }
     }));
   });
 
   return Promise.all(plist);
 }
 
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
 let options = {
-  config: null,
   platform: null,
   exclude: [],
   debug: false,
 };
 
 let argv = process.argv;
-
 for (let i = 2; i < argv.length; i++) {
   if (argv[i] === '--debug') { options.debug = true; continue; }
   if (argv[i].substring(0, 2) === '--') {
@@ -111,21 +205,25 @@ for (let i = 2; i < argv.length; i++) {
       options[name] = argv[i + 1];
       i++;
     } else {
-      console.error('Expected a known option');
+      console.error(`Expected a known option, got ${argv[i]}`);
       process.exit(1);
     }
   }
 }
 
 if (!options.platform) {
-  const hasGithubConfig = existsSync(path.join(process.cwd(), '.github/templates/templated.json'));
-  const hasAdoConfig = existsSync(path.join(process.cwd(), '.cicd/templates/templated.json'));
-  options.platform = hasGithubConfig ? 'github' : hasAdoConfig ? 'ado' : 'github';
-}
-if (!options.config) {
-  options.config = options.platform === 'github'
-    ? '.github/templates/templated.json'
-    : '.cicd/templates/templated.json';
+  const hasGithubManifest  = existsSync(path.join(process.cwd(), '.github/manifest.json'));
+  const hasAdoManifest     = existsSync(path.join(process.cwd(), '.cicd/manifest.json'));
+  const hasGithubTemplated = existsSync(path.join(process.cwd(), '.github/templates/templated.json'));
+  const hasAdoTemplated    = existsSync(path.join(process.cwd(), '.cicd/templates/templated.json'));
+
+  if (hasGithubManifest || hasGithubTemplated) {
+    options.platform = 'github';
+  } else if (hasAdoManifest || hasAdoTemplated) {
+    options.platform = 'ado';
+  } else {
+    options.platform = 'ado';
+  }
 }
 
 (async () => {

package/src/lib.mjs

@@ -70,14 +70,21 @@ export function makeSafe(name) {
 
 // Expand a job template string for one package, substituting all known placeholders.
 // {other} is replaced with a tagged sentinel {other:name} for later resolution.
-export function expandJobTemplate(template, { name, safe, packageName, fullName, jobRoot }) {
-  return template
+export function expandJobTemplate(template, { name, safe, packageName, fullName, jobRoot, metadata = {} }) {
+  let output = template
     .replace(/{other}/g, `{other:${name}}`)
     .replace(/{path}/g, jobRoot)
     .replace(/{name}/g, name)
     .replace(/{safe}/g, safe)
     .replace(/{package}/g, packageName)
     .replace(/{full}/g, fullName);
+
+  Object.entries(metadata).forEach(([key, value]) => {
+    if (value === undefined || value === null) return;
+    output = output.replace(new RegExp(`\\{${key}\\}`, 'g'), String(value));
+  });
+
+  return output;
 }
 
 // Resolve all {other:name} sentinels in the combined jobs string.