@aifabrix/builder 2.44.5 → 2.44.6

package/lib/utils/validation-run-poll.js

@@ -34,8 +34,8 @@ function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
 }
 
-function maybeLogPollProgress(envelope, verbosePoll, lastProgressLogAtRef) {
-  if (!verbosePoll || !envelope || typeof envelope !== 'object') return;
+function maybeLogPollProgress(envelope, verbosePoll, lastProgressLogAtRef, skipBecauseUi) {
+  if (skipBecauseUi || !verbosePoll || !envelope || typeof envelope !== 'object') return;
   const now = Date.now();
   if (now - lastProgressLogAtRef[0] < 5000) return;
   lastProgressLogAtRef[0] = now;
@@ -57,6 +57,21 @@ function isTerminalReportCompleteness(envelope) {
   return envelope.reportCompleteness === 'full';
 }
 
+function emitPollProgressLine(
+  envelope,
+  verbosePoll,
+  lastProgressLogAtRef,
+  onPollProgress,
+  attempt,
+  deadline
+) {
+  const hasPollUi = typeof onPollProgress === 'function';
+  maybeLogPollProgress(envelope, verbosePoll, lastProgressLogAtRef, hasPollUi);
+  if (hasPollUi) {
+    onPollProgress(envelope, attempt, { deadlineMs: deadline });
+  }
+}
+
 /**
  * Poll until reportCompleteness === 'full' or budget exhausted.
  * @async
@@ -66,7 +81,8 @@ function isTerminalReportCompleteness(envelope) {
  * @param {string} opts.testRunId
  * @param {number} opts.budgetMs - Remaining wall-clock budget for polls only (POST excluded)
  * @param {typeof getValidationRunWithTransportRetry} [opts.fetchRun] - Inject for tests (default: GET with transport retry)
- * @param {boolean} [opts.verbosePoll] - Log throttled progress (plan §3.13)
+ * @param {boolean} [opts.verbosePoll] - Log throttled progress (plan §3.13); skipped when `onPollProgress` is set
+ * @param {Function|null} [opts.onPollProgress] - `(envelope, attemptIndex, { deadlineMs })` each non-terminal poll
  * @param {number} [opts.pollRequestTimeoutMs] - Per-GET HTTP timeout (match validation aggregate budget)
  * @returns {Promise<{ envelope: Object|null, timedOut: boolean, lastApiResult: Object|null }>}
  */
@@ -78,7 +94,8 @@ async function pollValidationRunUntilComplete(opts) {
     budgetMs,
     fetchRun = getValidationRunWithTransportRetry,
     verbosePoll = false,
-    pollRequestTimeoutMs
+    pollRequestTimeoutMs,
+    onPollProgress = null
   } = opts;
   const pollTransportOpts =
     Number.isFinite(pollRequestTimeoutMs) && pollRequestTimeoutMs > 0
@@ -100,7 +117,14 @@ async function pollValidationRunUntilComplete(opts) {
       return { envelope, timedOut: false, lastApiResult };
     }
 
-    maybeLogPollProgress(envelope, verbosePoll, lastProgressLogAtRef);
+    emitPollProgressLine(
+      envelope,
+      verbosePoll,
+      lastProgressLogAtRef,
+      onPollProgress,
+      attempt,
+      deadline
+    );
 
     const delay = Math.min(nextPollDelayMs(attempt), Math.max(0, deadline - Date.now()));
     attempt += 1;

package/lib/utils/with-muted-logger.js

@@ -0,0 +1,53 @@
+/**
+ * @fileoverview Temporarily mute logger.log output for guided UX flows.
+ *
+ * Used by guided installer-style commands (e.g. up-platform default mode) to avoid
+ * streaming orchestration mechanics while preserving errors and warnings.
+ */
+
+'use strict';
+
+const logger = require('./logger');
+
+/**
+ * Run a function while muting logger.log/info.
+ *
+ * - logger.error and logger.warn are preserved.
+ * - An optional allowlist can let specific messages through (rare).
+ *
+ * @template T
+ * @param {() => Promise<T>} fn
+ * @param {{ allow?: ((...args: any[]) => boolean) }} [opts]
+ * @returns {Promise<T>}
+ */
+async function withMutedLogger(fn, opts = {}) {
+  const original = {
+    log: logger.log,
+    info: logger.info
+  };
+
+  const allow = typeof opts.allow === 'function' ? opts.allow : null;
+
+  const muted = (...args) => {
+    try {
+      if (allow && allow(...args)) {
+        return original.log(...args);
+      }
+    } catch {
+      // ignore allow errors; treat as muted
+    }
+    return undefined;
+  };
+
+  logger.log = muted;
+  logger.info = muted;
+  try {
+    return await fn();
+  } finally {
+    logger.log = original.log;
+    logger.info = original.info;
+  }
+}
+
+module.exports = { withMutedLogger };
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aifabrix/builder",
-  "version": "2.44.5",
+  "version": "2.44.6",
   "description": "AI Fabrix Local Fabric & Deployment SDK",
   "main": "lib/index.js",
   "bin": {

package/templates/applications/dataplane/application.yaml

@@ -8,7 +8,7 @@ app:
   version: 1.9.5
 
 # Image Configuration
-# Set tag to match your build (e.g. aifabrix build dataplane -t v1.0.0 then tag: v1.0.0)
+# Set tag to match your build (e.g. aifabrix build dataplane -t 1.0.0 then tag: 1.0.0)
 # Registry is required so the controller can pull the image (avoids "docker: not found" on the controller host).
 image:
   name: aifabrix/dataplane

package/templates/applications/dataplane/rbac.yaml

@@ -232,19 +232,19 @@ permissions:
     roles: ["aifabrix-platform-admin", "aifabrix-security-admin", "aifabrix-compliance-admin", "aifabrix-observer"]
     description: "Read group information"
   
-  # OpenAPI file management
-  - name: "openapi-file:read"
+  # OpenAPI / MCP spec bundle (mounted specs under /api/v1/specs)
+  - name: "spec:read"
     roles: ["aifabrix-platform-admin", "aifabrix-security-admin", "aifabrix-deployment-admin", "aifabrix-compliance-admin", "aifabrix-developer", "aifabrix-observer"]
-    description: "Read OpenAPI files"
-  
-  - name: "openapi-file:update"
+    description: "Read OpenAPI/MCP spec bundles"
+
+  - name: "spec:update"
     roles: ["aifabrix-platform-admin", "aifabrix-developer"]
-    description: "Update OpenAPI files"
-  
-  - name: "openapi-file:delete"
+    description: "Update OpenAPI/MCP spec bundles (uploaded or user-owned)"
+
+  - name: "spec:delete"
     roles: ["aifabrix-platform-admin", "aifabrix-developer"]
-    description: "Delete OpenAPI files"
-  
+    description: "Delete OpenAPI/MCP spec bundles (user-owned only; internal specs are not deletable via API)"
+
   # External data source write operations
   - name: "external-data-source:write"
     roles: ["aifabrix-platform-admin", "aifabrix-developer"]

package/templates/applications/keycloak/env.template

@@ -23,15 +23,17 @@ KC_HTTP_RELATIVE_PATH=url://vdir-public
 # must use the PUBLIC URL as issuer in all tokens so they match what the
 # controller expects (KEYCLOAK_SERVER_URL).
 # - Users log in via http://localhost:${KEYCLOAK_PUBLIC_PORT} (browser/CLI)
-# - Server calls Keycloak at http://keycloak:8080 for token exchange and refresh
+# - Server calls Keycloak at url://keycloak-internal for token exchange and refresh
 # - Controller sends Host: localhost:${KEYCLOAK_PUBLIC_PORT} so Keycloak validates issuer
 #   against public URL (requires KC_HOSTNAME_BACKCHANNEL_DYNAMIC=true)
 # When KC_HOSTNAME_BACKCHANNEL_DYNAMIC=true, hostname must be a full URL.
-# Use host-only origin (no /auth); KC_HTTP_RELATIVE_PATH carries the front-door path (url://vdir-public).
-# Hostname v2: port belongs in KC_HOSTNAME (url://host-public expands to e.g. http://localhost:8182 or
-# https://devNN.example.com). Do not set KC_HOSTNAME_PORT (deprecated v1; triggers Quarkus warnings).
-# KEYCLOAK_PUBLIC_PORT = application.yaml `port` (host-published) + dev×100; used by other apps / docs.
-KC_HOSTNAME=url://host-public
+# KC_HTTP_RELATIVE_PATH carries the runtime base path (url://vdir-public), and application.yaml
+# frontDoorRouting exposes the same path for internal server-to-server URLs when enabled.
+# Hostname v2: use a full public URL so Keycloak generates redirects that preserve the /auth base path.
+# `url://public` expands to the full front-door URL (including /auth when Traefik + frontDoorRouting.enabled are on).
+# NOTE: Prefer KC_HOSTNAME (not KC_HOSTNAME_URL). hostname-url triggers legacy hostname v1 warnings and, depending on
+# runtime, may not be treated as an active hostname for hostname-backchannel-dynamic.
+KC_HOSTNAME=url://public
 # nginx / Traefik send X-Forwarded-*; required when using an edge proxy (Keycloak 26+).
 KC_PROXY_HEADERS=xforwarded
 # Required for Host header to work: Keycloak resolves backchannel URL from request headers

package/templates/applications/miso-controller/application.yaml

@@ -7,6 +7,8 @@ app:
   version: '1.9.5'
 
 # Image Configuration
+# Set tag to match your build (e.g. aifabrix build miso-controller -t 1.0.0 then tag: 1.0.0)
+# Registry is required so the controller can pull the image (avoids "docker: not found" on the controller host).
 image:
   name: aifabrix/miso-controller
   tag: latest
@@ -57,6 +59,11 @@ build:
   language: typescript # Runtime language for template selection (typescript or python)
   reloadStart: pnpm run start:reload # When running with --reload
 
+# Repository Configuration (pipeline validate: REPOSITORY_URL_MISMATCH)
+repository:
+  enabled: true
+  repositoryUrl: https://github.com/aifabrix/aifabrix-miso
+
 # =============================================================================
 # Portal Input Configuration (Deployment Wizard)
 # =============================================================================

package/templates/applications/miso-controller/env.template

@@ -315,7 +315,7 @@ NPM_TOKEN=kv://BASH_NPM_TOKEN
 # url://public includes front-door path from application.yaml (e.g. /controller).
 MISO_WEB_SERVER_URL=url://public
 MISO_CONTROLLER_URL=url://internal
-
+MISO_RELATIVE_PATH=url://vdir-public
 # MISO Environment Configuration (miso, dev, tst, pro)
 MISO_ENVIRONMENT=miso
 

package/templates/applications/miso-controller/rbac.yaml

@@ -35,22 +35,22 @@ roles:
     groups: ['AI-Fabrix-Observers']
 
 permissions:
-  # Service User Management
-  - name: 'service-user:create'
+  # Integration clients (Keycloak OIDC clients + control-plane metadata)
+  - name: 'integration-client:create'
     roles: ['aifabrix-platform-admin', 'aifabrix-security-admin']
-    description: 'Create service users and API clients'
+    description: 'Create integration clients and Keycloak OIDC clients'
 
-  - name: 'service-user:read'
+  - name: 'integration-client:read'
     roles: ['aifabrix-platform-admin', 'aifabrix-security-admin', 'aifabrix-observer']
-    description: 'View service users and their configurations'
+    description: 'View integration clients and their configurations'
 
-  - name: 'service-user:update'
+  - name: 'integration-client:update'
     roles: ['aifabrix-platform-admin', 'aifabrix-security-admin']
-    description: 'Update service user configurations and regenerate secrets'
+    description: 'Update integration client configuration and regenerate secrets'
 
-  - name: 'service-user:delete'
+  - name: 'integration-client:delete'
     roles: ['aifabrix-platform-admin', 'aifabrix-security-admin']
-    description: 'Deactivate service users'
+    description: 'Deactivate integration clients'
 
   # User Management
   - name: 'users:create'

package/templates/external-system/README.md.hbs

@@ -2,147 +2,119 @@
 
 {{description}}
 
-## System Information
+## At a glance
 
-- **System Key**: `{{systemKey}}`
-- **System Type**: `{{systemType}}`
-- **Datasources**: {{datasourceCount}}
+| | |
+| --- | --- |
+| **System key** | `{{systemKey}}` |
+| **Type** | `{{systemType}}` |
+| **Datasources** | {{datasourceCount}} |
 
 ## Files
 
-- `application{{fileExt}}` – Application configuration with `app` and `externalIntegration` blocks
-- `{{systemKey}}-system{{fileExt}}` – External system definition (authentication, OpenAPI/MCP, RBAC)
+| File | Purpose |
+| --- | --- |
+| `application{{fileExt}}` | App manifest and `externalIntegration` |
+| `{{systemKey}}-system{{fileExt}}` | Auth, API definition, roles |
 {{#each datasources}}
-- `{{fileName}}` – Datasource: {{displayName}}
+| `{{fileName}}` | Datasource: {{displayName}} (**capabilities** here) |
 {{/each}}
-- `env.template` – Environment variables template (secrets, API keys)
-- `{{systemKey}}-deploy.json` – Deployment manifest (generated by `aifabrix json {{appName}}`)
-- `deploy.js` – Deploy script for the integration
-- `wizard.yaml` – Wizard configuration (if created via wizard)
+| `env.template` | Secrets and env placeholders |
+| `{{systemKey}}-deploy.json` | Deploy manifest (`aifabrix json {{appName}}`) |
+| `deploy.js` | Deploy helper |
+| `wizard.yaml` | Wizard input |
+| `{{rbacOptionalFile}}` | Roles and permissions |
 
-Optional: `{{rbacOptionalFile}}` – Roles and permissions merged into the system when
-present.
+## Typical workflow
 
-## Quick Start
+1. **Login** — `aifabrix login` (controller URL set via `aifabrix auth config` if needed).
+2. **Change config** — edit JSON/YAML under `integration/{{appName}}/`, or extend with `aifabrix wizard --app {{appName}}`.
+3. **Adjust operations** — use `aifabrix datasource capability …` to copy, add, or remove **capabilities** (see below).
+4. **Check locally** — `aifabrix validate {{appName}}` and `aifabrix datasource validate` on files you touched.
+5. **Align manifest** — `aifabrix repair {{appName}}` after bigger edits.
+6. **Publish** — `aifabrix upload {{appName}}` or `aifabrix deploy {{appName}}`.
 
-Login to your controller
+Rollback an experiment: `datasource capability remove …` → validate → upload/deploy again.
 
-```bash
-aifabrix auth config --set-controller URL --set-environment dev
-aifabrix login
-```
+## Capabilities (per datasource)
 
-### 1. Extend External System
+Each `*-datasource-*` file lists **capabilities**: named slices that tie together HTTP/API definitions and execution steps. Use the CLI to clone or drop them safely instead of editing huge JSON by hand.
 
-Use the interactive wizard to extend your existing system:
+**After any change:** run `aifabrix datasource validate <file-or-key>` (or `aifabrix validate {{appName}}` for the whole app).
 
-```bash
-aifabrix wizard --app {{appName}}
-```
+| Command | What it does |
+| --- | --- |
+| `datasource capability copy` | Clone one capability to a new name (`--from` / `--as`). Supports `--dry-run`, `--overwrite`, backups. If `exposed.profiles.<from>` exists, it’s copied to `exposed.profiles.<as>`. |
+| `datasource capability create` (or `add`) | New capability from `--from`, `--template`, or `--openapi-operation`. |
+| `datasource capability remove` | Remove one capability (`--capability`; optional `--profile`). Use `--dry-run` first. |
+| `datasource capability validate` | Schema check for the file or one `--capability`. |
+| `datasource capability diff` | Compare two files for one capability. |
+| `datasource capability edit` | Edit one capability’s API definition, runtime steps, or exposure profile in your editor (TTY). |
+| `datasource capability relate` | Link this datasource to another (foreign-key style metadata). See `--help` for flags. |
 
-### 2. Configure Authentication and Datasources
+Short alias (recommended): `af ds cap <command> …`  
+Full command: `aifabrix datasource capability <command> …`
 
-Edit files in `integration/{{appName}}/`:
-
-- **Authentication**: `{{systemKey}}-system{{fileExt}}` (auth type, credentials placeholders)
-- **Field mappings**: `{{systemKey}}-datasource-*{{fileExt}}` (dimensions, attributes, operations)
-- **Credential and configuration**: `env.template` (security settings and configuration variables)
-{{#if secretPaths}}{{#if secretPaths.length}}
-
-### Secrets
-
-Secrets are resolved from `.aifabrix` or key vault. Set them with:
+Examples (use the first datasource from the Files table):
 
 ```bash
-{{#each secretPaths}}
-aifabrix secret set {{path}} VALUE   # {{description}}
-{{/each}}
-```
-{{/if}}{{/if}}
-
-### 3. Validate configuration (local only)
-
-`aifabrix validate` runs **on your machine**: it loads files under
-`integration/{{appName}}/`, checks them against the application and
-external-system / external-datasource JSON schemas, and runs related manifest rules.
-It does **not** call the dataplane or any other remote API.
-
-```bash
-aifabrix validate {{appName}}
-```
+# Preview cloning "create" to "createBasic" (no write)
+af ds cap copy {{#if hasDatasources}}{{datasources.[0].datasourceKey}}{{else}}<datasource-key>{{/if}} --from create --as createBasic --dry-run
 
-Use this before upload or deploy to catch structural and policy errors early.
+# Apply the copy, then validate
+af ds cap copy {{#if hasDatasources}}{{datasources.[0].datasourceKey}}{{else}}<datasource-key>{{/if}} --from create --as createBasic
+af ds validate {{#if hasDatasources}}{{datasources.[0].datasourceKey}}{{else}}<datasource-key>{{/if}}
 
-### 4. Repair Deployment Manifest
-
-**Run repair regularly.** It keeps naming conventions, filenames, and the deployment
-manifest aligned with AI Fabrix platform best practices. Use it after editing
-datasources, env.template, or system config—and run it often to catch drift early.
-
-```bash
-aifabrix repair {{appName}}
+# Remove a capability you no longer need
+af ds cap remove {{#if hasDatasources}}{{datasources.[0].datasourceKey}}{{else}}<datasource-key>{{/if}} --capability createBasic --dry-run
 ```
 
-Options:
+For flags, run `af ds cap <command> --help`.
 
-- `--auth METHOD` — Set authentication method (`oauth2`, `aad`, `apikey`, `basic`,
-     `queryParam`, `oidc`, `hmac`, `none`); updates system file and env.template
-- `--dry-run` — Report changes only; do not write
-- `--rbac` — Ensure RBAC permissions per datasource and add default Admin/Reader roles if none exist
-- `--expose` — Set `exposed.attributes` on each datasource to all `fieldMappings.attributes` keys
-- `--sync` — Add default sync section to datasources that lack it
-- `--test` — Generate `testPayload.payloadTemplate` and `testPayload.expectedResult` from attributes
+## Single datasource lifecycle (production-ready)
 
-### 5. Upload to dataplane
+Start with one datasource and iterate until it’s stable.
 
 ```bash
-aifabrix upload {{appName}}
+af ds validate {{#if hasDatasources}}{{datasources.[0].datasourceKey}}{{else}}<datasource-key>{{/if}}
+af ds test {{#if hasDatasources}}{{datasources.[0].datasourceKey}}{{else}}<datasource-key>{{/if}} --debug --sync
+af ds test-integration {{#if hasDatasources}}{{datasources.[0].datasourceKey}}{{else}}<datasource-key>{{/if}} --sync
+af ds test-e2e {{#if hasDatasources}}{{datasources.[0].datasourceKey}}{{else}}<datasource-key>{{/if}} --debug --sync
 ```
 
-## Testing
-
-| Command | Where it runs | Calls dataplane? |
-| --- | --- | --- |
-| `aifabrix validate {{appName}}` | Local (schemas / files) | No |
-| `aifabrix test {{appName}}` | Local (manifest / payload checks) | No |
-| `aifabrix test-integration {{appName}}` | Auth + dataplane | Yes |
-| `aifabrix test-e2e {{appName}}` | Auth + dataplane | Yes |
-| Datasource `test` / `test-integration` / `test-e2e` | Auth + dataplane | Yes |
+{{#if secretPaths}}{{#if secretPaths.length}}
 
-So: **validate** (and **`test`**) stay offline; **all integration and E2E test
-commands** exercise the system **via the API** (after login and a reachable
-dataplane).
+## Secrets
 
-### Local checks (no API)
+Store values the CLI expects (no secrets in Git):
 
 ```bash
-aifabrix validate {{appName}}
-aifabrix test {{appName}}
+{{#each secretPaths}}
+aifabrix secret set {{path}} <your value>   # {{description}}
+{{/each}}
 ```
+{{/if}}{{/if}}
 
-### Integration tests (dataplane API)
-
-```bash
-aifabrix test-integration {{appName}}
-```
+## Repair
 
-### End-to-end tests (dataplane API)
+Keeps filenames, lists, and deploy manifest in sync after manual edits.
 
 ```bash
-aifabrix test-e2e {{appName}}
+aifabrix repair {{appName}}
 ```
 
-Options:
+Useful flags: `--dry-run`, `--auth <method>`, `--rbac`, `--expose`, `--sync`, `--test`.
 
-- `-e`, `--env ENV` — Environment: `dev`, `tst`, or `pro` (builder: dev/tst for container)
-- `-v`, `--verbose` — Show detailed step output and poll progress
-- `-d`, `--debug` — Include debug output and write log to `integration/{{appName}}/logs/`
-- `--no-async` — Use sync mode (no polling); single POST per datasource
+## Validate and test
 
-### E2E tests per datasource
+| Command | Network |
+| --- | --- |
+| `aifabrix validate {{appName}}` | Off — schemas and files only |
+| `aifabrix test {{appName}}` | Off — local checks |
+| `aifabrix test-integration {{appName}}` | On — needs login + dataplane |
+| `aifabrix test-e2e {{appName}}` | On — full pipeline per datasource |
 
-To run a full E2E test for a single datasource (config, credential, sync, data,
-CIP), use `aifabrix datasource test-e2e` with the datasource key and app:
+Single datasource E2E:
 
 {{#if hasDatasources}}
 ```bash
@@ -152,40 +124,28 @@ aifabrix datasource test-e2e {{datasourceKey}} --app {{../appName}}
 
 {{/each}}
 ```
+{{else}}
+```bash
+aifabrix datasource test-e2e <datasource-key> --app {{appName}}
+```
 {{/if}}
 
-Options:
-
-- `-a`, `--app {{appName}}` — App key (default: resolve from cwd if inside `integration/{{appName}}/`)
-- `-e`, `--env ENV` — Environment: `dev`, `tst`, or `pro`
-- `-v`, `--verbose` — Show detailed step output and poll progress
-- `-d`, `--debug` — Include debug output and write log to `integration/{{appName}}/logs/`
-- `--no-run-scenarios` — Skip expanding `testPayload.scenarios` in capacity step
-- `--no-cleanup` — Disable cleanup after test (body cleanup: false)
-- `--primary-key-value VALUE` — Primary key value or path to JSON file (e.g.
-  `@pk.json`) for body `primaryKeyValue`
-- `--no-async` — Use sync mode (no polling); single POST, no asyncRun
-
-## Deployment
+Common E2E flags: `-v` / `--verbose`, `-d` / `--debug`, `--no-async`.
 
-Deploy via miso-controller pipeline (same as regular apps). Auth and controller
-come from `aifabrix login` and `aifabrix auth config`:
+## Deploy
 
 ```bash
 aifabrix deploy {{appName}}
 ```
 
-## Delete
+## Remove app
 
 ```bash
 aifabrix delete {{appName}}
 ```
 
-## Troubleshooting
+## When something fails
 
-- **Local validation errors**: Run `aifabrix validate {{appName}}` (and
-  `aifabrix test {{appName}}`) — these only inspect files on disk, not the dataplane.
-- **Deployment / auth**: Run
-  `aifabrix auth config --set-controller URL --set-environment ENV` and
-  `aifabrix login` before `aifabrix deploy`.
-- **File not found**: Run commands from the project root (where `package.json` and `integration/` live).
+- **Validation errors** — Fix JSON/schema issues; run `aifabrix validate {{appName}}` and `aifabrix datasource validate` on the file you changed.
+- **Auth** — `aifabrix auth config` + `aifabrix login` before upload, deploy, or remote tests.
+- **Wrong folder** — Run CLI from the project root (where `integration/{{appName}}/` lives).