@aifabrix/builder 2.40.2 → 2.42.0

package/scripts/install-local.js

@@ -97,6 +97,39 @@ function displaySuccessMessage(currentVersion, newVersion) {
   console.log('Run "aifabrix --version" to verify.');
 }
 
+/**
+ * Run pnpm link --global and npm link from project root (handles pnpm global bin not set).
+ * @param {string} projectRoot - Path to project root
+ * @returns {void}
+ * @throws {Error} If linking fails when pnpm global bin is not configured
+ */
+function runPnpmLink(projectRoot) {
+  let pnpmLinked = false;
+  try {
+    execSync('pnpm link --global', { stdio: 'inherit', cwd: projectRoot });
+    pnpmLinked = true;
+  } catch (pnpmErr) {
+    const msg = (pnpmErr.message || String(pnpmErr));
+    if (msg.includes('global bin directory') || msg.includes('ERR_PNPM_NO_GLOBAL_BIN_DIR')) {
+      console.log(
+        '⚠️  pnpm global bin is not set up. Run "pnpm setup" and add PNPM_HOME to PATH, or we will use npm link.\n'
+      );
+    } else {
+      throw pnpmErr;
+    }
+  }
+  try {
+    execSync('npm link', { stdio: 'inherit', cwd: projectRoot });
+  } catch {
+    if (!pnpmLinked) {
+      console.error(
+        '\n💡 To fix: run "pnpm setup" and add the suggested line to your shell config, then run install:local again.'
+      );
+      throw new Error('Linking failed. pnpm global bin not configured and npm link failed.');
+    }
+  }
+}
+
 /**
  * Install local package globally
  * @returns {void}
@@ -107,31 +140,17 @@ function installLocal() {
   const currentVersion = getCurrentVersion();
 
   console.log(`Detected package manager: ${pm}\n`);
-
-  // Show version comparison
   displayVersionInfo(currentVersion, packageVersion);
-
   console.log('Linking @aifabrix/builder globally...\n');
 
   try {
     const projectRoot = path.join(__dirname, '..');
     if (pm === 'pnpm') {
-      // Update pnpm global.
-      execSync('pnpm link --global', { stdio: 'inherit', cwd: projectRoot });
-      // Also run npm link so npm's global bin points here; often PATH has
-      // npm's global bin before pnpm's, so "aifabrix" would otherwise stay old.
-      try {
-        execSync('npm link', { stdio: 'inherit', cwd: projectRoot });
-      } catch {
-        // npm may not be available or may fail; pnpm link already ran
-      }
+      runPnpmLink(projectRoot);
     } else {
       execSync('npm link', { stdio: 'inherit', cwd: projectRoot });
     }
-
-    // Get new version after linking
     const newVersion = getCurrentVersion();
-
     displaySuccessMessage(currentVersion, newVersion);
   } catch (error) {
     console.error('\n❌ Failed to link package:', error.message);

package/templates/README.md

@@ -70,7 +70,6 @@ Extra workflow steps are located in `templates/github/steps/`. When you use `--g
 - `{{databases}}` - Array of database configurations
 
 ### Build Configuration
-- `{{build.localPort}}` - Local development port (different from Docker port)
 - `{{mountVolume}}` - Volume mount path for local development
 
 ## Usage

package/templates/applications/README.md.hbs

@@ -38,7 +38,7 @@ aifabrix resolve {{appName}}
 aifabrix run {{appName}}
 ```
 
-**Access your app:** http://localhost:{{localPort}}
+**Access your app:** http://localhost:{{port}}
 
 **View logs:**
 ```bash
@@ -118,7 +118,7 @@ aifabrix build {{appName}} --language typescript  # Override language detection
 ### Run Options
 
 ```bash
-aifabrix run {{appName}} --port {{localPort}}     # Override local port
+aifabrix run {{appName}} --port {{port}}     # Override port
 aifabrix run {{appName}} --debug                  # Debug output
 ```
 
@@ -166,7 +166,7 @@ Controller URL and environment (for `deploy`, `app register`, etc.) are set via
 
 - **"Docker not running"** → Start Docker Desktop
 - **"Not logged in"** → Run `aifabrix login` first
-- **"Port already in use"** → Use `aifabrix run {{appName}} --port <port>` or set `build.localPort` in `application.yaml` (default: {{localPort}})
+- **"Port already in use"** → Use `aifabrix run {{appName}} --port <port>` or set `port` in `application.yaml` (default: {{port}})
 - **"Authentication failed"** → Run `aifabrix login` again
 - **"Build fails"** → Check Docker is running and `aifabrix-secrets` in `config.yaml` is configured correctly
 - **"Can't connect"** → Verify infrastructure is running{{#if hasDatabase}} and PostgreSQL is accessible{{/if}}
@@ -203,4 +203,4 @@ aifabrix json {{appName}}
 
 ---
 
-**Application**: {{appName}} | **Port**: {{localPort}} | **Registry**: {{registry}} | **Image**: {{imageName}}:latest
+**Application**: {{appName}} | **Port**: {{port}} | **Registry**: {{registry}} | **Image**: {{imageName}}:latest

package/templates/applications/dataplane/application.yaml

@@ -5,7 +5,7 @@ app:
   description: "AI Fabrix Dataplane is a secure, in-tenant integration and automation layer that supplies governed, normalized, and explainable enterprise data to AI agents. Using CIP as a declarative standard, it enforces RBAC and ABAC, executes integrations, and exposes trusted data via MCP and OpenAPI."
   type: webapp
   language: python  # Explicitly specify Python language
-  version: 1.7.0
+  version: 1.8.0
 
 # Image Configuration
 # Set tag to match your build (e.g. aifabrix build dataplane -t v1.0.0 then tag: v1.0.0)
@@ -48,11 +48,12 @@ authentication:
 # Build Configuration
 # Dataplane builds from published image; context is project root (like miso-controller)
 build:
-  context: ../..                # Docker build context (relative to builder/dataplane/)
+  context: ../..                  # Docker build context (relative to builder/dataplane/)
   dockerfile: builder/dataplane/Dockerfile  # Dockerfile path (relative to project root)
-  envOutputPath: ../../.env     # Copy to repo root for local dev
-  localPort: 3011              # Port for local development (different from Docker port)
-  language: python             # Runtime language for template selection (typescript or python)
+  envOutputPath: ../../.env       # Copy to repo root for local dev
+  localPort: 3011                 # Port for local development (different from Docker port)
+  language: python                # Runtime language for template selection (typescript or python)
+  reloadStart: uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-3001} --reload  # PORT set from port above at run time; default 3001 must match port
 
 # =============================================================================
 # Portal Input Configuration (Deployment Wizard)

package/templates/applications/dataplane/env.template

@@ -1,5 +1,5 @@
 # Environment Variables Template
-# Use kv:// references for secrets (resolved from .aifabrix/secrets.yaml)
+# Use key-value refs (format: kv://secret-key) for secrets (resolved from .aifabrix/secrets.yaml)
 # Use ${VAR} for environment-specific values
 
 # =============================================================================
@@ -7,9 +7,9 @@
 # =============================================================================
 
 # HTTP port for the app
-PORT=3001
-# development | staging | production
-ENVIRONMENT=development
+PORT=${PORT}
+# dev | tst | pro
+ENVIRONMENT=dev
 # Enable debug mode
 DEBUG=false
 # Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL
@@ -28,10 +28,10 @@ API_KEY=kv://miso-controller-api-key-secretKeyVault
 
 # API Configuration
 API_V1_STR=/api/v1
-VERSION=1.7.0
+VERSION=1.8.0
 # Base URL for the dataplane web server (used for default OAuth2 callback URL when redirectUri is omitted)
-DATAPLANE_WEB_SERVER_URL=kv://dataplane-web-server-urlKeyVault
-DATAPLANE_INTERNAL_URL=kv://dataplane-internal-server-urlKeyVault
+DATAPLANE_WEB_SERVER_URL=kv://dataplane-web-server-url
+DATAPLANE_INTERNAL_URL=kv://dataplane-internal-server-url
 
 # CORS Configuration
 ALLOWED_ORIGINS=http://localhost:*
@@ -43,6 +43,11 @@ ENCRYPTION_KEY=kv://secrets-encryptionKeyVault
 # =============================================================================
 # DATABASE CONFIGURATION
 # =============================================================================
+# Multiple-database layout: set all four URL env vars for four separate databases
+# (dataplane, dataplane-vector, dataplane-logs, dataplane-records). If any
+# dedicated URL is unset, that database's tables use DATABASE_URL (main).
+# See docs/DATABASE_TABLES_LOCATION.md for table-to-database mapping.
+# =============================================================================
 
 # Primary app database URL
 DATABASE_URL=kv://databases-dataplane-0-urlKeyVault
@@ -89,16 +94,16 @@ MISO_CLIENTSECRET=kv://dataplane-client-secretKeyVault
 
 # Keycloak Configuration (for OAuth2 endpoints)
 # Public: used by OpenAPI OAuth2 / browser (authorizationUrl, tokenUrl).
-KEYCLOAK_SERVER_URL=kv://keycloak-server-urlKeyVault
+KEYCLOAK_SERVER_URL=kv://keycloak-server-url
 # Internal (same role as MISO_CONTROLLER_URL): future server-side Keycloak (e.g. JWKS). Not used by dataplane today.
-KEYCLOAK_INTERNAL_SERVER_URL=kv://keycloak-internal-server-urlKeyVault
+KEYCLOAK_INTERNAL_SERVER_URL=kv://keycloak-internal-server-url
 KEYCLOAK_REALM=aifabrix
 
 # =============================================================================
 # MISO CONTROLLER CONFIGURATION
 # =============================================================================
 # Public: browser redirects and CORS for client_token; set when controller is behind a different public URL.
-MISO_WEB_SERVER_URL=kv://miso-controller-web-server-urlKeyVault
+MISO_WEB_SERVER_URL=kv://miso-controller-web-server-url
 # Internal: server-to-controller API calls (auth, pipeline, status, RBAC).
 MISO_CONTROLLER_URL=http://${MISO_HOST}:${MISO_PORT}
 

package/templates/applications/dataplane/rbac.yaml

@@ -38,8 +38,8 @@ roles:
 permissions:
   # Credential management
   - name: "credential:create"
-    roles: ["aifabrix-platform-admin"]
-    description: "Create credentials"
+    roles: ["aifabrix-platform-admin", "aifabrix-deployment-admin", "aifabrix-developer"]
+    description: "Create credentials (and store kv:// secrets for upload/publish)"
   
   - name: "credential:read"
     roles: ["aifabrix-platform-admin", "aifabrix-security-admin", "aifabrix-compliance-admin", "aifabrix-observer"]

package/templates/applications/keycloak/env.template

@@ -39,6 +39,8 @@ KC_HEALTH_ENABLED=true
 # Expose health endpoints on main HTTP port (like Keycloak 24.0)
 # Set to false to expose on main port instead of management port (9000)
 KC_HTTP_MANAGEMENT_HEALTH_ENABLED=false
+# Single-instance: use local cache so /health/ready passes (avoids Infinispan cluster check)
+KC_CACHE=local
 
 # =============================================================================
 # DATABASE CONFIGURATION

package/templates/applications/miso-controller/application.yaml

@@ -47,6 +47,7 @@ build:
   envOutputPath: ../../packages/miso-controller/.env # Copy .env to repo root for local dev (relative to builder/) (if null, no .env file is copied) (if empty, .env file is copied to repo root)
   localPort: 3010 # Port for local development (different from Docker port)
   language: typescript # Runtime language for template selection (typescript or python)
+  reloadStart: pnpm run start:reload # When running with --reload
 
 # =============================================================================
 # Portal Input Configuration (Deployment Wizard)

package/templates/applications/miso-controller/env.template

@@ -49,8 +49,8 @@ ONBOARDING_CREATE_DEV_ENV=true
 
 # NODE_ENV: production for Docker (serves pre-built static files), development for local dev
 # In Docker, this should be production to prevent Vite dev server initialization
-NODE_ENV=${NODE_ENV}
-PORT=${MISO_PORT}
+NODE_ENV=dev
+PORT=${PORT}
 AUTO_CREATE_TABLES=true
 FAST_STARTUP=false
 ALLOWED_ORIGINS=http://localhost:*
@@ -59,7 +59,7 @@ ENABLE_API_DOCS=true
 # Rate Limiting Configuration (for local development)
 # Set DISABLE_RATE_LIMIT=true to disable rate limiting entirely (local development only)
 DISABLE_RATE_LIMIT=true
-# RATE_LIMIT_WINDOW_MS=900000  # 15 minutes in milliseconds (default: 900000)
+# RATE_LIMIT_WINDOW_MS=600000  # 10 minutes in milliseconds (default: 600000)
 # RATE_LIMIT_MAX=100           # Max requests per window (default: 100)
 
 # Package Version (auto-set by npm/pnpm, optional override)
@@ -105,13 +105,12 @@ REDIS_PERMISSIONS_TTL=900
 # (KeycloakConfiguration + Application url/internalUrl). Sync env->DB at startup via
 # sync-application-urls-from-env.service.
 #
-# Azure Entra SSO during onboarding: true = skip (default), false = onboard Azure Entra SSO client in Keycloak
-# (onboarding Azure SSO requires Entra admin consent and Azure app credentials).
-KEYCLOAK_SKIP_AZURE_ENTRA_SSO=false
+# NOTE: Do NOT onboard Azure Entra SSO in Keycloak during onboarding (skipAzureEntraSso=true).
+# KEYCLOAK_SKIP_AZURE_ENTRA_SSO=false
 
 KEYCLOAK_REALM=aifabrix
-KEYCLOAK_SERVER_URL=kv://keycloak-server-urlKeyVault
-KEYCLOAK_INTERNAL_SERVER_URL=kv://keycloak-internal-server-urlKeyVault
+KEYCLOAK_SERVER_URL=kv://keycloak-server-url
+KEYCLOAK_INTERNAL_SERVER_URL=kv://keycloak-internal-server-url
 KEYCLOAK_CLIENT_ID=miso-controller
 KEYCLOAK_CLIENT_SECRET=kv://keycloak-client-secretKeyVault
 KEYCLOAK_ADMIN_USERNAME=admin
@@ -279,6 +278,9 @@ JWT_SECRET=kv://miso-controller-jwt-secretKeyVault
 # When API_KEY is set, a matching Bearer token bypasses OAuth2 validation
 API_KEY=kv://miso-controller-api-key-secretKeyVault
 
+# NPM token for private package (npmjs.org)
+NPM_TOKEN=kv://npm-token-secretKeyVault
+
 # =============================================================================
 # MISO CONTROLLER CONFIGURATION
 # =============================================================================
@@ -287,8 +289,8 @@ API_KEY=kv://miso-controller-api-key-secretKeyVault
 # Used to generate correct server URLs in OpenAPI spec and Keycloak callback URLs
 # For Docker: use localhost with mapped port (e.g., localhost:3100)
 # For production: use public domain (e.g., https://miso.example.com)
-MISO_WEB_SERVER_URL=kv://miso-controller-web-server-urlKeyVault
-MISO_CONTROLLER_URL=kv://miso-controller-internal-server-urlKeyVault
+MISO_WEB_SERVER_URL=kv://miso-controller-web-server-url
+MISO_CONTROLLER_URL=kv://miso-controller-internal-server-url
 
 # MISO Environment Configuration (miso, dev, tst, pro)
 MISO_ENVIRONMENT=miso

package/templates/external-system/README.md.hbs

@@ -10,25 +10,29 @@
 
 ## Files
 
-- `application.yaml` – Application configuration with `app` and `externalIntegration` blocks
-- `{{systemKey}}-system.yaml` – External system definition (authentication, OpenAPI/MCP, RBAC)
+- `application{{fileExt}}` – Application configuration with `app` and `externalIntegration` blocks
+- `{{systemKey}}-system{{fileExt}}` – External system definition (authentication, OpenAPI/MCP, RBAC)
 {{#each datasources}}
 - `{{fileName}}` – Datasource: {{displayName}}
 {{/each}}
 - `env.template` – Environment variables template (secrets, API keys)
-- `{{systemKey}}-deploy.json` – Deployment manifest (generated by `aifabrix json`)
+- `{{systemKey}}-deploy.json` – Deployment manifest (generated by `aifabrix json {{appName}}`)
+- `deploy.js` – Deploy script for the integration
+- `wizard.yaml` – Wizard configuration (if created via wizard)
 
 Optional: `rbac.yaml` – Roles and permissions merged into the system when present.
 
 ## Quick Start
 
-### 1. Create External System
-
+Login to your controller
 ```bash
-aifabrix create {{appName}} --type external
+aifabrix auth config --set-controller <url> --set-environment dev
+aifabrix login
 ```
 
-Or use the interactive wizard:
+### 1. Extend External System
+
+Use the interactive wizard to extend your existing system:
 
 ```bash
 aifabrix wizard --app {{appName}}
@@ -38,36 +42,35 @@ aifabrix wizard --app {{appName}}
 
 Edit files in `integration/{{appName}}/`:
 
-- **Authentication**: `{{systemKey}}-system.yaml` (auth type, credentials placeholders)
-- **Field mappings**: `{{systemKey}}-datasource-*.yaml` (dimensions, attributes, operations)
+- **Authentication**: `{{systemKey}}-system{{fileExt}}` (auth type, credentials placeholders)
+- **Field mappings**: `{{systemKey}}-datasource-*-datasource{{fileExt}}` (dimensions, attributes, operations)
+- **Credential and configuration**: `env.template` (security settings and configuration variables)
 
 ### 3. Validate Configuration
 
 ```bash
-aifabrix validate {{appName}} --type external
-```
-
-### 4. Generate Deployment Manifest
-
-```bash
-aifabrix json {{appName}} --type external
+aifabrix validate {{appName}}
 ```
 
-This creates `{{systemKey}}-deploy.json` in `integration/{{appName}}/`.
-
-### 5. Deploy
+### 4. Repair Deployment Manifest
 
-Controller URL and environment are read from config. Configure and log in first:
+**Run repair regularly.** It keeps naming conventions, filenames, and the deployment manifest aligned with AI Fabrix platform best practices. Use it after editing datasources, env.template, or system config—and run it often to catch drift early.
 
 ```bash
-aifabrix auth config --set-controller <url> --set-environment dev
-aifabrix login
+aifabrix repair {{appName}}
 ```
 
-Then deploy:
+Options:
+  --dry-run   Report changes only; do not write
+  --rbac      Ensure RBAC permissions per datasource and add default Admin/Reader roles if none exist
+  --expose    Set exposed.attributes on each datasource to all fieldMappings.attributes keys
+  --sync      Add default sync section to datasources that lack it
+  --test      Generate testPayload.payloadTemplate and testPayload.expectedResult from attributes
+
+### 5. Upload to dataplane
 
 ```bash
-aifabrix deploy {{appName}}
+aifabrix upload {{appName}}
 ```
 
 ## Testing
@@ -84,6 +87,43 @@ aifabrix test {{appName}}
 aifabrix test-integration {{appName}}
 ```
 
+### End-to-end Tests (Via Dataplane)
+
+```bash
+aifabrix test-e2e {{appName}}
+```
+
+Options:
+  -e, --env <env>  Environment: dev, tst, or pro (builder: dev/tst for container)
+  -v, --verbose    Show detailed step output and poll progress
+  --debug          Include debug output and write log to integration/{{appName}}/logs/
+  --no-async       Use sync mode (no polling); single POST per datasource
+
+### E2E tests per datasource
+
+To run a full E2E test for a single datasource (config, credential, sync, data, CIP), use `aifabrix datasource test-e2e` with the datasource key and app:
+
+{{#if hasDatasources}}
+```bash
+{{#each datasources}}
+# {{displayName}}
+aifabrix datasource test-e2e {{datasourceKey}} --app {{../appName}}
+
+{{/each}}
+```
+{{/if}}
+
+Options:
+  -a, --app {{appName}}              App key (default: resolve from cwd if inside integration/{{appName}}/)
+  -e, --env <env>                    Environment: dev, tst, or pro
+  -v, --verbose                      Show detailed step output and poll progress
+  --debug                            Include debug output and write log to integration/{{appName}}/logs/
+  --test-crud                        Enable CRUD lifecycle test (body testCrud: true)
+  --record-id <id>                   Record ID for test (body recordId)
+  --no-cleanup                       Disable cleanup after test (body cleanup: false)
+  --primary-key-value <value|@path>  Primary key value or path to JSON file (e.g. @pk.json) for body primaryKeyValue
+  --no-async                         Use sync mode (no polling); single POST, no asyncRun
+
 ## Deployment
 
 Deploy via miso-controller pipeline (same as regular apps). Auth and controller come from `aifabrix login` and `aifabrix auth config`:
@@ -94,6 +134,6 @@ aifabrix deploy {{appName}}
 
 ## Troubleshooting
 
-- **Validation errors**: Run `aifabrix validate {{appName}} --type external` to see schema and manifest errors.
+- **Validation errors**: Run `aifabrix validate {{appName}}` to see schema and manifest errors.
 - **Deployment / auth**: Run `aifabrix auth config --set-controller <url> --set-environment <env>` and `aifabrix login` before `aifabrix deploy`.
 - **File not found**: Run commands from the project root (where `package.json` and `integration/` live).

package/templates/external-system/deploy.js.hbs

@@ -10,6 +10,8 @@ const { execSync } = require('child_process');
 const path = require('path');
 
 const scriptDir = __dirname;
+// Project root (repo containing integration/ and builder/) so deploy/test-integration resolve app paths correctly
+const projectRoot = path.join(scriptDir, '..', '..');
 const appKey = '{{systemKey}}';
 const env = process.env.ENVIRONMENT || 'dev';
 // Controller URL: from config (aifabrix auth config) or set CONTROLLER env before running
@@ -57,12 +59,12 @@ run('aifabrix validate "' + path.join(scriptDir, '{{this}}') + '"');
 console.log('✅ Validation passed');
 
 console.log('🚀 Deploying ' + appKey + '...');
-run('aifabrix deploy ' + appKey);
+run('aifabrix deploy ' + appKey, { cwd: projectRoot });
 console.log('✅ Deployment complete');
 
 if (process.env.RUN_TESTS !== 'false') {
   console.log('🧪 Running integration tests...');
-  run('aifabrix test-integration ' + appKey);
+  run('aifabrix test-integration ' + appKey, { cwd: projectRoot });
   console.log('✅ Tests passed');
 }