@aifabrix/builder 2.33.1 → 2.33.3

package/lib/utils/parse-image-ref.js

@@ -0,0 +1,27 @@
+/**
+ * Parse full image reference (registry/name:tag or name:tag) into { name, tag }
+ *
+ * @fileoverview Image reference parsing for compose and run overrides
+ * @author AI Fabrix Team
+ * @version 2.0.0
+ */
+
+/**
+ * Parses full image reference (registry/name:tag or name:tag) into { name, tag }
+ * @param {string} imageRef - Full image reference (e.g. myreg/keycloak:v1 or keycloak:latest)
+ * @returns {{ name: string, tag: string }|null} Parsed name and tag, or null if invalid
+ */
+function parseImageOverride(imageRef) {
+  if (!imageRef || typeof imageRef !== 'string') {
+    return null;
+  }
+  const lastColon = imageRef.lastIndexOf(':');
+  if (lastColon <= 0) {
+    return { name: imageRef.trim(), tag: 'latest' };
+  }
+  const name = imageRef.substring(0, lastColon).trim();
+  const tag = imageRef.substring(lastColon + 1).trim() || 'latest';
+  return { name, tag };
+}
+
+module.exports = { parseImageOverride };

package/lib/utils/paths.js

@@ -30,18 +30,33 @@ function safeHomedir() {
   return process.env.HOME || process.env.USERPROFILE || '/';
 }
 
+/**
+ * Returns the path to the config file (AIFABRIX_HOME env or ~/.aifabrix).
+ * Used so getAifabrixHome can read from the same location as config.js.
+ * @returns {string} Absolute path to config directory
+ */
+function getConfigDirForPaths() {
+  if (process.env.AIFABRIX_HOME && typeof process.env.AIFABRIX_HOME === 'string') {
+    return path.resolve(process.env.AIFABRIX_HOME.trim());
+  }
+  return path.join(safeHomedir(), '.aifabrix');
+}
+
 /**
  * Returns the base AI Fabrix directory.
- * Resolved from config.yaml `aifabrix-home` (stored under OS home).
- * Falls back to ~/.aifabrix when not specified.
+ * Priority: AIFABRIX_HOME env → config.yaml `aifabrix-home` (from AIFABRIX_HOME or ~/.aifabrix) → ~/.aifabrix.
  *
  * @returns {string} Absolute path to the AI Fabrix home directory
  */
 function getAifabrixHome() {
+  if (process.env.AIFABRIX_HOME && typeof process.env.AIFABRIX_HOME === 'string') {
+    return path.resolve(process.env.AIFABRIX_HOME.trim());
+  }
   const isTestEnv = process.env.NODE_ENV === 'test' || process.env.JEST_WORKER_ID !== undefined;
   if (!isTestEnv) {
     try {
-      const configPath = path.join(safeHomedir(), '.aifabrix', 'config.yaml');
+      const configDir = getConfigDirForPaths();
+      const configPath = path.join(configDir, 'config.yaml');
       if (fs.existsSync(configPath)) {
         const content = fs.readFileSync(configPath, 'utf8');
         const config = yaml.load(content) || {};
@@ -278,7 +293,9 @@ function getIntegrationPath(appName) {
 }
 
 /**
- * Gets the builder folder path for regular applications
+ * Gets the builder folder path for regular applications.
+ * When AIFABRIX_BUILDER_DIR is set (e.g. by up-miso/up-dataplane from config aifabrix-env-config),
+ * uses that as builder root instead of cwd/builder.
  * @param {string} appName - Application name
  * @returns {string} Absolute path to builder directory
  */
@@ -286,6 +303,12 @@ function getBuilderPath(appName) {
   if (!appName || typeof appName !== 'string') {
     throw new Error('App name is required and must be a string');
   }
+  const builderRoot = process.env.AIFABRIX_BUILDER_DIR && typeof process.env.AIFABRIX_BUILDER_DIR === 'string'
+    ? process.env.AIFABRIX_BUILDER_DIR.trim()
+    : null;
+  if (builderRoot) {
+    return path.join(builderRoot, appName);
+  }
   return path.join(process.cwd(), 'builder', appName);
 }
 
@@ -442,6 +465,7 @@ async function detectAppType(appName, options = {}) {
 
 module.exports = {
   getAifabrixHome,
+  getConfigDirForPaths,
   getApplicationsBaseDir,
   getDevDirectory,
   getAppPath,

package/lib/utils/secrets-generator.js

@@ -41,6 +41,36 @@ function findMissingSecretKeys(envTemplate, existingSecrets) {
   return missingKeys;
 }
 
+/**
+ * Generate database password value for a key (databases-*-passwordKeyVault)
+ * @param {string} key - Secret key name
+ * @returns {string|null} Password string or null if key does not match
+ */
+function generateDbPasswordValue(key) {
+  const dbPasswordMatch = key.match(/^databases-([a-z0-9-_]+)-\d+-passwordKeyVault$/i);
+  if (!dbPasswordMatch) return null;
+  const appName = dbPasswordMatch[1];
+  if (appName === 'miso-controller') return 'miso_pass123';
+  const dbName = appName.replace(/-/g, '_');
+  return `${dbName}_pass123`;
+}
+
+/**
+ * Generate database URL value for a key (databases-*-urlKeyVault)
+ * @param {string} key - Secret key name
+ * @returns {string|null} URL string or null if key does not match
+ */
+function generateDbUrlValue(key) {
+  const dbUrlMatch = key.match(/^databases-([a-z0-9-_]+)-\d+-urlKeyVault$/i);
+  if (!dbUrlMatch) return null;
+  const appName = dbUrlMatch[1];
+  if (appName === 'miso-controller') {
+    return 'postgresql://miso_user:miso_pass123@${DB_HOST}:${DB_PORT}/miso';
+  }
+  const dbName = appName.replace(/-/g, '_');
+  return `postgresql://${dbName}_user:${dbName}_pass123@\${DB_HOST}:\${DB_PORT}/${dbName}`;
+}
+
 /**
  * Generates secret value based on key name
  * @function generateSecretValue
@@ -51,22 +81,14 @@ function generateSecretValue(key) {
   const keyLower = key.toLowerCase();
 
   if (keyLower.includes('password')) {
-    const dbPasswordMatch = key.match(/^databases-([a-z0-9-_]+)-\d+-passwordKeyVault$/i);
-    if (dbPasswordMatch) {
-      const appName = dbPasswordMatch[1];
-      const dbName = appName.replace(/-/g, '_');
-      return `${dbName}_pass123`;
-    }
+    const dbPassword = generateDbPasswordValue(key);
+    if (dbPassword !== null) return dbPassword;
     return crypto.randomBytes(32).toString('base64');
   }
 
   if (keyLower.includes('url') || keyLower.includes('uri')) {
-    const dbUrlMatch = key.match(/^databases-([a-z0-9-_]+)-\d+-urlKeyVault$/i);
-    if (dbUrlMatch) {
-      const appName = dbUrlMatch[1];
-      const dbName = appName.replace(/-/g, '_');
-      return `postgresql://${dbName}_user:${dbName}_pass123@\${DB_HOST}:\${DB_PORT}/${dbName}`;
-    }
+    const dbUrl = generateDbUrlValue(key);
+    if (dbUrl !== null) return dbUrl;
     return '';
   }
 

package/lib/utils/secrets-helpers.js

@@ -263,8 +263,7 @@ async function getLocalEnvWithOverrides() {
   let localEnv = await getEnvHosts('local');
 
   try {
-    const os = require('os');
-    const cfgPath = path.join(os.homedir(), '.aifabrix', 'config.yaml');
+    const cfgPath = config.CONFIG_FILE;
     if (fs.existsSync(cfgPath)) {
       const cfgContent = fs.readFileSync(cfgPath, 'utf8');
       const cfg = yaml.load(cfgContent) || {};

package/lib/utils/token-manager-refresh.js

@@ -10,6 +10,7 @@
 
 const config = require('../core/config');
 const { refreshDeviceToken: apiRefreshDeviceToken } = require('./api');
+const { isTokenEncrypted } = require('./token-encryption');
 
 /**
  * Validates refresh token parameters
@@ -138,6 +139,10 @@ async function refreshDeviceToken(controllerUrl, refreshToken) {
   if (!refreshToken || typeof refreshToken !== 'string') {
     throw new Error('Refresh token is required');
   }
+  // Never send encrypted token to the API (causes 401). Decryption should happen in getDeviceToken; this is a safeguard.
+  if (isTokenEncrypted(refreshToken)) {
+    throw new Error('Refresh token is still encrypted; decryption may have failed. Run "aifabrix login" to authenticate again.');
+  }
 
   try {
     // Call API refresh endpoint

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aifabrix/builder",
-  "version": "2.33.1",
+  "version": "2.33.3",
   "description": "AI Fabrix Local Fabric & Deployment SDK",
   "main": "lib/index.js",
   "bin": {

package/templates/applications/dataplane/Dockerfile

@@ -0,0 +1,16 @@
+# AI Fabrix Dataplane - Build from base image
+# This repo has no application source; use the published image as baseline.
+# Build: docker build -t aifabrix/dataplane:local .
+# Or use the image directly: docker run aifabrix/dataplane:latest
+
+FROM aifabrix/dataplane:latest
+
+# Expose port (documentation; base image may already set this)
+EXPOSE 3001
+
+# Health check (documentation; base image may already set this)
+HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
+    CMD curl -f http://localhost:3001/health || exit 1
+
+# CMD inherited from base image; override only if needed
+# CMD inherited: uvicorn app.main:app --host 0.0.0.0 --port 3001

package/templates/applications/dataplane/README.md

@@ -0,0 +1,205 @@
+# Dataplane Builder
+
+Build, run, and deploy Dataplane using `@aifabrix/builder`.
+
+---
+
+## Quick Start
+
+### 1. Install
+
+```bash
+npm install -g @aifabrix/builder
+```
+
+### 2. First Time Setup
+
+```bash
+# Check your environment
+aifabrix doctor
+
+# Login to controller (change your own port)
+aifabrix login --method device --environment dev --controller http://localhost:3100
+
+# Register your application (gets you credentials automatically)
+aifabrix app register dataplane
+```
+
+### 3. Build & Run Locally
+
+```bash
+# Build the Docker image
+aifabrix build dataplane
+
+# Run locally
+aifabrix run dataplane
+```
+
+**Access your app:** <http://localhost:3111> (host port from `build.localPort`; container uses 3001)
+
+---
+
+## Testing dataplane (use DATAPLANE_TEST_GUIDE)
+
+**Use the builder's Dataplane Test Guide** for auth, health, wizard, external systems, and pipeline checks:
+
+- **In aifabrix-builder:** `integration/hubspot/DATAPLANE_TEST_GUIDE.md`
+- **Dataplane base URL:** `http://localhost:3111`
+- **Controller:** `http://localhost:3110` (login, token)
+
+The guide defines: token setup, `/health`, wizard API, external systems API, pipeline API, and quick checks. 
+Keep `build.localPort` in `variables.yaml` at **3111** so it matches that guide.
+
+**View logs:**
+
+```bash
+docker logs aifabrix-dataplane -f
+```
+
+**Stop:**
+
+```bash
+docker stop aifabrix-dataplane
+```
+
+### 4. Deploy to Azure
+
+```bash
+# Build with version tag
+aifabrix build dataplane --tag v1.0.0
+
+# Push to registry
+aifabrix push dataplane --registry myacr.azurecr.io --tag "v1.0.0,latest"
+
+# Deploy to miso-controller
+aifabrix deploy dataplane
+```
+
+---
+
+## Using miso-client
+
+> [miso-client](https://github.com/esystemsdev/aifabrix-miso-client)
+
+After registering your app, you automatically get credentials in your secret file. Use miso-client for login, RBAC, audit logs, etc.
+
+**Rotate credentials if needed:**
+
+```bash
+aifabrix app rotate-secret dataplane
+```
+
+---
+
+## Reference
+
+### Common Commands
+
+```bash
+# Development
+aifabrix build dataplane                        # Build app
+aifabrix run dataplane                          # Run locally
+aifabrix dockerfile dataplane --force           # Generate Dockerfile
+aifabrix resolve dataplane                      # Generate .env file
+
+# Deployment
+aifabrix json dataplane                         # Preview deployment JSON
+aifabrix genkey dataplane                       # Generate deployment key
+aifabrix push dataplane --registry myacr.azurecr.io # Push to ACR
+aifabrix deploy dataplane --controller <url>    # Deploy to Azure
+
+# Management
+aifabrix app register dataplane
+aifabrix app list
+aifabrix app rotate-secret dataplane
+
+# Utilities
+aifabrix doctor                                   # Check environment
+aifabrix login --method device  # Login
+aifabrix --help                                   # Get help
+```
+
+### Build Options
+
+```bash
+aifabrix build dataplane --tag v1.0.0           # Custom tag
+aifabrix build dataplane --force-template       # Force template regeneration
+aifabrix build dataplane --language typescript  # Override language detection
+```
+
+### Run Options
+
+```bash
+aifabrix run dataplane --port 3000          # Custom port
+aifabrix run dataplane --debug                  # Debug output
+```
+
+### Push Options
+
+```bash
+aifabrix push dataplane --registry myacr.azurecr.io --tag v1.0.0
+aifabrix push dataplane --registry myacr.azurecr.io --tag "v1.0.0,latest,stable"
+```
+
+### Deploy Options
+
+```bash
+aifabrix deploy dataplane
+aifabrix deploy dataplane --no-poll
+```
+
+### Login Methods
+
+```bash
+# Device code flow
+aifabrix login --method device --environment dev
+
+# Credentials (reads from secrets.local.yaml)
+aifabrix login --method credentials --app dataplane --environment dev
+
+# Explicit credentials
+aifabrix login --method credentials --app dataplane --client-id $CLIENT_ID --client-secret $CLIENT_SECRET --environment dev
+```
+
+### Environment Variables
+
+```bash
+export AIFABRIX_HOME=/custom/path
+export AIFABRIX_SECRETS=/path/to/secrets.yaml
+```
+
+---
+
+## Troubleshooting
+
+- **"Docker not running"** → Start Docker Desktop
+- **"Not logged in"** → Run `aifabrix login` first
+- **"Port already in use"** → Use `--port` flag or change `build.localPort` in `variables.yaml` (default: 3111, must match DATAPLANE_TEST_GUIDE)
+- **"Authentication failed"** → Run `aifabrix login` again
+- **"Build fails"** → Check Docker is running and `variables.yaml` → `build.secrets` path is correct
+- **"Can't connect"** → Verify infrastructure is running and PostgreSQL is accessible
+
+**Regenerate files:**
+
+```bash
+aifabrix resolve dataplane --force
+aifabrix json dataplane
+aifabrix genkey dataplane
+```
+
+---
+
+## Prerequisites
+
+- `@aifabrix/builder` installed globally
+- Docker Desktop running
+- Azure CLI installed (for push command)
+- Authenticated with controller (for deploy command)
+- PostgreSQL database (ensure infrastructure is running)
+- Redis (ensure infrastructure is running)
+- File storage configured
+- Authentication/RBAC configured
+
+---
+
+**Application**: dataplane | **Port**: 3111 (local) / 3001 (container) | **Registry**: myacr.azurecr.io | **Image**: aifabrix/dataplane:latest

package/templates/applications/dataplane/env.template

@@ -0,0 +1,143 @@
+# Environment Variables Template
+# Use kv:// references for secrets (resolved from .aifabrix/secrets.yaml)
+# Use ${VAR} for environment-specific values
+
+# =============================================================================
+# APPLICATION ENVIRONMENT
+# =============================================================================
+
+PORT=3001
+ENVIRONMENT=development
+DEBUG=false
+LOG_LEVEL=INFO
+LOG_FORMAT=json
+LOG_FILE_PATH=/mnt/data/logs/app.log
+LOCAL_MODE=false
+
+# When API_KEY is set, a matching Bearer token bypasses OAuth2 validation
+API_KEY=kv://miso-controller-api-key-secretKeyVault
+
+# API Configuration
+API_V1_STR=/api/v1
+VERSION=1.6.0
+
+# CORS Configuration
+ALLOWED_ORIGINS=http://localhost:*
+IDE_CORS_ORIGINS=
+
+# Encryption Configuration
+ENCRYPTION_KEY=kv://secrets-encryptionKeyVault
+
+# =============================================================================
+# DATABASE CONFIGURATION
+# =============================================================================
+
+DATABASE_URL=kv://databases-dataplane-0-urlKeyVault
+DB_0_PASSWORD=kv://databases-dataplane-0-passwordKeyVault
+
+# Vector and document store DB: chunks, embeddings, vector indexes (pgvector).
+# Binaries path: config.processing.fileStoragePath or /data/documents.
+VECTOR_DATABASE_URL=kv://databases-dataplane-1-urlKeyVault
+DB_1_PASSWORD=kv://databases-dataplane-1-passwordKeyVault
+
+# Logs Database Configuration (for execution, audit, ABAC traces)
+LOGS_DATABASE_URL=kv://databases-dataplane-2-urlKeyVault
+DB_2_PASSWORD=kv://databases-dataplane-2-passwordKeyVault
+
+# Records Database Configuration (for external records storage)
+RECORDS_DATABASE_URL=kv://databases-dataplane-3-urlKeyVault
+DB_3_PASSWORD=kv://databases-dataplane-3-passwordKeyVault
+
+# =============================================================================
+# REDIS CONFIGURATION
+# =============================================================================
+# Connects to external redis from aifabrix-setup
+
+REDIS_URL=kv://redis-url
+
+# =============================================================================
+# CACHE CONFIGURATION
+# =============================================================================
+
+CACHE_ENABLED=true
+CACHE_CIP_EXECUTION_TTL=1800
+CACHE_METADATA_FILTER_TTL=3600
+
+# =============================================================================
+# AUTHENTICATION CONFIGURATION
+# =============================================================================
+
+# MISO Application Client Credentials (per application)
+MISO_CLIENTID=kv://dataplane-client-idKeyVault
+MISO_CLIENTSECRET=kv://dataplane-client-secretKeyVault
+
+# Keycloak Configuration (for OAuth2 endpoints)
+KEYCLOAK_SERVER_URL=kv://keycloak-server-urlKeyVault
+KEYCLOAK_REALM=aifabrix
+
+# MISO Controller URL
+MISO_CONTROLLER_URL=http://${MISO_HOST}:${MISO_PORT}
+
+# =============================================================================
+# AI/LLM CONFIGURATION
+# =============================================================================
+
+# OpenAI Configuration
+OPENAI_API_KEY=kv://secrets-openaiApiKeyVault
+
+# Azure OpenAI Configuration
+AZURE_OPENAI_ENDPOINT=
+AZURE_OPENAI_API_KEY=kv://secrets-azureOpenaiApiKeyVault
+AZURE_OPENAI_API_VERSION=2024-02-15-preview
+AZURE_OPENAI_DEPLOYMENT_NAME=gpt-4o
+
+# =============================================================================
+# AUDIT CONFIGURATION
+# =============================================================================
+
+# General Audit Settings
+AUDIT_ENABLED=true
+MISO_LOG_FORWARD_ERRORS=true
+AUTH_AUDIT_ENABLED=true
+
+# ABAC Audit Configuration
+ABAC_AUDIT_ENABLED=true
+ABAC_AUDIT_DETAIL_LEVEL=summary
+ABAC_EXPLAIN_MODE_ENABLED=false
+ABAC_PERFORMANCE_THRESHOLD_MS=1000
+
+# RBAC Audit Configuration
+RBAC_AUDIT_ENABLED=true
+RBAC_AUDIT_DETAIL_LEVEL=summary
+RBAC_EXPLAIN_MODE_ENABLED=false
+
+# =============================================================================
+# OBSERVABILITY CONFIGURATION
+# =============================================================================
+
+# OpenTelemetry Configuration
+OPENTELEMETRY_ENABLED=false
+OPENTELEMETRY_ENDPOINT=
+
+# =============================================================================
+# CIP EXECUTION CONFIGURATION
+# =============================================================================
+# These control CIP (Composable Integration Pipeline) execution behavior
+
+CIP_EXECUTION_MAX_RESPONSE_SIZE_MB=100.0
+CIP_EXECUTION_MAX_RECORDS=100000
+CIP_EXECUTION_OPERATION_TIMEOUT=300.0
+CIP_EXECUTION_HTTP_TIMEOUT=30.0
+CIP_EXECUTION_MAX_RETRIES=3
+CIP_EXECUTION_RETRY_BACKOFF_FACTOR=2.0
+CIP_EXECUTION_RETRY_INITIAL_DELAY=1.0
+
+# Circuit Breaker Configuration
+CIP_EXECUTION_CIRCUIT_BREAKER_FAILURE_THRESHOLD=5
+CIP_EXECUTION_CIRCUIT_BREAKER_TIME_WINDOW=60
+CIP_EXECUTION_CIRCUIT_BREAKER_SUCCESS_THRESHOLD=2
+CIP_EXECUTION_CIRCUIT_BREAKER_HALF_OPEN_TIMEOUT=30
+
+# Rate Limiting Configuration
+CIP_EXECUTION_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
+CIP_EXECUTION_RATE_LIMIT_BURST_SIZE=20