@aifabrix/builder 2.40.2 → 2.41.0

package/templates/applications/dataplane/env.template

@@ -7,7 +7,7 @@
 # =============================================================================
 
 # HTTP port for the app
-PORT=3001
+PORT=${PORT}
 # development | staging | production
 ENVIRONMENT=development
 # Enable debug mode
@@ -30,8 +30,8 @@ API_KEY=kv://miso-controller-api-key-secretKeyVault
 API_V1_STR=/api/v1
 VERSION=1.7.0
 # Base URL for the dataplane web server (used for default OAuth2 callback URL when redirectUri is omitted)
-DATAPLANE_WEB_SERVER_URL=kv://dataplane-web-server-urlKeyVault
-DATAPLANE_INTERNAL_URL=kv://dataplane-internal-server-urlKeyVault
+DATAPLANE_WEB_SERVER_URL=kv://dataplane-web-server-url
+DATAPLANE_INTERNAL_URL=kv://dataplane-internal-server-url
 
 # CORS Configuration
 ALLOWED_ORIGINS=http://localhost:*
@@ -43,6 +43,11 @@ ENCRYPTION_KEY=kv://secrets-encryptionKeyVault
 # =============================================================================
 # DATABASE CONFIGURATION
 # =============================================================================
+# Multiple-database layout: set all four URL env vars for four separate databases
+# (dataplane, dataplane-vector, dataplane-logs, dataplane-records). If any
+# dedicated URL is unset, that database's tables use DATABASE_URL (main).
+# See docs/DATABASE_TABLES_LOCATION.md for table-to-database mapping.
+# =============================================================================
 
 # Primary app database URL
 DATABASE_URL=kv://databases-dataplane-0-urlKeyVault
@@ -89,18 +94,18 @@ MISO_CLIENTSECRET=kv://dataplane-client-secretKeyVault
 
 # Keycloak Configuration (for OAuth2 endpoints)
 # Public: used by OpenAPI OAuth2 / browser (authorizationUrl, tokenUrl).
-KEYCLOAK_SERVER_URL=kv://keycloak-server-urlKeyVault
+KEYCLOAK_SERVER_URL=kv://keycloak-server-url
 # Internal (same role as MISO_CONTROLLER_URL): future server-side Keycloak (e.g. JWKS). Not used by dataplane today.
-KEYCLOAK_INTERNAL_SERVER_URL=kv://keycloak-internal-server-urlKeyVault
+KEYCLOAK_INTERNAL_SERVER_URL=kv://keycloak-internal-server-url
 KEYCLOAK_REALM=aifabrix
 
 # =============================================================================
 # MISO CONTROLLER CONFIGURATION
 # =============================================================================
 # Public: browser redirects and CORS for client_token; set when controller is behind a different public URL.
-MISO_WEB_SERVER_URL=kv://miso-controller-web-server-urlKeyVault
+MISO_WEB_SERVER_URL=kv://miso-controller-web-server-url
 # Internal: server-to-controller API calls (auth, pipeline, status, RBAC).
-MISO_CONTROLLER_URL=http://${MISO_HOST}:${MISO_PORT}
+MISO_CONTROLLER_URL=kv://miso-controller-internal-server-url
 
 # Pipeline env key for controller URLs: /api/v1/pipeline/{envKey}/validate and /deploy.
 # Set MISO_PIPELINE_ENV_KEY=dev when controller uses dev (e.g. MISO_CLIENTID=miso-controller-dev-dataplane).

package/templates/applications/keycloak/env.template

@@ -39,6 +39,8 @@ KC_HEALTH_ENABLED=true
 # Expose health endpoints on main HTTP port (like Keycloak 24.0)
 # Set to false to expose on main port instead of management port (9000)
 KC_HTTP_MANAGEMENT_HEALTH_ENABLED=false
+# Single-instance: use local cache so /health/ready passes (avoids Infinispan cluster check)
+KC_CACHE=local
 
 # =============================================================================
 # DATABASE CONFIGURATION

package/templates/applications/miso-controller/application.yaml

@@ -47,6 +47,7 @@ build:
   envOutputPath: ../../packages/miso-controller/.env # Copy .env to repo root for local dev (relative to builder/) (if null, no .env file is copied) (if empty, .env file is copied to repo root)
   localPort: 3010 # Port for local development (different from Docker port)
   language: typescript # Runtime language for template selection (typescript or python)
+  reloadStart: pnpm run start:reload # When running with --reload
 
 # =============================================================================
 # Portal Input Configuration (Deployment Wizard)

package/templates/applications/miso-controller/env.template

@@ -49,8 +49,8 @@ ONBOARDING_CREATE_DEV_ENV=true
 
 # NODE_ENV: production for Docker (serves pre-built static files), development for local dev
 # In Docker, this should be production to prevent Vite dev server initialization
-NODE_ENV=${NODE_ENV}
-PORT=${MISO_PORT}
+NODE_ENV=dev
+PORT=${PORT}
 AUTO_CREATE_TABLES=true
 FAST_STARTUP=false
 ALLOWED_ORIGINS=http://localhost:*
@@ -105,13 +105,12 @@ REDIS_PERMISSIONS_TTL=900
 # (KeycloakConfiguration + Application url/internalUrl). Sync env->DB at startup via
 # sync-application-urls-from-env.service.
 #
-# Azure Entra SSO during onboarding: true = skip (default), false = onboard Azure Entra SSO client in Keycloak
-# (onboarding Azure SSO requires Entra admin consent and Azure app credentials).
-KEYCLOAK_SKIP_AZURE_ENTRA_SSO=false
+# NOTE: Do NOT onboard Azure Entra SSO in Keycloak during onboarding (skipAzureEntraSso=true).
+# KEYCLOAK_SKIP_AZURE_ENTRA_SSO=false
 
 KEYCLOAK_REALM=aifabrix
-KEYCLOAK_SERVER_URL=kv://keycloak-server-urlKeyVault
-KEYCLOAK_INTERNAL_SERVER_URL=kv://keycloak-internal-server-urlKeyVault
+KEYCLOAK_SERVER_URL=kv://keycloak-server-url
+KEYCLOAK_INTERNAL_SERVER_URL=kv://keycloak-internal-server-url
 KEYCLOAK_CLIENT_ID=miso-controller
 KEYCLOAK_CLIENT_SECRET=kv://keycloak-client-secretKeyVault
 KEYCLOAK_ADMIN_USERNAME=admin
@@ -279,6 +278,9 @@ JWT_SECRET=kv://miso-controller-jwt-secretKeyVault
 # When API_KEY is set, a matching Bearer token bypasses OAuth2 validation
 API_KEY=kv://miso-controller-api-key-secretKeyVault
 
+# NPM token for private package (npmjs.org)
+NPM_TOKEN=kv://npm-token-secretKeyVault
+
 # =============================================================================
 # MISO CONTROLLER CONFIGURATION
 # =============================================================================
@@ -287,8 +289,8 @@ API_KEY=kv://miso-controller-api-key-secretKeyVault
 # Used to generate correct server URLs in OpenAPI spec and Keycloak callback URLs
 # For Docker: use localhost with mapped port (e.g., localhost:3100)
 # For production: use public domain (e.g., https://miso.example.com)
-MISO_WEB_SERVER_URL=kv://miso-controller-web-server-urlKeyVault
-MISO_CONTROLLER_URL=kv://miso-controller-internal-server-urlKeyVault
+MISO_WEB_SERVER_URL=kv://miso-controller-web-server-url
+MISO_CONTROLLER_URL=kv://miso-controller-internal-server-url
 
 # MISO Environment Configuration (miso, dev, tst, pro)
 MISO_ENVIRONMENT=miso

package/templates/python/docker-compose.hbs

@@ -25,23 +25,39 @@ services:
       - "traefik.http.routers.{{app.key}}.middlewares={{app.key}}-stripprefix"
       {{/unless}}
     {{/if}}
+    environment:
+      - MISO_ENVIRONMENT={{misoEnvironment}}
+    {{#if reloadStartCommand}}
+      - PORT={{containerPort}}
+    {{/if}}
     {{#if traefik.enabled}}
     {{#unless (eq traefik.path "/")}}
-    environment:
       - BASE_PATH={{traefik.path}}
       - X_FORWARDED_PREFIX={{traefik.path}}
     {{/unless}}
     {{/if}}
     env_file:
       - {{envFile}}
+    {{#if reloadStartCommand}}
+    command: ["sh", "-c", "cd /app && {{reloadStartCommand}}"]
+    {{/if}}
     ports:
       - "{{hostPort}}:{{containerPort}}"
     networks:
       - {{networkName}}
+    {{#if devMountPath}}
+    volumes:
+      - {{devMountPath}}:/app
+      {{#if requiresStorage}}
+      - {{#if (eq devId 0)}}aifabrix_{{app.key}}_data{{else}}aifabrix_dev{{devId}}_{{app.key}}_data{{/if}}:/mnt/data
+      {{/if}}
+    user: "${AIFABRIX_UID:-1000}:${AIFABRIX_GID:-1000}"
+    {{else}}
     {{#if requiresStorage}}
     volumes:
       - {{#if (eq devId 0)}}aifabrix_{{app.key}}_data{{else}}aifabrix_dev{{devId}}_{{app.key}}_data{{/if}}:/mnt/data
     {{/if}}
+    {{/if}}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:{{port}}{{healthCheck.path}}"]
       interval: {{healthCheck.interval}}s
@@ -56,78 +72,88 @@ services:
     {{/if}}
 
   {{#if requiresDatabase}}
-  # Database Initialization
+  # Database Initialization (uses infra pgpass when available: ~/.aifabrix/infra/pgpass)
+  # env_file: .env.run.admin only (POSTGRES_PASSWORD + DB_*); admin secrets never in app container
   db-init:
     image: pgvector/pgvector:pg15
     container_name: {{containerName}}-db-init
     entrypoint: []
     env_file:
-      - ${ADMIN_SECRETS_PATH}
-      - {{envFile}}
+      - {{#if dbInitEnvFile}}{{dbInitEnvFile}}{{else}}{{envFile}}{{/if}}
     environment:
       POSTGRES_DB: postgres
       PGHOST: postgres
       PGPORT: "5432"
       PGUSER: pgadmin
-      {{#if databases}}
-      {{#each databases}}
-      DB_{{@index}}_PASSWORD: {{lookup ../databasePasswords.array @index}}
-      {{/each}}
-      {{else}}
-      DB_PASSWORD: {{lookup databasePasswords.array 0}}
+      {{#if useInfraPgpass}}
+      PGPASSFILE: /run/pgpass
       {{/if}}
     networks:
       - {{networkName}}
-    volumes: []
+    volumes:
+      {{#if useInfraPgpass}}
+      - {{infraPgpassPath}}:/run/pgpass:ro
+      {{else}}
+      []
+      {{/if}}
     tmpfs:
       - /var/lib/postgresql/data
     command:
       - sh
       - -c
       - |
-        export PGHOST=postgres PGPORT=5432 PGUSER=pgadmin &&
-        export PGPASSWORD="${POSTGRES_PASSWORD}" &&
+        {{#unless useInfraPgpass}}
+        export PGPASSWORD="$${POSTGRES_PASSWORD}" &&
+        {{/unless}}
         echo 'Waiting for PostgreSQL to be ready...' &&
         counter=0 &&
-        while [ ${counter:-0} -lt 30 ]; do
-          if pg_isready -h postgres -p 5432 -U pgadmin >/dev/null 2>&1; then
+        while [ $${counter:-0} -lt 30 ]; do
+          if pg_isready >/dev/null 2>&1; then
             echo 'PostgreSQL is ready!'
             break
           fi
           echo 'Waiting for PostgreSQL...'
           sleep 1
-          counter=$((${counter:-0} + 1))
+          counter=$$(($${counter:-0} + 1))
         done &&
         {{#if databases}}
         {{#each databases}}
         echo 'Creating {{name}} database and user...' &&
-        if psql -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '{{name}}'" 2>/dev/null | grep -q '^1$'; then
-          echo 'Database "{{name}}" already exists, all ok.'
+        if psql -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '{{name}}'" 2>/dev/null | grep -q '^1$$'; then
+          echo 'Database "{{name}}" already exists, syncing user password...' &&
+          psql -d postgres -c "ALTER USER \"{{pgUserName name}}\" WITH PASSWORD '"$${DB_{{@index}}_PASSWORD}"';" &&
+          echo 'Database "{{name}}" ok.'
         else
           echo 'Creating database "{{name}}"...' &&
           psql -d postgres -c "CREATE DATABASE \"{{name}}\";" &&
           echo 'Dropping old user if exists...' &&
           psql -d postgres -c "DROP USER IF EXISTS \"{{pgUserOld name}}\";" || true &&
           echo 'Creating user "{{pgUserName name}}"...' &&
-          psql -d postgres -c 'CREATE USER "{{pgUserName name}}" WITH PASSWORD '\''${DB_{{@index}}_PASSWORD}'\'';' &&
+          psql -d postgres -c "CREATE USER \"{{pgUserName name}}\" WITH PASSWORD '"$${DB_{{@index}}_PASSWORD}"';" &&
           echo 'Granting privileges...' &&
           psql -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"{{name}}\" TO \"{{pgUserName name}}\";" &&
           psql -d {{name}} -c "ALTER SCHEMA public OWNER TO \"{{pgUserName name}}\";" &&
           psql -d {{name}} -c "GRANT ALL ON SCHEMA public TO \"{{pgUserName name}}\";" &&
           echo 'Database "{{name}}" created successfully!'
         fi &&
+        {{#if (isVectorDatabase name)}}
+        psql -d {{name}} -c "CREATE EXTENSION IF NOT EXISTS vector;" &&
+        echo 'pgvector extension enabled on "{{name}}".' &&
+        {{/if}}
         {{/each}}
         {{else}}
         echo 'Creating {{app.key}} database and user...' &&
-        if psql -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '{{app.key}}'" 2>/dev/null | grep -q '^1$'; then
-          echo 'Database "{{app.key}}" already exists, all ok.'
+        if psql -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '{{app.key}}'" 2>/dev/null | grep -q '^1$$'; then
+          echo 'Database "{{app.key}}" already exists, syncing user password...' &&
+          psql -d postgres -c "ALTER USER \"{{pgUserName app.key}}\" WITH PASSWORD '"$${DB_0_PASSWORD:-$$DB_PASSWORD}"';" &&
+          echo 'Database "{{app.key}}" ok.'
         else
           echo 'Creating database "{{app.key}}"...' &&
           psql -d postgres -c "CREATE DATABASE \"{{app.key}}\";" &&
           echo 'Dropping old user if exists...' &&
           psql -d postgres -c "DROP USER IF EXISTS \"{{pgUserOld app.key}}\";" || true &&
           echo 'Creating user "{{pgUserName app.key}}"...' &&
-          psql -d postgres -c 'CREATE USER "{{pgUserName app.key}}" WITH PASSWORD '\''${DB_0_PASSWORD:-${DB_PASSWORD}}'\'';' &&
+          psql -d postgres -c "CREATE USER \"{{pgUserName app.key}}\" WITH PASSWORD '"$${DB_0_PASSWORD:-$$DB_PASSWORD}"';" &&
           echo 'Granting privileges...' &&
           psql -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"{{app.key}}\" TO \"{{pgUserName app.key}}\";" &&
           psql -d {{app.key}} -c "ALTER SCHEMA public OWNER TO \"{{pgUserName app.key}}\";" &&
@@ -154,4 +180,4 @@ volumes:
 
 networks:
   {{networkName}}:
-    external: true
+    external: true

package/templates/typescript/docker-compose.hbs

@@ -25,23 +25,39 @@ services:
       - "traefik.http.routers.{{app.key}}.middlewares={{app.key}}-stripprefix"
       {{/unless}}
     {{/if}}
+    environment:
+      - MISO_ENVIRONMENT={{misoEnvironment}}
+    {{#if reloadStartCommand}}
+      - PORT={{containerPort}}
+    {{/if}}
     {{#if traefik.enabled}}
     {{#unless (eq traefik.path "/")}}
-    environment:
       - BASE_PATH={{traefik.path}}
       - X_FORWARDED_PREFIX={{traefik.path}}
     {{/unless}}
     {{/if}}
     env_file:
       - {{envFile}}
+    {{#if reloadStartCommand}}
+    command: ["sh", "-c", "cd /app && {{reloadStartCommand}}"]
+    {{/if}}
     ports:
       - "{{hostPort}}:{{containerPort}}"
     networks:
       - {{networkName}}
+    {{#if devMountPath}}
+    volumes:
+      - {{devMountPath}}:/app
+      {{#if requiresStorage}}
+      - {{#if (eq devId 0)}}aifabrix_{{app.key}}_data{{else}}aifabrix_dev{{devId}}_{{app.key}}_data{{/if}}:/mnt/data
+      {{/if}}
+    user: "${AIFABRIX_UID:-1000}:${AIFABRIX_GID:-1000}"
+    {{else}}
     {{#if requiresStorage}}
     volumes:
       - {{#if (eq devId 0)}}aifabrix_{{app.key}}_data{{else}}aifabrix_dev{{devId}}_{{app.key}}_data{{/if}}:/mnt/data
     {{/if}}
+    {{/if}}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:{{port}}{{healthCheck.path}}"]
       interval: {{healthCheck.interval}}s
@@ -56,78 +72,88 @@ services:
     {{/if}}
 
   {{#if requiresDatabase}}
-  # Database Initialization
+  # Database Initialization (uses infra pgpass when available: ~/.aifabrix/infra/pgpass)
+  # env_file: .env.run.admin only (POSTGRES_PASSWORD + DB_*); admin secrets never in app container
   db-init:
     image: pgvector/pgvector:pg15
     container_name: {{containerName}}-db-init
     entrypoint: []
     env_file:
-      - ${ADMIN_SECRETS_PATH}
-      - {{envFile}}
+      - {{#if dbInitEnvFile}}{{dbInitEnvFile}}{{else}}{{envFile}}{{/if}}
     environment:
       POSTGRES_DB: postgres
       PGHOST: postgres
       PGPORT: "5432"
       PGUSER: pgadmin
-      {{#if databases}}
-      {{#each databases}}
-      DB_{{@index}}_PASSWORD: {{lookup ../databasePasswords.array @index}}
-      {{/each}}
-      {{else}}
-      DB_PASSWORD: {{lookup databasePasswords.array 0}}
+      {{#if useInfraPgpass}}
+      PGPASSFILE: /run/pgpass
       {{/if}}
     networks:
       - {{networkName}}
-    volumes: []
+    volumes:
+      {{#if useInfraPgpass}}
+      - {{infraPgpassPath}}:/run/pgpass:ro
+      {{else}}
+      []
+      {{/if}}
     tmpfs:
       - /var/lib/postgresql/data
     command:
       - sh
       - -c
       - |
-        export PGHOST=postgres PGPORT=5432 PGUSER=pgadmin &&
-        export PGPASSWORD="${POSTGRES_PASSWORD}" &&
+        {{#unless useInfraPgpass}}
+        export PGPASSWORD="$${POSTGRES_PASSWORD}" &&
+        {{/unless}}
         echo 'Waiting for PostgreSQL to be ready...' &&
         counter=0 &&
-        while [ ${counter:-0} -lt 30 ]; do
-          if pg_isready -h postgres -p 5432 -U pgadmin >/dev/null 2>&1; then
+        while [ $${counter:-0} -lt 30 ]; do
+          if pg_isready >/dev/null 2>&1; then
             echo 'PostgreSQL is ready!'
             break
           fi
           echo 'Waiting for PostgreSQL...'
           sleep 1
-          counter=$((${counter:-0} + 1))
+          counter=$$(($${counter:-0} + 1))
         done &&
         {{#if databases}}
         {{#each databases}}
         echo 'Creating {{name}} database and user...' &&
-        if psql -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '{{name}}'" 2>/dev/null | grep -q '^1$'; then
-          echo 'Database "{{name}}" already exists, all ok.'
+        if psql -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '{{name}}'" 2>/dev/null | grep -q '^1$$'; then
+          echo 'Database "{{name}}" already exists, syncing user password...' &&
+          psql -d postgres -c "ALTER USER \"{{pgUserName name}}\" WITH PASSWORD '"$${DB_{{@index}}_PASSWORD}"';" &&
+          echo 'Database "{{name}}" ok.'
         else
           echo 'Creating database "{{name}}"...' &&
           psql -d postgres -c "CREATE DATABASE \"{{name}}\";" &&
           echo 'Dropping old user if exists...' &&
           psql -d postgres -c "DROP USER IF EXISTS \"{{pgUserOld name}}\";" || true &&
           echo 'Creating user "{{pgUserName name}}"...' &&
-          psql -d postgres -c 'CREATE USER "{{pgUserName name}}" WITH PASSWORD '\''${DB_{{@index}}_PASSWORD}'\'';' &&
+          psql -d postgres -c "CREATE USER \"{{pgUserName name}}\" WITH PASSWORD '"$${DB_{{@index}}_PASSWORD}"';" &&
           echo 'Granting privileges...' &&
           psql -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"{{name}}\" TO \"{{pgUserName name}}\";" &&
           psql -d {{name}} -c "ALTER SCHEMA public OWNER TO \"{{pgUserName name}}\";" &&
           psql -d {{name}} -c "GRANT ALL ON SCHEMA public TO \"{{pgUserName name}}\";" &&
           echo 'Database "{{name}}" created successfully!'
         fi &&
+        {{#if (isVectorDatabase name)}}
+        psql -d {{name}} -c "CREATE EXTENSION IF NOT EXISTS vector;" &&
+        echo 'pgvector extension enabled on "{{name}}".' &&
+        {{/if}}
         {{/each}}
         {{else}}
         echo 'Creating {{app.key}} database and user...' &&
-        if psql -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '{{app.key}}'" 2>/dev/null | grep -q '^1$'; then
-          echo 'Database "{{app.key}}" already exists, all ok.'
+        if psql -d postgres -tAc "SELECT 1 FROM pg_database WHERE datname = '{{app.key}}'" 2>/dev/null | grep -q '^1$$'; then
+          echo 'Database "{{app.key}}" already exists, syncing user password...' &&
+          psql -d postgres -c "ALTER USER \"{{pgUserName app.key}}\" WITH PASSWORD '"$${DB_0_PASSWORD:-$$DB_PASSWORD}"';" &&
+          echo 'Database "{{app.key}}" ok.'
         else
           echo 'Creating database "{{app.key}}"...' &&
           psql -d postgres -c "CREATE DATABASE \"{{app.key}}\";" &&
           echo 'Dropping old user if exists...' &&
           psql -d postgres -c "DROP USER IF EXISTS \"{{pgUserOld app.key}}\";" || true &&
           echo 'Creating user "{{pgUserName app.key}}"...' &&
-          psql -d postgres -c 'CREATE USER "{{pgUserName app.key}}" WITH PASSWORD '\''${DB_0_PASSWORD:-${DB_PASSWORD}}'\'';' &&
+          psql -d postgres -c "CREATE USER \"{{pgUserName app.key}}\" WITH PASSWORD '"$${DB_0_PASSWORD:-$$DB_PASSWORD}"';" &&
           echo 'Granting privileges...' &&
           psql -d postgres -c "GRANT ALL PRIVILEGES ON DATABASE \"{{app.key}}\" TO \"{{pgUserName app.key}}\";" &&
           psql -d {{app.key}} -c "ALTER SCHEMA public OWNER TO \"{{pgUserName app.key}}\";" &&