@aifabrix/builder 2.48.0 → 2.49.1

package/scripts/test-dataplane-bootstrap.js

@@ -0,0 +1,331 @@
+#!/usr/bin/env node
+/* eslint-disable no-console, require-jsdoc, max-lines-per-function, max-statements */
+
+/**
+ * Diagnose and optionally exercise dataplane bootstrap provisioning (plan 197.0).
+ *
+ * @fileoverview Manual test / diagnostic for dataplane bootstrap API
+ *
+ * Usage (from aifabrix-builder repo root):
+ *   node scripts/test-dataplane-bootstrap.js
+ *   node scripts/test-dataplane-bootstrap.js --probe-api
+ *   node scripts/test-dataplane-bootstrap.js --call-bootstrap
+ *   node scripts/test-dataplane-bootstrap.js --controller-url http://localhost:3100 --probe-api
+ */
+
+'use strict';
+
+const path = require('path');
+
+const ROOT = path.join(__dirname, '..');
+
+function parseArgs(argv) {
+  const out = {
+    controllerUrl: null,
+    envKey: 'dev',
+    probeApi: false,
+    callBootstrap: false,
+    help: false
+  };
+  for (let i = 2; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--help' || arg === '-h') {
+      out.help = true;
+    } else if (arg === '--probe-api') {
+      out.probeApi = true;
+    } else if (arg === '--call-bootstrap') {
+      out.callBootstrap = true;
+      out.probeApi = true;
+    } else if (arg === '--controller-url' && argv[i + 1]) {
+      out.controllerUrl = String(argv[++i]).trim();
+    } else if (arg === '--env-key' && argv[i + 1]) {
+      out.envKey = String(argv[++i]).trim().toLowerCase();
+    } else {
+      throw new Error(`Unknown argument: ${arg}`);
+    }
+  }
+  return out;
+}
+
+function printHelp() {
+  console.log(`\
+Dataplane bootstrap diagnostic
+
+  node scripts/test-dataplane-bootstrap.js [options]
+
+Options:
+  --controller-url <url>  Override controller base URL (default: ~/.aifabrix/config.yaml)
+  --env-key <key>         Environment for bootstrap (default: dev)
+  --probe-api             POST .../applications/bootstrap (idempotent register/check)
+  --call-bootstrap        Same as --probe-api + save credentials locally (up-dataplane path)
+  -h, --help              Show this help
+
+Default (no flags): read-only gate checks — no API mutation.
+
+Bootstrap avoids device login when ALL are true:
+  - environment is dev
+  - Docker CLI + Compose on this machine
+  - merged secrets have miso-controller-secrets-encryptionKeyVault (= controller ENCRYPTION_KEY)
+  - controller /health OK
+`);
+}
+
+function line(label, value, ok) {
+  const mark = ok === true ? 'OK' : ok === false ? 'FAIL' : '—';
+  console.log(`  [${mark}] ${label}: ${value}`);
+}
+
+function loadBuilderModules() {
+  return {
+    config: require(path.join(ROOT, 'lib/core/config')),
+    docker: require(path.join(ROOT, 'lib/utils/docker')),
+    loadSecrets: require(path.join(ROOT, 'lib/core/secrets-load')).loadSecrets,
+    checkHealthEndpoint: require(path.join(ROOT, 'lib/utils/health-check-request')).checkHealthEndpoint,
+    resolveControllerUrl: require(path.join(ROOT, 'lib/utils/controller-url')).resolveControllerUrl,
+    canUseBootstrapProvision: require(path.join(ROOT, 'lib/commands/up-dataplane-bootstrap'))
+      .canUseBootstrapProvision,
+    resolveBootstrapEncryptionKey: require(path.join(ROOT, 'lib/commands/up-dataplane-bootstrap'))
+      .resolveBootstrapEncryptionKey,
+    bootstrapDataplaneApplication: require(path.join(ROOT, 'lib/commands/up-dataplane-bootstrap'))
+      .bootstrapDataplaneApplication,
+    buildDataplaneImageRef: require(path.join(ROOT, 'lib/commands/up-dataplane')).buildDataplaneImageRef,
+    bootstrapApplication: require(path.join(ROOT, 'lib/api/applications.api')).bootstrapApplication,
+    loadVariablesYaml: require(path.join(ROOT, 'lib/utils/app-register-config')).loadVariablesYaml,
+    extractAppConfiguration: require(path.join(ROOT, 'lib/utils/app-register-config')).extractAppConfiguration,
+    buildRegistrationData: require(path.join(ROOT, 'lib/app/register')).buildRegistrationData,
+    applyLocalhostPortAdjustment: require(path.join(ROOT, 'lib/app/register')).applyLocalhostPortAdjustment
+  };
+}
+
+async function runGateChecks(mods, controllerOverride) {
+  console.log('Prerequisites (canUseBootstrapProvision gates):');
+  const gates = {
+    devEnv: await checkEnvironmentGate(mods.config),
+    docker: await checkDockerGate(mods.docker),
+    encryptionKey: await checkEncryptionKeyGate(
+      mods.resolveBootstrapEncryptionKey,
+      mods.loadSecrets
+    )
+  };
+
+  const controllerUrl = await resolveControllerBaseUrl(
+    mods.config,
+    mods.resolveControllerUrl,
+    controllerOverride
+  );
+  line('controller URL', controllerUrl, null);
+  gates.controllerHealthy = await checkControllerHealth(controllerUrl, mods.checkHealthEndpoint);
+
+  const eligible = await mods.canUseBootstrapProvision({ skipInfraCheck: true });
+  console.log('');
+  line('canUseBootstrapProvision', eligible ? 'true' : 'false', eligible);
+  printDeviceLoginReasons(gates);
+
+  return { gates, controllerUrl, eligible };
+}
+
+async function checkEnvironmentGate(config) {
+  const cfg = await config.getConfig();
+  const env = cfg && cfg.environment ? String(cfg.environment) : 'dev';
+  const ok = env === 'dev';
+  line('config environment', env, ok);
+  if (!ok) {
+    console.log('    → Bootstrap only runs in dev. Set: aifabrix auth --set-environment dev');
+  }
+  return ok;
+}
+
+async function checkDockerGate(docker) {
+  try {
+    await docker.ensureDockerAndCompose();
+    line('Docker CLI + Compose', 'available', true);
+    return true;
+  } catch (err) {
+    line('Docker CLI + Compose', err.message || 'unavailable', false);
+    console.log('    → Bootstrap requires local Docker (same machine as aifabrix run dataplane).');
+    return false;
+  }
+}
+
+async function checkEncryptionKeyGate(resolveBootstrapEncryptionKey, loadSecrets) {
+  const key = await resolveBootstrapEncryptionKey();
+  const ok = Boolean(key && String(key).trim());
+  line(
+    'bootstrap ENCRYPTION_KEY (merged secrets)',
+    ok ? `present (${String(key).length} chars, not shown)` : 'missing',
+    ok
+  );
+  if (!ok) {
+    const secrets = await loadSecrets(undefined);
+    const keys = secrets && typeof secrets === 'object' ? Object.keys(secrets) : [];
+    const hints = keys.filter((k) => /encryption/i.test(k));
+    console.log(
+      '    → Need miso-controller-secrets-encryptionKeyVault in secrets (after up-miso resolve).'
+    );
+    if (hints.length) {
+      console.log(`    → Secret keys containing "encryption": ${hints.join(', ')}`);
+    }
+    console.log(
+      '    → config.yaml secrets-encryption is CLI file crypto only — not the bootstrap header key.'
+    );
+  }
+  return ok;
+}
+
+async function resolveControllerBaseUrl(config, resolveControllerUrl, override) {
+  if (override) {
+    return override.replace(/\/+$/, '');
+  }
+  const cfg = await config.getConfig();
+  const platform = cfg.platformController || cfg.controller;
+  if (platform && String(platform).trim()) {
+    return String(platform).trim().replace(/\/+$/, '');
+  }
+  return (await resolveControllerUrl()).replace(/\/+$/, '');
+}
+
+async function checkControllerHealth(controllerUrl, checkHealthEndpoint) {
+  const healthUrl = `${controllerUrl}/health`;
+  try {
+    const ok = await checkHealthEndpoint(healthUrl, false);
+    line('controller health', healthUrl, ok);
+    if (!ok) {
+      console.log('    → Start platform: aifabrix up-infra && aifabrix up-miso (or af setup).');
+    }
+    return ok;
+  } catch (err) {
+    line('controller health', `${healthUrl} (${err.message})`, false);
+    return false;
+  }
+}
+
+async function buildRegistrationPayload(mods, controllerUrl, envKey) {
+  const { variables } = await mods.loadVariablesYaml('dataplane');
+  const appConfig = await mods.extractAppConfiguration(variables, 'dataplane', {});
+  await mods.applyLocalhostPortAdjustment(appConfig, controllerUrl, {});
+  const imageOverride = mods.buildDataplaneImageRef({});
+  return mods.buildRegistrationData(appConfig, {
+    imageOverride,
+    image: imageOverride,
+    envKey
+  });
+}
+
+async function probeBootstrapApi(controllerUrl, encryptionKey, envKey, mods) {
+  const pathSuffix = `/api/v1/environments/${envKey}/applications/bootstrap`;
+  console.log('');
+  console.log('Bootstrap API probe (--probe-api):');
+  line('endpoint', `${controllerUrl}${pathSuffix}`, null);
+
+  const registrationData = await buildRegistrationPayload(mods, controllerUrl, envKey);
+  const response = await mods.bootstrapApplication(
+    controllerUrl,
+    envKey,
+    encryptionKey,
+    registrationData
+  );
+
+  if (!response.success) {
+    return reportBootstrapFailure(response);
+  }
+
+  const data = response.data?.data || response.data;
+  const status = data?.status || (data?.application ? 'created' : 'ok');
+  const appKey = data?.application?.key || registrationData.key;
+  line('bootstrap API', `OK (${status}, app: ${appKey})`, true);
+  return true;
+}
+
+function reportBootstrapFailure(response) {
+  const msg = response.formattedError || response.error || 'request failed';
+  if (/401|403|encryption|Unauthorized/i.test(String(msg))) {
+    line('x-encryption-key', 'rejected by controller', false);
+    console.log('    → Controller ENCRYPTION_KEY does not match merged secrets vault key.');
+    return false;
+  }
+  if (/503|not configured/i.test(String(msg))) {
+    line('bootstrap on controller', 'disabled (ENCRYPTION_KEY unset on controller)', false);
+    return false;
+  }
+  line('bootstrap API', msg, false);
+  return false;
+}
+
+function printDeviceLoginReasons(gates) {
+  console.log('');
+  console.log('Why setup may ask for device login instead of bootstrap:');
+  if (!gates.devEnv) console.log('  • environment is not dev');
+  if (!gates.docker) console.log('  • Docker/Compose not available on this host');
+  if (!gates.encryptionKey) console.log('  • bootstrap encryption key missing from merged secrets');
+  if (!gates.controllerHealthy) {
+    console.log('  • controller not healthy (setup may still run auth before dataplane step)');
+  }
+  if (gates.devEnv && gates.docker && gates.encryptionKey && gates.controllerHealthy) {
+    console.log('  • All bootstrap gates pass — setup should show "Bootstrap provision (no device login)"');
+    console.log('  • If dataplane still missing, check setup logs after "Starting Dataplane..."');
+  }
+}
+
+async function main() {
+  const args = parseArgs(process.argv);
+  if (args.help) {
+    printHelp();
+    process.exit(0);
+  }
+
+  console.log('Dataplane bootstrap diagnostic');
+  console.log('================================');
+  console.log('');
+
+  const mods = loadBuilderModules();
+  await mods.config.ensureSecretsEncryptionKey();
+
+  const { gates, controllerUrl, eligible } = await runGateChecks(mods, args.controllerUrl);
+
+  if (!gates.controllerHealthy || !gates.encryptionKey) {
+    console.log('');
+    console.log('Fix failing gates above, then re-run with --probe-api.');
+    process.exit(1);
+  }
+
+  if (!args.probeApi) {
+    console.log('');
+    console.log('Read-only checks done. To exercise bootstrap API:');
+    console.log('  node scripts/test-dataplane-bootstrap.js --probe-api');
+    process.exit(eligible ? 0 : 1);
+  }
+
+  const encryptionKey = await mods.resolveBootstrapEncryptionKey();
+  const apiOk = await probeBootstrapApi(controllerUrl, encryptionKey, args.envKey, mods);
+  if (!apiOk) {
+    process.exit(1);
+  }
+
+  if (!args.callBootstrap) {
+    process.exit(0);
+  }
+
+  if (!eligible) {
+    console.error('');
+    console.error('Refusing --call-bootstrap: canUseBootstrapProvision is false.');
+    process.exit(1);
+  }
+
+  console.log('');
+  console.log('Saving credentials via bootstrapDataplaneApplication...');
+  await mods.bootstrapDataplaneApplication(
+    { skipInfraCheck: true },
+    controllerUrl,
+    args.envKey,
+    mods.buildDataplaneImageRef
+  );
+  console.log('');
+  console.log('Done. Next: aifabrix deploy dataplane --local  or  aifabrix run dataplane');
+  process.exit(0);
+}
+
+main().catch((err) => {
+  console.error('');
+  console.error(`Error: ${err.message}`);
+  process.exit(1);
+});

package/templates/applications/dataplane/application.yaml

@@ -5,7 +5,7 @@ app:
   description: "AI Fabrix Dataplane is a secure, in-tenant integration and automation layer that supplies governed, normalized, and explainable enterprise data to AI agents. Using CIP as a declarative standard, it enforces RBAC and ABAC, executes integrations, and exposes trusted data via MCP and OpenAPI."
   type: webapp
   language: python  # Explicitly specify Python language
-  version: 1.9.5
+  version: 2.0.0
 
 # Image Configuration
 # Set tag to match your build (e.g. aifabrix build dataplane -t 1.0.0 then tag: 1.0.0)

package/templates/applications/dataplane/env.template

@@ -21,24 +21,29 @@ LOG_FORMAT=json
 # Production with persistent logs: use /mnt/data/logs/app.log and mount a volume at /mnt/data.
 LOG_FILE_PATH=/mnt/data/logs/app.log
 # If true, run without Redis/Celery (single process). Set false on API/worker/beat in production.
-LOCAL_MODE=true
+LOCAL_MODE=false
 
 # Celery worker process tuning (dataplane-worker container; optional overrides)
 # CELERY_WORKER_QUEUES=external_data,document_sync,document_processing,audit
 # CELERY_WORKER_CONCURRENCY=2
+# CELERY_WORKER_RUNTIME is set automatically by python -m app.core.celery_worker_main (NullPool per task).
 # CELERY_LOG_LEVEL=info
 
 # When API_KEY is set, a matching Bearer token bypasses OAuth2 validation
 # Same kv:// as miso-controller API_KEY so pipeline Bearer bypass uses one secrets.local entry
 API_KEY=kv://miso-controller-secrets-apiKeyVault
 # Comma-separated realm roles attached to API_KEY bypass (JWT realm_access.roles). Default: operator.
-# Add integration RBAC role keys from published manifests when manual tests need selector/MCP access, e.g.:
-# API_KEY_BYPASS_ROLES=operator,<role-from-manifest-rbac>
-# API_KEY_BYPASS_ROLES=operator
+# Dev manual tests (437 governance, enterprise-llm llm.call): use full-access locally; restrict in tst/pro.
+API_KEY_BYPASS_ROLES=full-access
+# Example narrower dev set: operator,openai-admin,azure-openai-admin,ollama-admin
+
+# Host-run manual test suite: PG idle cleanup + lifecycle certification DB snapshot (dev02 example).
+# Inside Docker containers use postgres:5432 in DATABASE_URL; on host use localhost:5632 or docker exec.
+# AIFABRIX_PG_DOCKER_CONTAINER=aifabrix-dev02-postgres
 
 # API Configuration
 API_V1_STR=/api/v1
-VERSION=1.9.5
+VERSION=2.0.0
 # Minimum supported aifabrix-builder CLI semver (advertised on /health and /api/v1/health; empty = no gate)
 # MIN_BUILDER_CLI_VERSION=2.45.0
 MIN_BUILDER_CLI_VERSION=2.45.5
@@ -86,6 +91,11 @@ DB_2_PASSWORD=kv://databases-dataplane-2-passwordKeyVault
 RECORDS_DATABASE_URL=kv://databases-dataplane-3-urlKeyVault
 DB_3_PASSWORD=kv://databases-dataplane-3-passwordKeyVault
 
+# Pool budget when main/logs/records/vector share one PostgreSQL host (default 24).
+# Host-native make dev (PORT 3011 + developerId×100, e.g. 3611) auto-caps to 12 on shared dev Postgres
+# so Docker dataplane + miso-controller do not exhaust max_connections alongside make dev.
+# SAME_SERVER_POOL_BUDGET=24
+
 # =============================================================================
 # REDIS CONFIGURATION
 # =============================================================================
@@ -145,6 +155,7 @@ MISO_PIPELINE_ENV_KEY=
 
 # LLM: chat/embed route through governed enterprise-llm only. Provider credentials
 # live on LLM provider external systems (integration/openai, azure-openai, ollama).
+# Shared secret for integration/openai: secrets-openaiApiKeyVault (af secret set ... --shared)
 AI_VALIDATION_LLM_MODEL=gpt-4o-mini
 
 # =============================================================================
@@ -182,7 +193,14 @@ AUDIT_ASYNC_SHADOW_ENQUEUE=false
 # Set true for local dev policy CRUD; pair with VITE_POLICY_MANAGEMENT_ENABLED=true in app-ui.
 POLICY_MANAGEMENT_API_ENABLED=false
 
-# Developer Access tab help links (optional; empty hides banner links in UI).
+# VIEW_MODULES — optional comma-separated UI module allowlist (unset = all modules).
+# Keys: dashboard, work, content-review, assistants, connected-systems, templates,
+# audit-logs, ai-trust, administration (deep link only; never sidebar in V1).
+# Example pilot: VIEW_MODULES=dashboard,work,content-review
+# Pair with VITE_VIEW_MODULES in app-ui (same value).
+# VIEW_MODULES=
+# VITE_VIEW_MODULES=
+
 # VITE_DEVELOPER_GUIDE_URL=
 # VITE_API_EXAMPLES_URL=
 # VITE_MCP_INTEGRATION_GUIDE_URL=

package/templates/applications/dataplane/rbac.yaml

@@ -241,6 +241,17 @@ permissions:
   - name: "group:read"
     roles: ["aifabrix-platform-admin", "aifabrix-security-admin", "aifabrix-compliance-admin", "aifabrix-observer"]
     description: "Read group information"
+
+  # Administration workspace (Dataplane UI /api/v1/admin/* shell)
+  - name: "administration:read"
+    roles:
+      [
+        "aifabrix-platform-admin",
+        "aifabrix-security-admin",
+        "aifabrix-deployment-admin",
+        "aifabrix-compliance-admin",
+      ]
+    description: "Access Administration workspace (environment settings, diagnostics, secrets navigation)"
   
   # OpenAPI / MCP spec bundle (mounted specs under /api/v1/specs)
   - name: "spec:read"

package/templates/applications/miso-controller/application.yaml

@@ -59,7 +59,7 @@ build:
   dockerfile: builder/miso-controller/Dockerfile # Dockerfile name (relative to project root)
   envOutputPath: ../../packages/miso-controller/.env # Copy .env to repo root for local dev (relative to builder/) (if null, no .env file is copied) (if empty, .env file is copied to repo root)
   language: typescript # Runtime language for template selection (typescript or python)
-  reloadStart: pnpm run start:reload # When running with --reload
+  reloadStart: pnpm --filter @aifabrix/miso-controller run dev:watch:public # Docker --reload: nodemon backend + pre-built UI (no Vite in container)
 
 # Repository Configuration (pipeline validate: REPOSITORY_URL_MISMATCH)
 repository:

package/templates/applications/miso-controller/env.template

@@ -157,6 +157,8 @@ WAIT_FOR_KEYCLOAK=true
 # For production: use public domain (e.g., https://miso.example.com)
 # url://public includes front-door path from application.yaml (e.g. /controller).
 MISO_WEB_SERVER_URL=url://public
+# Provider-grant OAuth redirect_uri base (Azure/start-dev also set this explicitly; code falls back to MISO_WEB_SERVER_URL)
+CONTROLLER_BASE_URL=url://public
 MISO_CONTROLLER_URL=url://internal
 MISO_RELATIVE_PATH=url://vdir-public
 ALLOWED_ORIGINS=http://localhost:*,url://host-public,url://host-private,url://dataplane-host-public,url://dataplane-host-private
@@ -241,6 +243,9 @@ DEPLOYMENT=database
 # Encryption Key for Database Secrets
 ENCRYPTION_KEY=kv://miso-controller-secrets-encryptionKeyVault
 
+# Comma-separated application keys allowed for POST .../applications/bootstrap (install-time only)
+BOOTSTRAP_APPKEYS=dataplane
+
 # Key Vault Integration (for security parameter encryption)
 # Set to true to enable Azure Key Vault for storing security parameters
 # When false (default), uses local AES-256-GCM encryption with ENCRYPTION_KEY

package/templates/infra/compose.yaml.hbs

@@ -116,6 +116,9 @@ services:
       - "--providers.docker=true"
       - "--providers.docker.exposedbydefault=false"
       - "--providers.docker.network={{networkName}}"
+{{#if traefikScoped}}
+      - "--providers.docker.constraints=Label(`aifabrix.traefik.network`,`{{networkName}}`)"
+{{/if}}
       - "--providers.docker.allowEmptyServices=true"
       - "--log.level=INFO"
       - "--entrypoints.web.address=:80"
@@ -123,29 +126,30 @@ services:
 {{#if traefik.trustForwardedHeaders}}
       - "--entrypoints.web.forwardedHeaders.insecure=true"
 {{/if}}
-{{#if traefik.certStore}}
-{{#if traefik.certFile}}
-      - "--certificatesstores.{{traefik.certStore}}.defaultcertificate.certfile={{traefik.certFile}}"
-{{/if}}
-{{#if traefik.keyFile}}
-      - "--certificatesstores.{{traefik.certStore}}.defaultcertificate.keyfile={{traefik.keyFile}}"
-{{/if}}
+{{#if traefik.tlsDynamicContainerPath}}
+      - "--providers.file.filename={{traefik.tlsDynamicContainerPath}}"
 {{/if}}
     ports:
       - "{{traefikHttpPort}}:80"
       - "{{traefikHttpsPort}}:443"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock:ro
+{{#if traefik.tlsDynamicBind}}
+      - type: bind
+        source: "{{traefik.tlsDynamicBind}}"
+        target: "{{traefik.tlsDynamicContainerPath}}"
+        read_only: true
+{{/if}}
 {{#if traefik.certFile}}
       - type: bind
         source: "{{traefik.certFile}}"
-        target: "{{traefik.certFile}}"
+        target: "{{traefik.certContainerPath}}"
         read_only: true
 {{/if}}
 {{#if traefik.keyFile}}
       - type: bind
         source: "{{traefik.keyFile}}"
-        target: "{{traefik.keyFile}}"
+        target: "{{traefik.keyContainerPath}}"
         read_only: true
 {{/if}}
     networks:

package/templates/python/docker-compose.hbs

@@ -9,6 +9,9 @@ services:
     {{#if traefik.enabled}}
     labels:
       - "traefik.enable=true"
+{{#if traefikScoped}}
+      - "aifabrix.traefik.network={{networkName}}"
+{{/if}}
       - "traefik.http.routers.{{app.key}}.rule=Host(`{{traefik.host}}`) && PathPrefix(`{{traefik.path}}`)"
       {{#if traefik.tls}}
       - "traefik.http.routers.{{app.key}}.entrypoints=websecure"

package/templates/typescript/docker-compose.hbs

@@ -9,6 +9,9 @@ services:
     {{#if traefik.enabled}}
     labels:
       - "traefik.enable=true"
+{{#if traefikScoped}}
+      - "aifabrix.traefik.network={{networkName}}"
+{{/if}}
       - "traefik.http.routers.{{app.key}}.rule=Host(`{{traefik.host}}`) && PathPrefix(`{{traefik.path}}`)"
       {{#if traefik.tls}}
       - "traefik.http.routers.{{app.key}}.entrypoints=websecure"