@aifabrix/builder 2.55.2 → 2.57.0

package/lib/deployment/marketplace-environment.js

@@ -0,0 +1,305 @@
+'use strict';
+
+// Marketplace journals are durable runtime state. Use the repository's real-sync
+// adapter so a caller (or another Jest suite) mocking `fs` cannot make an
+// existing completed journal disappear and trigger a duplicate ARM deployment.
+const fs = require('../internal/fs-real-sync');
+const path = require('path');
+const crypto = require('crypto');
+const { spawnSync } = require('child_process');
+const config = require('../core/config');
+const { getRuntimeConfigDir } = require('../core/config-runtime-paths');
+
+const SHIPPED_TEMPLATE = path.resolve(__dirname, '../../templates/marketplace/main.json');
+// Resolved per call (not a module-load constant) so it honors AIFABRIX_HOME/AIFABRIX_CONFIG
+// the same way the rest of the CLI's config does -- including the Jest sandbox tests use.
+function journalDir() {
+  return path.join(getRuntimeConfigDir(), 'installations');
+}
+const SAFE_NAME = /^[a-z][a-z0-9-]{2,39}$/;
+const SUBSCRIPTION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+function readJson(file, label) {
+  const resolved = path.resolve(file);
+  let document;
+  try {
+    document = JSON.parse(fs.readFileSync(resolved, 'utf8'));
+  } catch (_) {
+    throw new Error(`${label} is missing or invalid JSON: ${resolved}`);
+  }
+  return { document, resolved };
+}
+
+function parameterValue(parameters, name) {
+  return parameters?.parameters?.[name]?.value;
+}
+
+function validateStageTarget(envKey, options) {
+  if (!['dev', 'tst', 'pro'].includes(envKey)) {
+    throw new Error('Marketplace platform stage requires an entitled workload environment: dev, tst or pro');
+  }
+  const installId = String(options.installId || '');
+  const resourceGroup = String(options.resourceGroup || '');
+  if (!SAFE_NAME.test(installId) || (resourceGroup && !SAFE_NAME.test(resourceGroup))) {
+    throw new Error('Install ID and resource group must be lowercase Azure-safe names (3-40 characters)');
+  }
+  const stage = options.stage || 'infra';
+  if (!['infra', 'platform', 'all'].includes(stage)) {
+    throw new Error('Stage must be infra, platform or all');
+  }
+  if (stage === 'all' && !options.controllerUrl) {
+    throw new Error('--stage all requires --controller-url from a completed wizard; use --stage infra for a new installation');
+  }
+  if (stage === 'platform') {
+    return { envKey, installId, stage, controllerUrl: options.controllerUrl };
+  }
+  return { envKey, installId, resourceGroup, stage };
+}
+
+function validateInfraFiles(target, options) {
+  const { resourceGroup } = target;
+  if (!resourceGroup || !SUBSCRIPTION_ID.test(String(options.subscription || ''))) {
+    throw new Error('Infrastructure stage requires --subscription and --resource-group');
+  }
+  if (!options.parameters) {
+    throw new Error('Infrastructure stage requires a secure --parameters file');
+  }
+  const template = readJson(options.marketplaceJson || SHIPPED_TEMPLATE, 'Marketplace template');
+  const params = readJson(options.parameters, 'ARM parameter file');
+  if (fs.statSync(params.resolved).mode & 0o077) {
+    throw new Error('ARM parameter file may contain secrets and must be owner-only (chmod 600)');
+  }
+  if (!template.document.resources || !template.document.parameters || !params.document.parameters) {
+    throw new Error('Expected an ARM deployment template and deploymentParameters JSON');
+  }
+  return { template, params };
+}
+
+function validateInfraParameters(target, options, files) {
+  const { installId, resourceGroup } = target;
+  const { params } = files;
+  if (typeof parameterValue(params.document, 'adminPassword') !== 'string' ||
+      !parameterValue(params.document, 'adminPassword')) {
+    throw new Error('Secure ARM parameter file must supply adminPassword');
+  }
+  // Bicep module 14_marketplace-mori-bootstrap.bicep's own contract: empty/absent
+  // marketplaceSubscriptionId is a valid, intentional direct deploy -- Mori falls back
+  // to Community. Only reject a wrong *type*; do not require a non-empty value here,
+  // that would force a fabricated id past Mori's real Marketplace Fulfillment check.
+  const marketplaceSubscriptionId = parameterValue(params.document, 'marketplaceSubscriptionId');
+  if (marketplaceSubscriptionId !== undefined && typeof marketplaceSubscriptionId !== 'string') {
+    throw new Error('marketplaceSubscriptionId must be a string when present (empty string means a direct deploy, Community fallback)');
+  }
+  const nameOverride = parameterValue(params.document, 'infrastructureNameOverride');
+  if ((nameOverride && nameOverride !== installId) || (!nameOverride && resourceGroup !== installId)) {
+    throw new Error('Installation naming mismatch: infrastructureNameOverride must equal --install-id, or install ID must equal resource group');
+  }
+  const location = parameterValue(params.document, 'location') || options.location;
+  if (typeof location !== 'string' || !/^[a-z0-9]+$/i.test(location)) {
+    throw new Error('Infrastructure stage requires a valid Azure location in the parameter file or --location');
+  }
+  return location;
+}
+
+function preflight(envKey, options) {
+  const target = validateStageTarget(envKey, options);
+  if (target.stage === 'platform') return target;
+  const files = validateInfraFiles(target, options);
+  const location = validateInfraParameters(target, options, files);
+  return {
+    ...target, location,
+    subscription: options.subscription,
+    template: files.template.resolved,
+    parameters: files.params.resolved,
+    templateHash: crypto.createHash('sha256').update(fs.readFileSync(files.template.resolved)).digest('hex')
+  };
+}
+
+function journalPath(installId) {
+  return path.join(journalDir(), `${installId}.json`);
+}
+
+function readJournal(installId) {
+  const file = journalPath(installId);
+  if (!fs.existsSync(file)) throw new Error(`No infrastructure journal for ${installId}. Run --stage infra first.`);
+  return JSON.parse(fs.readFileSync(file, 'utf8'));
+}
+
+function writeJournal(record) {
+  fs.mkdirSync(journalDir(), { recursive: true, mode: 0o700 });
+  fs.writeFileSync(journalPath(record.installId), `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 });
+}
+
+function runAz(args) {
+  const result = spawnSync('az', args, { encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 });
+  if (result.error || result.status !== 0) {
+    throw new Error(`Azure CLI ${args.slice(0, 3).join(' ')} failed (exit ${result.status ?? 'unavailable'}). Check Azure deployment diagnostics; secret-bearing output was withheld.`);
+  }
+  return result.stdout.trim();
+}
+
+function azureArgs(base, subscription) {
+  return [...base, '--subscription', subscription];
+}
+
+function checkExistingJournal(input) {
+  const old = fs.existsSync(journalPath(input.installId)) ? readJournal(input.installId) : null;
+  if (old) {
+    // Workspace mode (inputHash present) owns template/parameter updates through the input hash;
+    // the legacy explicit mode still rejects any template change for an existing install ID.
+    if (old.subscription !== input.subscription || old.resourceGroup !== input.resourceGroup ||
+        (!input.inputHash && old.templateHash !== input.templateHash)) {
+      throw new Error('Existing install journal has a different subscription, group or template; use a new install ID');
+    }
+    if (old.infraComplete && old.inputHash === input.inputHash) return old;
+  }
+  return null;
+}
+
+function isOwnResumableDeployment(input, az) {
+  // ARM deployments are declarative/idempotent by name: re-running `deployment group
+  // create` with the same name only touches resources that still need work (e.g. a
+  // nested deployment script that failed) and leaves already-succeeded ones alone. A
+  // resource group that already has resources from *our own* prior attempt at this
+  // install ID is therefore safe to resume into, not "not empty" in the rejecting sense.
+  const deploymentName = `${input.installId}-marketplace-infra`;
+  const deployments = JSON.parse(az(azureArgs([
+    'deployment', 'group', 'list', '--resource-group', input.resourceGroup, '--output', 'json'
+  ], input.subscription)));
+  return deployments.some(deployment => deployment.name === deploymentName);
+}
+
+function ensureEmptyGroup(input, az) {
+  const account = JSON.parse(az(['account', 'show', '--output', 'json']));
+  if (account.id !== input.subscription) {
+    throw new Error('Selected az account subscription differs from --subscription; run az account set before deployment');
+  }
+  const exists = az(azureArgs(['group', 'exists', '--name', input.resourceGroup, '--output', 'tsv'], input.subscription)) === 'true';
+  if (exists) {
+    const resources = JSON.parse(az(azureArgs([
+      'resource', 'list', '--resource-group', input.resourceGroup, '--output', 'json'
+    ], input.subscription)));
+    if (resources.length && !isOwnResumableDeployment(input, az)) {
+      throw new Error('Target resource group is not empty; choose a new isolated group');
+    }
+  } else {
+    az(azureArgs(['group', 'create', '--name', input.resourceGroup, '--location', input.location, '--output', 'none'], input.subscription));
+  }
+}
+
+function runArmDeployment(input, az) {
+  const deploymentName = `${input.installId}-marketplace-infra`;
+  const common = [
+    '--resource-group', input.resourceGroup, '--name', deploymentName,
+    '--template-file', input.template, '--parameters', `@${input.parameters}`,
+    '--no-prompt', 'true'
+  ];
+  az(azureArgs(['deployment', 'group', 'validate', ...common, '--output', 'none'], input.subscription));
+  az(azureArgs(['deployment', 'group', 'what-if', ...common,
+    '--result-format', 'ResourceIdOnly', '--output', 'none'], input.subscription));
+  return { deploymentName, deployment: JSON.parse(az(azureArgs([
+    'deployment', 'group', 'create', ...common, '--output', 'json'
+  ], input.subscription))) };
+}
+
+function deployInfra(input, az = runAz) {
+  const old = checkExistingJournal(input);
+  if (old) return old;
+  ensureEmptyGroup(input, az);
+  const { deploymentName, deployment } = runArmDeployment(input, az);
+  const outputMap = deployment.properties?.outputs || {};
+  const outputs = Object.fromEntries(Object.entries(outputMap).map(([key, item]) => [key, item.value]));
+  if (!outputs.webAppUrl || !outputs.keyVaultName || !outputs.resourceGroupName) {
+    throw new Error('ARM deployment succeeded without required controller/vault/group outputs; inspect deployment before resume');
+  }
+  const record = {
+    installId: input.installId, envKey: input.envKey, subscription: input.subscription,
+    resourceGroup: input.resourceGroup, templateHash: input.templateHash, inputHash: input.inputHash,
+    deploymentName, deploymentId: deployment.id, correlationId: deployment.properties?.correlationId || null,
+    outputs, infraComplete: true,
+    platformComplete: false, recordedAt: new Date().toISOString()
+  };
+  writeJournal(record);
+  return record;
+}
+
+function assertAppDeploymentComplete(appName, deployment) {
+  if (deployment?.usedExternalDeploy || deployment?.result?.status?.status !== 'completed') {
+    throw new Error(`${appName} deployment did not reach completed status; resume platform stage after inspecting controller deployment`);
+  }
+}
+
+async function deployWorkloadApps(input, dependencies) {
+  const deployApp = dependencies.deployApp || require('../app').deployApp;
+  const deployEnvironment = dependencies.deployEnvironment || require('./environment').deployEnvironment;
+  const environment = await deployEnvironment(input.envKey, { poll: true });
+  if (environment?.status !== 'ready') {
+    throw new Error(`Environment ${input.envKey} did not reach ready status; resume platform stage after controller deployment completes`);
+  }
+  await (dependencies.setEnvironment || config.setCurrentEnvironment)(input.envKey);
+  assertAppDeploymentComplete('dataplane', await deployApp('dataplane', { yes: true, poll: true }));
+  assertAppDeploymentComplete('builder-api', await deployApp('builder-api', { yes: true, poll: true }));
+}
+
+async function deployPlatform(input, dependencies = {}) {
+  const record = (dependencies.readJournal || readJournal)(input.installId);
+  if (record.envKey !== input.envKey) throw new Error('Journal workload environment differs from requested environment');
+  if (record.platformComplete) return record;
+  const controller = input.controllerUrl || record.publicControllerUrl;
+  if (typeof controller !== 'string' || !/^https:\/\/[^/]+\/miso\/?$/.test(controller)) {
+    throw new Error('Platform stage requires --controller-url https://<ready-public-host>/miso from the completed wizard');
+  }
+  if (!record.runtimeReady) {
+    throw new Error('Controller runtime is not ready; rerun infrastructure readiness before the platform stage');
+  }
+  await (dependencies.setControllerUrl || config.setControllerUrl)(controller);
+  await deployWorkloadApps(input, dependencies);
+  record.publicControllerUrl = controller.replace(/\/$/, '');
+  record.platformComplete = true;
+  record.platformRecordedAt = new Date().toISOString();
+  (dependencies.writeJournal || writeJournal)(record);
+  return record;
+}
+
+async function deployMarketplaceEnvironment(envKey, options, dependencies = {}) {
+  const input = preflight(envKey, options);
+  if (input.stage === 'platform') return deployPlatform(input, dependencies);
+  const record = deployInfra(input, dependencies.az);
+  const probe = dependencies.probe || probeController;
+  await probe(record.outputs.webAppUrl);
+  record.runtimeReady = true;
+  writeJournal(record);
+  if (input.stage === 'infra') return record;
+  return deployPlatform({ ...input, controllerUrl: options.controllerUrl }, dependencies);
+}
+
+async function probeController(controllerUrl) {
+  return probeHttps(controllerUrl, '/api/v1/health/readiness', 'Controller');
+}
+
+/**
+ * Waits until an HTTPS endpoint answers successfully (App Service startup lags ARM success).
+ * @param {string} baseUrl - HTTPS base URL from ARM outputs
+ * @param {string} pathname - Probe path
+ * @param {string} label - Service label for errors
+ * @param {number} [attempts=30] - Attempts at 10-second intervals
+ * @returns {Promise<void>}
+ */
+async function probeHttps(baseUrl, pathname, label, attempts = 30) {
+  if (typeof baseUrl !== 'string' || !baseUrl.startsWith('https://')) {
+    throw new Error(`ARM output does not contain a valid HTTPS ${label} URL`);
+  }
+  const url = new URL(pathname, baseUrl);
+  for (let attempt = 0; attempt < attempts; attempt++) {
+    try {
+      const response = await fetch(url, { signal: AbortSignal.timeout(5000) });
+      if (response.ok) return;
+    } catch (_) {
+      // App Service startup can lag successful ARM provisioning.
+    }
+    await new Promise(resolve => setTimeout(resolve, 10000));
+  }
+  throw new Error(`${label} readiness did not pass at ${url.origin}; inspect App Service before resuming`);
+}
+
+module.exports = { preflight, deployInfra, deployPlatform, deployMarketplaceEnvironment, runAz, probeHttps, SAFE_NAME };

package/lib/external-system/deploy.js

@@ -28,7 +28,6 @@ const { generateControllerManifest } = require('../generator/external-controller
 const { validateExternalSystemComplete } = require('../validation/validate');
 const { displayValidationResults } = require('../validation/validate-display');
 const { syncDeployJsonFromSources } = require('./sync-deploy-manifest');
-const { runDeployGateOrchestrator } = require('../commands/deploy-gate-orchestrator');
 const { runPostPublishDataSync } = require('../commands/run-post-publish-data-sync');
 const { uploadDigitalWorkerIfNeeded, deployRoleAssistantShellIfNeeded } = require('./deploy-helpers');
 const { createDataplanePublishedCheck } = require('./deploy-dataplane-published');
@@ -199,7 +198,9 @@ function buildExternalDeployPollOpts(options, extra = {}) {
   return {
     ...options,
     ...extra,
-    poll: true,
+    // Default true (CLI blocks and shows a spinner); explicit false (builder-api web path,
+    // plan 191.0 task 3) accepts and returns immediately — the caller polls its own status route.
+    poll: options.poll !== false,
     pollInterval:
       options.pollInterval !== undefined ? options.pollInterval : EXTERNAL_DEPLOY_POLL_INTERVAL_MS,
     pollMaxAttempts:
@@ -210,28 +211,15 @@ function buildExternalDeployPollOpts(options, extra = {}) {
   };
 }
 
-async function promoteExternalSystemLifecycleAfterDeploy(manifest, dataplaneUrl, authConfig) {
-  const { buildUploadPayload } = require('../commands/upload');
-  const { uploadApplicationViaPipelineWithAuthRetry } = require('../utils/pipeline-upload-auth-retry');
-  const payload = buildUploadPayload(manifest, { status: 'published' });
-  const res = await uploadApplicationViaPipelineWithAuthRetry(
-    dataplaneUrl,
-    authConfig,
-    payload,
-    manifest.key
-  );
-  if (res?.success === false) {
-    throw new Error(
-      res.formattedError || res.error || 'Failed to promote external system lifecycle after deploy'
-    );
-  }
-  return res;
-}
-
+/**
+ * Reads back dataplane readiness after Miso's deploy call. Miso (226.0/191.0 addendum) already
+ * decided and executed fast-qualified promotion or the legacy full ladder server-side — Builder
+ * never re-publishes here; that was the duplicate work this task removes.
+ */
 async function finalizeExternalDeployDataplaneWork(manifest, controllerUrl, environment, authConfig, options) {
   const compact = options.verbose !== true;
   const silent = compact;
-  const ctx = await fetchDataplaneDeployReadiness(
+  const refreshedCtx = await fetchDataplaneDeployReadiness(
     controllerUrl,
     environment,
     authConfig,
@@ -239,21 +227,6 @@ async function finalizeExternalDeployDataplaneWork(manifest, controllerUrl, envi
     { dataplaneUrl: options.knownDataplaneUrl, silent }
   );
 
-  if (ctx.dataplaneUrl && !ctx.error) {
-    await promoteExternalSystemLifecycleAfterDeploy(manifest, ctx.dataplaneUrl, authConfig);
-  }
-
-  const refreshedCtx =
-    ctx.dataplaneUrl && !ctx.error
-      ? await fetchDataplaneDeployReadiness(
-        controllerUrl,
-        environment,
-        authConfig,
-        manifest.key,
-        { dataplaneUrl: ctx.dataplaneUrl, silent: true }
-      )
-      : ctx;
-
   const probeData = await runOptionalExternalDeployProbe(refreshedCtx, manifest.key, authConfig, options);
   return { refreshedCtx, probeData };
 }
@@ -294,6 +267,21 @@ async function runPostDeployExternalSystemSteps(manifest, controllerUrl, environ
   });
 }
 
+// Builder-api web path (plan 191.0 task 3): accept-only, raw controller accept response
+// (deploymentId, not-yet-terminal status) — caller polls getDeployStatus for terminal truth.
+function logCompactDeployFailure(manifest, environment, deploymentOutcome, options, result) {
+  logDeployReadinessSummary({
+    environment,
+    dataplaneUrl: null,
+    manifest,
+    deploymentOk: false,
+    deploymentDetail: deploymentOutcome,
+    verbose: false,
+    gateOutcomes: options.gateOutcomes || null,
+    draftDeploymentId: result?.deploymentId || null
+  });
+}
+
 async function executeDeployAndDisplay(manifest, controllerUrl, environment, authConfig, options) {
   const compact = options.verbose !== true;
   const deployer = require('../deployment/deployer');
@@ -313,6 +301,10 @@ async function executeDeployAndDisplay(manifest, controllerUrl, environment, aut
     })
   );
 
+  if (options.poll === false) {
+    return result;
+  }
+
   const deploymentOutcome = parseControllerDeploymentOutcome(result);
   if (!compact) {
     logImmediateControllerDeploymentOutcome(deploymentOutcome);
@@ -320,16 +312,7 @@ async function executeDeployAndDisplay(manifest, controllerUrl, environment, aut
 
   if (!deploymentOutcome.ok) {
     if (compact) {
-      logDeployReadinessSummary({
-        environment,
-        dataplaneUrl: null,
-        manifest,
-        deploymentOk: false,
-        deploymentDetail: deploymentOutcome,
-        verbose: false,
-        gateOutcomes: options.gateOutcomes || null,
-        draftDeploymentId: result?.deploymentId || null
-      });
+      logCompactDeployFailure(manifest, environment, deploymentOutcome, options, result);
     }
     return result;
   }
@@ -422,15 +405,28 @@ async function validateDataplaneManifestForDeploy(manifest, authConfig, dataplan
   }
 }
 
+/**
+ * Deploy no longer runs local unit/integration tests or optional --gate certification steps
+ * (plan 191.0 task 2): that ladder now runs once, durably, during `aifabrix upload`. A caller
+ * still passing --gate gets an actionable notice instead of a silently-ignored flag.
+ */
+function warnIfDeployGateFlagIsNowNoop(systemKey, options) {
+  if (!options.gate) {
+    return;
+  }
+  logger.log(
+    chalk.yellow(
+      `⚠ --gate is no longer run during deploy; it runs once during "aifabrix upload ${systemKey}".`
+    )
+  );
+}
+
 async function runConnectedSystemDeployAfterDetect(appName, options, compact) {
   if (!compact) {
     logger.log(chalk.blue(`\n🚀 Deploying external system: ${appName}`));
   }
   await validateExternalSystemForDeploy(appName, options, compact);
-  const gateOutcomes = await runDeployGateOrchestrator(appName, {
-    ...options,
-    quietDeployUi: options.json === true
-  });
+  warnIfDeployGateFlagIsNowNoop(appName, options);
   await syncDeployJsonFromSources(appName);
   const manifest = await generateControllerManifest(appName, options);
   const digitalWorkerResult = await uploadDigitalWorkerIfNeeded(
@@ -454,7 +450,6 @@ async function runConnectedSystemDeployAfterDetect(appName, options, compact) {
   await validateDataplaneManifestForDeploy(manifest, authConfig, dataplaneUrl, compact);
   return executeDeployAndDisplay(manifest, controllerUrl, environment, authConfig, {
     ...options,
-    gateOutcomes,
     knownDataplaneUrl: dataplaneUrl
   });
 }

package/lib/generator/builders.js

@@ -359,6 +359,10 @@ function addSimpleOptionalFields(deployment, variables) {
   if (variables.environmentScopedResources === true) {
     deployment.environmentScopedResources = true;
   }
+  // Non-arrays pass through so schema validation reports them; an empty list stays omitted.
+  if (variables.required !== undefined && !(Array.isArray(variables.required) && variables.required.length === 0)) {
+    deployment.required = variables.required;
+  }
 }
 
 function buildOptionalFields(deployment, variables, rbac) {