@aifabrix/builder 2.0.0 → 2.0.3

package/lib/validator.js

@@ -14,6 +14,9 @@ const path = require('path');
 const yaml = require('js-yaml');
 const Ajv = require('ajv');
 const applicationSchema = require('./schema/application-schema.json');
+const { transformVariablesForValidation } = require('./utils/variable-transformer');
+const { checkEnvironment } = require('./utils/environment-checker');
+const { formatValidationErrors } = require('./utils/error-formatter');
 
 /**
  * Validates variables.yaml file against application schema
@@ -49,9 +52,12 @@ async function validateVariables(appName) {
     throw new Error(`Invalid YAML syntax in variables.yaml: ${error.message}`);
   }
 
+  // Transform nested structure to flat schema format
+  const transformed = transformVariablesForValidation(variables, appName);
+
   const ajv = new Ajv({ allErrors: true, strict: false });
   const validate = ajv.compile(applicationSchema);
-  const valid = validate(variables);
+  const valid = validate(transformed);
 
   return {
     valid,
@@ -209,125 +215,33 @@ async function validateEnvTemplate(appName) {
 }
 
 /**
- * Checks the development environment for common issues
- * Validates Docker, ports, secrets, and other requirements
+ * Validates deployment JSON against application schema
+ * Ensures generated aifabrix-deploy.json matches the schema structure
  *
- * @async
- * @function checkEnvironment
- * @returns {Promise<Object>} Environment check result
- * @throws {Error} If critical issues are found
+ * @function validateDeploymentJson
+ * @param {Object} deployment - Deployment JSON object to validate
+ * @returns {Object} Validation result with errors
  *
  * @example
- * const result = await checkEnvironment();
- * // Returns: { docker: 'ok', ports: 'ok', secrets: 'missing', recommendations: [...] }
+ * const result = validateDeploymentJson(deployment);
+ * // Returns: { valid: true, errors: [] }
  */
-async function checkDocker() {
-  try {
-    const { exec } = require('child_process');
-    const { promisify } = require('util');
-    const execAsync = promisify(exec);
-
-    await execAsync('docker --version');
-    await execAsync('docker-compose --version');
-    return 'ok';
-  } catch (error) {
-    return 'error';
-  }
-}
-
-async function checkPorts() {
-  const requiredPorts = [5432, 6379, 5050, 8081];
-  const netstat = require('net');
-  let portIssues = 0;
-
-  for (const port of requiredPorts) {
-    try {
-      await new Promise((resolve, reject) => {
-        const server = netstat.createServer();
-        server.listen(port, () => {
-          server.close(resolve);
-        });
-        server.on('error', reject);
-      });
-    } catch (error) {
-      portIssues++;
-    }
-  }
-
-  return portIssues === 0 ? 'ok' : 'warning';
-}
-
-function checkSecrets() {
-  const os = require('os');
-  const secretsPath = path.join(os.homedir(), '.aifabrix', 'secrets.yaml');
-  return fs.existsSync(secretsPath) ? 'ok' : 'missing';
-}
-
-async function checkEnvironment() {
-  const result = {
-    docker: 'unknown',
-    ports: 'unknown',
-    secrets: 'unknown',
-    recommendations: []
-  };
-
-  // Check Docker
-  result.docker = await checkDocker();
-  if (result.docker === 'error') {
-    result.recommendations.push('Install Docker and Docker Compose');
-  }
-
-  // Check ports
-  result.ports = await checkPorts();
-  if (result.ports === 'warning') {
-    result.recommendations.push('Some required ports (5432, 6379, 5050, 8081) are in use');
-  }
-
-  // Check secrets
-  result.secrets = checkSecrets();
-  if (result.secrets === 'missing') {
-    result.recommendations.push('Create secrets file: ~/.aifabrix/secrets.yaml');
+function validateDeploymentJson(deployment) {
+  if (!deployment || typeof deployment !== 'object') {
+    return {
+      valid: false,
+      errors: ['Deployment must be an object']
+    };
   }
 
-  return result;
-}
+  const ajv = new Ajv({ allErrors: true, strict: false });
+  const validate = ajv.compile(applicationSchema);
+  const valid = validate(deployment);
 
-/**
- * Formats validation errors into developer-friendly messages
- * Converts technical schema errors into actionable advice
- *
- * @function formatValidationErrors
- * @param {Array} errors - Raw validation errors from Ajv
- * @returns {Array} Formatted error messages
- *
- * @example
- * const messages = formatValidationErrors(ajvErrors);
- * // Returns: ['Port must be between 1 and 65535', 'Missing required field: displayName']
- */
-function formatSingleError(error) {
-  const path = error.instancePath ? error.instancePath.slice(1) : 'root';
-  const field = path ? `Field "${path}"` : 'Configuration';
-
-  const errorMessages = {
-    required: `${field}: Missing required property "${error.params.missingProperty}"`,
-    type: `${field}: Expected ${error.params.type}, got ${typeof error.data}`,
-    minimum: `${field}: Value must be at least ${error.params.limit}`,
-    maximum: `${field}: Value must be at most ${error.params.limit}`,
-    minLength: `${field}: Must be at least ${error.params.limit} characters`,
-    maxLength: `${field}: Must be at most ${error.params.limit} characters`,
-    pattern: `${field}: Invalid format`,
-    enum: `${field}: Must be one of: ${error.params.allowedValues?.join(', ') || 'unknown'}`
+  return {
+    valid,
+    errors: valid ? [] : formatValidationErrors(validate.errors)
   };
-
-  return errorMessages[error.keyword] || `${field}: ${error.message}`;
-}
-
-function formatValidationErrors(errors) {
-  if (!Array.isArray(errors)) {
-    return ['Unknown validation error'];
-  }
-
-  return errors.map(formatSingleError);
 }
 
 /**
@@ -371,6 +285,7 @@ module.exports = {
   validateVariables,
   validateRbac,
   validateEnvTemplate,
+  validateDeploymentJson,
   checkEnvironment,
   formatValidationErrors,
   validateApplication

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aifabrix/builder",
-  "version": "2.0.0",
+  "version": "2.0.3",
   "description": "AI Fabrix Local Fabric & Deployment SDK",
   "main": "lib/index.js",
   "bin": {
@@ -10,15 +10,18 @@
     "test": "jest --coverage",
     "test:watch": "jest --watch",
     "test:ci": "jest --ci --coverage --watchAll=false",
+    "test:integration": "jest --config jest.config.integration.js --runInBand",
+    "test:integration:python": "cross-env TEST_LANGUAGE=python jest --config jest.config.integration.js --runInBand",
+    "test:integration:typescript": "cross-env TEST_LANGUAGE=typescript jest --config jest.config.integration.js --runInBand",
     "lint": "eslint . --ext .js",
     "lint:fix": "eslint . --ext .js --fix",
     "lint:ci": "eslint . --ext .js --format json --output-file eslint-report.json",
     "dev": "node bin/aifabrix.js",
     "build": "npm run lint && npm run test:ci",
+    "pack": "npm run build && npm pack",
     "validate": "npm run build",
     "prepublishOnly": "npm run validate",
-    "precommit": "npm run lint:fix && npm run test",
-    "posttest": "npm run lint"
+    "precommit": "npm run lint:fix && npm run test"
   },
   "keywords": [
     "aifabrix",
@@ -34,19 +37,20 @@
     "node": ">=18.0.0"
   },
   "dependencies": {
-    "commander": "^11.1.0",
-    "js-yaml": "^4.1.0",
     "ajv": "^8.12.0",
-    "handlebars": "^4.7.8",
     "axios": "^1.6.0",
     "chalk": "^4.1.2",
-    "ora": "^5.4.1",
-    "inquirer": "^8.2.5"
+    "commander": "^11.1.0",
+    "handlebars": "^4.7.8",
+    "inquirer": "^8.2.5",
+    "js-yaml": "^4.1.0",
+    "ora": "^5.4.1"
   },
   "devDependencies": {
-    "jest": "^29.7.0",
+    "@types/node": "^20.10.0",
+    "cross-env": "^10.1.0",
     "eslint": "^8.55.0",
-    "@types/node": "^20.10.0"
+    "jest": "^29.7.0"
   },
   "repository": {
     "type": "git",

package/templates/README.md

@@ -1,15 +1,56 @@
 # AI Fabrix Builder Templates
 
-These are Handlebars (.hbs) template files that generate Docker files for AI Fabrix applications. They should NOT be linted as Dockerfiles since they contain template variables that will be replaced during generation.
+These are Handlebars (.hbs) template files that generate Docker files and application configurations for AI Fabrix applications. They should NOT be linted as Dockerfiles since they contain template variables that will be replaced during generation.
 
-## Template Files
+## Template Structure
+
+The templates directory is organized as follows:
+
+### Application Templates (for `--template` flag)
+
+Application templates are folder-based and located under `templates/applications/`. When you use `--template <name>`, the tool looks for `templates/applications/<name>/` and copies all files from that folder to `builder/<app>/`.
+
+**Example:**
+- `templates/applications/miso-controller/` - Miso Controller application template
+- `templates/applications/keycloak/` - Keycloak application template
+
+**Template Validation:**
+- Template folder must exist in `templates/applications/<name>/`
+- Template folder must contain at least one file
+- Hidden files (starting with `.`) are skipped
+- If a template includes a `Dockerfile`, it will be copied to `builder/<app>/Dockerfile` along with other files
+
+### Language Templates
 
 - `python/Dockerfile.hbs` - Python application Dockerfile template
 - `python/docker-compose.hbs` - Python application docker-compose template
 - `typescript/Dockerfile.hbs` - TypeScript/Node.js application Dockerfile template
 - `typescript/docker-compose.hbs` - TypeScript/Node.js application docker-compose template
+
+### Infrastructure Templates
+
 - `infra/compose.yaml` - Infrastructure services docker-compose template
 
+### GitHub Workflow Templates
+
+- `github/ci.yaml.hbs` - Continuous Integration workflow
+- `github/pr-checks.yaml.hbs` - Pull Request checks workflow
+- `github/release.yaml.hbs` - Release and publish workflow
+- `github/test.yaml.hbs` - Test workflow
+
+### GitHub Workflow Step Templates (for `--github-steps` flag)
+
+Extra workflow steps are located in `templates/github/steps/`. When you use `--github-steps <steps>`, the tool loads step templates from `templates/github/steps/{step}.hbs` and includes them in the generated workflows.
+
+**Example:**
+- `github/steps/npm.hbs` - NPM publishing step
+- `github/steps/test.hbs` - Custom test step
+
+**Step Templates:**
+- Step templates are Handlebars templates that generate workflow job definitions
+- They receive the same context as the main workflow templates
+- Step templates are rendered and included in workflow files based on the `githubSteps` array
+
 ## Template Variables
 
 ### Application Configuration
@@ -34,7 +75,38 @@ These are Handlebars (.hbs) template files that generate Docker files for AI Fab
 
 ## Usage
 
-These templates are processed by the AI Fabrix Builder SDK based on the application schema defined in `variables.yaml`. The generated files will be valid Docker files after template processing.
+### Application Templates
+
+Use `--template <name>` when creating an application:
+
+```bash
+aifabrix create myapp --template miso-controller --port 3000
+```
+
+This validates that `templates/applications/miso-controller/` exists and copies all files (including Dockerfile if present) from it to `builder/myapp/`.
+
+### GitHub Workflow Steps
+
+Use `--github-steps <steps>` when creating an application with GitHub workflows:
+
+```bash
+aifabrix create myapp --github --github-steps npm
+```
+
+This loads step templates from:
+- `templates/github/steps/npm.hbs`
+
+And includes them in the generated workflow files (e.g., `release.yaml`).
+
+**Currently available step templates:**
+- `npm.hbs` - Adds NPM publishing job to release workflow
+
+**Creating custom step templates:**
+Create your own step templates in `templates/github/steps/{your-step}.hbs` and reference them in `--github-steps`.
+
+### Language and Infrastructure Templates
+
+These templates are processed automatically by the AI Fabrix Builder SDK based on the application schema defined in `variables.yaml`. The generated files will be valid Docker files after template processing.
 
 ## VS Code Configuration
 

package/templates/applications/keycloak/Dockerfile

@@ -0,0 +1,36 @@
+# Keycloak Identity and Access Management with Custom Themes
+# This Dockerfile extends Keycloak 24.0 with custom themes
+
+FROM quay.io/keycloak/keycloak:24.0
+
+# Set working directory
+WORKDIR /opt/keycloak
+
+# Copy custom themes from share directory (only if themes/ folder exists)
+USER root
+# Use a shell script approach: copy themes if it exists, otherwise skip
+# First, copy everything from build context to a staging area
+COPY . /tmp/build-context/
+# Then conditionally copy themes if the directory exists
+RUN if [ -d "/tmp/build-context/themes" ] && [ "$(ls -A /tmp/build-context/themes)" ]; then \
+      cp -r /tmp/build-context/themes/* /opt/keycloak/themes/ && \
+      chown -R keycloak:keycloak /opt/keycloak/themes/ && \
+      echo "Themes copied successfully"; \
+    else \
+      echo "No themes directory found, skipping theme copy"; \
+    fi && \
+    rm -rf /tmp/build-context
+
+# Build Keycloak with health checks enabled
+# This enables the /health, /health/live, /health/ready, and /health/started endpoints
+RUN /opt/keycloak/bin/kc.sh build --health-enabled=true
+
+# Switch back to keycloak user
+USER keycloak
+
+# Keycloak runs on port 8080 internally, exposed as 8082
+EXPOSE 8080
+
+# Default Keycloak command (can be overridden in docker-compose.yaml)
+# Health checks are enabled via the build step above
+CMD ["start-dev"]

package/templates/applications/keycloak/env.template

@@ -0,0 +1,32 @@
+# Environment Variables Template
+# Use kv:// references for secrets (resolved from .aifabrix/secrets.yaml)
+# Use ${VAR} for environment-specific values
+
+# =============================================================================
+# APPLICATION ENVIRONMENT
+# =============================================================================
+
+KEYCLOAK_ADMIN=admin
+KEYCLOAK_ADMIN_PASSWORD=kv://keycloak-admin-passwordKeyVault
+KC_HOSTNAME_STRICT=false
+KC_HTTP_ENABLED=true
+
+# =============================================================================
+# HEALTH CHECK CONFIGURATION
+# =============================================================================
+# Enable health check endpoints: /health, /health/live, /health/ready, /health/started
+
+KC_HEALTH_ENABLED=true
+
+# =============================================================================
+# DATABASE CONFIGURATION
+# =============================================================================
+# Connects to postgres service in Docker network (postgres) or localhost (local)
+
+KC_DB=postgres
+KC_DB_URL_HOST=${DB_HOST}
+KC_DB_URL_PORT=5432
+KC_DB_URL_DATABASE=keycloak
+KC_DB_USERNAME=keycloak_user
+KC_DB_PASSWORD=kv://databases-keycloak-0-passwordKeyVault
+

package/templates/applications/keycloak/rbac.yaml

@@ -0,0 +1,37 @@
+roles:
+  - name: "AI Fabrix Admin"
+    value: "aifabrix-admin"
+    description: "Full access to all application features and configurations"
+  
+  - name: "AI Fabrix User"
+    value: "aifabrix-user"
+    description: "Basic user access to the application"
+  
+  - name: "AI Fabrix Developer"
+    value: "aifabrix-developer"
+    description: "Developer access for testing and debugging"
+
+permissions:
+  - name: "myapp:read"
+    roles: 
+      - "aifabrix-user"
+      - "aifabrix-admin"
+      - "aifabrix-developer"
+    description: "Read access to application data"
+  
+  - name: "myapp:write"
+    roles:
+      - "aifabrix-admin"
+      - "aifabrix-developer"
+    description: "Create and edit application data"
+  
+  - name: "myapp:delete"
+    roles:
+      - "aifabrix-admin"
+    description: "Delete application data"
+  
+  - name: "myapp:admin"
+    roles:
+      - "aifabrix-admin"
+    description: "Administrative access to application configuration"
+

package/templates/applications/keycloak/variables.yaml

@@ -0,0 +1,56 @@
+# Application Metadata
+app:
+  key: keycloak
+  displayName: "AI Fabrix Keycloak"
+  description: "Identity and Access Management"
+  type: webapp
+
+# Image Configuration
+image:
+  name: aifabrix/keycloak
+  tag: "latest"
+  registry: devflowiseacr.azurecr.io
+  registryMode: acr
+
+# Port Configuration
+port: 8082
+
+# Azure Requirements
+requires:
+  database: true
+  databases:
+    - name: keycloak
+  redis: false
+  storage: false
+
+# Health Check
+healthCheck:
+  path: /health
+  interval: 30
+  probePath: /health
+  probeRequestType: GET
+  probeProtocol: Https
+  probeIntervalInSeconds: 120
+
+# Authentication
+authentication:
+  type: keycloak
+  enableSSO: true
+  requiredRoles: ["aifabrix-user"]
+  endpoints:
+    local: "http://localhost:8082/auth/callback"
+
+# Build Configuration
+build:
+  context: ..                                           # Docker build context (relative to builder/)
+  dockerfile: builder/Dockerfile                        # Dockerfile name (empty = use template)
+  envOutputPath: .env                                   # Copy .env to repo root for local dev
+  localPort: 8082                                       # Port for local development (different from Docker port)
+  containerPort: 8080                                   # Container port (different from local port)
+  language: typescript                                  # Runtime language for template selection
+  secrets:                                              # Path to secrets file (optional)
+
+# Docker Compose
+compose:
+  file: docker-compose.yaml
+  service: keycloak

package/templates/applications/miso-controller/Dockerfile

@@ -0,0 +1,125 @@
+# AI Fabrix Miso Controller - Optimized Dockerfile
+FROM node:18-alpine
+
+# Install only essential runtime dependencies
+RUN apk add --no-cache \
+    curl \
+    wget \
+    openssl \
+    openssl-dev
+
+# Install PNPM globally
+RUN npm install -g pnpm
+
+# Install tsconfig-paths globally for path resolution
+RUN npm install -g tsconfig-paths
+
+# Set working directory
+WORKDIR /app
+
+# Copy package files for dependency resolution
+COPY package*.json ./
+COPY pnpm-workspace.yaml ./
+COPY packages/miso-azure/package*.json ./packages/miso-azure/
+COPY packages/miso-controller/package*.json ./packages/miso-controller/
+COPY packages/miso-controller/bin ./packages/miso-controller/bin
+COPY packages/miso-ui/package*.json ./packages/miso-ui/
+
+# Create .npmrc for hoisting to fix module resolution in Docker
+RUN echo "shamefully-hoist=true" > .npmrc
+
+# Install all dependencies (including dev for building)
+RUN pnpm install
+
+# Copy only the source code we need
+COPY packages/miso-azure/src ./packages/miso-azure/src
+COPY packages/miso-azure/tsconfig.json ./packages/miso-azure/
+COPY packages/miso-controller/src ./packages/miso-controller/src
+COPY packages/miso-controller/tsconfig.json ./packages/miso-controller/
+COPY packages/miso-controller/tsconfig.docker.json ./packages/miso-controller/
+COPY packages/miso-controller/openapi/openapi-complete.yaml ./packages/miso-controller/dist/openapi/
+COPY packages/miso-controller/openapi/openapi-complete.json ./packages/miso-controller/dist/openapi/
+COPY packages/miso-ui/src ./packages/miso-ui/src
+COPY packages/miso-ui/tsconfig.json ./packages/miso-ui/
+COPY packages/miso-ui/tsconfig.app.json ./packages/miso-ui/
+COPY packages/miso-ui/tsconfig.node.json ./packages/miso-ui/
+COPY packages/miso-ui/vite.config.ts ./packages/miso-ui/
+COPY packages/miso-ui/index.html ./packages/miso-ui/
+
+# Fix Rollup native module issue for Alpine Linux
+RUN pnpm add @rollup/rollup-linux-x64-musl @types/express-serve-static-core --workspace-root
+
+# Build packages
+WORKDIR /app/packages/miso-azure
+RUN pnpm run build
+
+WORKDIR /app/packages/miso-ui  
+RUN pnpm run build
+
+WORKDIR /app/packages/miso-controller
+RUN pnpm run db:generate
+RUN pnpm exec tsc -p tsconfig.docker.json || true
+# Copy sensitive-fields.config.json to dist folder
+RUN mkdir -p dist/src/services/logging && \
+    cp src/services/logging/sensitive-fields.config.json dist/src/services/logging/ || true
+
+# Return to root to prune correctly (needed to keep workspace dependencies)
+WORKDIR /app
+
+# Remove source files and build artifacts, but preserve Prisma schema files and OpenAPI files
+RUN mkdir -p packages/miso-controller/dist/database/prisma && \
+    cp packages/miso-controller/src/database/prisma/*.prisma packages/miso-controller/dist/database/prisma/ || true
+RUN rm -rf packages/miso-azure/src packages/miso-controller/src packages/miso-ui/src
+RUN rm -rf packages/miso-azure/tsconfig.json packages/miso-ui/tsconfig.json
+RUN rm -rf packages/*/node_modules packages/*/.git packages/*/.github
+
+# Reinstall to recreate symlinks after removing source files
+# This ensures workspace dependencies are properly linked
+RUN pnpm install --frozen-lockfile --filter "@aifabrix/miso-controller..." || pnpm install --frozen-lockfile
+
+# Manually remove known dev dependencies
+RUN rm -rf node_modules/markdownlint node_modules/markdownlint-cli node_modules/prettier
+RUN rm -rf .pnpm/markdownlint* .pnpm/prettier*
+
+# Create non-root user
+RUN addgroup -g 1001 -S nodejs && \
+    adduser -S miso -u 1001
+
+# Create necessary directories for mount points (with fallback to local storage)
+RUN mkdir -p /mnt/data/logs /mnt/data/data /mnt/data/backup /mnt/data/config
+RUN mkdir -p /app/data/logs /app/data/data /app/data/backup /app/data/config
+
+# Change ownership of app directory and mount points
+RUN chown -R miso:nodejs /app /mnt/data
+
+# Switch to non-root user
+USER miso
+
+# Set working directory to controller
+WORKDIR /app/packages/miso-controller
+
+# Expose port
+EXPOSE 3000
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
+  CMD wget --no-verbose --tries=1 --spider http://localhost:3000/health || exit 1
+
+# Set environment variable for tsconfig-paths
+ENV TS_NODE_PROJECT=tsconfig.docker.json
+
+# Set default data paths (can be overridden by environment variables)
+ENV LOG_PATH=/mnt/data/logs
+ENV DATA_PATH=/mnt/data/data
+ENV BACKUP_PATH=/mnt/data/backup
+ENV CONFIG_PATH=/mnt/data/config
+
+# Create symlinks for fallback to local storage if mounts are not provided
+RUN ln -sf /app/data/logs /mnt/data/logs || true
+RUN ln -sf /app/data/data /mnt/data/data || true  
+RUN ln -sf /app/data/backup /mnt/data/backup || true
+RUN ln -sf /app/data/config /mnt/data/config || true
+
+# Start the application
+CMD ["node", "-r", "tsconfig-paths/register", "dist/src/server.js"]
+