@gnar-engine/cli 1.0.4 → 1.0.6

package/install-from-clone.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -e
+
+TARGET_DIR="$HOME/.gnarengine"
+CLI_DIR="$(pwd)"
+
+echo "Installing Gnar Engine CLI from local source for user $USER..."
+
+mkdir -p "$TARGET_DIR"
+
+# Install dependencies
+cd "$CLI_DIR"
+npm install
+
+# Link CLI to custom global folder
+npm link
+
+# Bin path for npm link
+BIN_PATH="$TARGET_DIR/bin"
+
+# Add to shell PATH if not already present
+for SHELLRC in "$HOME/.bashrc" "$HOME/.zshrc"; do
+    [ -f "$SHELLRC" ] || continue
+    if ! grep -q "$BIN_PATH" "$SHELLRC"; then
+        echo "export PATH=\"$BIN_PATH:\$PATH\"" >>"$SHELLRC"
+        echo "Added $BIN_PATH to PATH in $SHELLRC"
+    fi
+done
+
+echo "Gnar Engine CLI installed! Restart your terminal and run 'gnar --help'"

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gnar-engine/cli",
-    "version": "1.0.4",
+    "version": "1.0.6",
     "description": "Gnar Engine Development Framework CLI: Project bootstrap, scaffolder & control plane.",
     "type": "module",
     "bin": {

package/src/cli.js

@@ -6,6 +6,7 @@ import { registerDevCommands } from './dev/commands.js';
 import { registerControlCommands } from './control/commands.js';
 import { registerScaffolderCommands } from './scaffolder/commands.js';
 import { registerAgentCommands } from './agent/commands.js';
+import path from 'path';
 
 // Create a new program
 const program = new Command();
@@ -28,3 +29,4 @@ G n a r  E n g i n e -  A powerful, AI ready microservice framework for modern a
 
 // Parse CLI input
 program.parse(process.argv);
+

package/src/config.js

@@ -1,3 +1,4 @@
+import path from 'path';
 
 export const gnarEngineCliConfig = {
 
@@ -7,3 +8,10 @@ export const gnarEngineCliConfig = {
     corePath: '/usr/gnar_engine/app/node_modules/@gnar-engine/core'
 
 }
+
+export const directories = {
+    scaffolderServiceTemplates: path.join(import.meta.dirname, '../templates/service'),
+    scaffolderEntityTemplates: path.join(import.meta.dirname, '../templates/entity'),
+    bootstrap: path.join(import.meta.dirname, '../bootstrap'),
+    provisioner: path.join(import.meta.dirname, './provisioner')
+};

package/src/dev/commands.js

@@ -10,8 +10,10 @@ export function registerDevCommands(program) {
 	devCmd
 		.command('up')
 		.description('🛠️  Up Development Containers')
-		.option('-b, --build', 'Ruild without cache')
+		.option('-b, --build', 'Build without cache')
         .option('-d, --detach', 'Run containers in background')
+        .option('-t --test', 'Run the tests with ephemeral databases')
+        .option('--test-service <service>', 'Run the tests for the specified service with ephemeral databases (e.g. --test-service user)')
         .addOption(new Option('--core-dev').hideHelp())
         .action(async (options) => {
 			let response = {};
@@ -27,13 +29,19 @@ export function registerDevCommands(program) {
 			// Change to the active profile directory
 			const projectDir = activeProfile.PROJECT_DIR;
 
+            if (options.testService) {
+                options.test = true;
+            }
+
 			try {
 				up({
                     projectDir: projectDir,
                     build: options.build || false,
                     detach: options.detach || false,
-                    coreDev: options.coreDev || false
-                });
+                    coreDev: options.coreDev || false,
+                    test: options.test || false,
+                    testService: options.testService || ''
+                 });
 			} catch (err) {
 				console.error("❌ Error running containers:", err.message);
 				process.exit(1);

package/src/dev/dev.service.js

@@ -5,6 +5,7 @@ import fs from "fs/promises";
 import path from "path";
 import yaml from "js-yaml";
 import { gnarEngineCliConfig } from "../config.js";
+import { directories } from "../config.js";
 
 const docker = new Docker();
 
@@ -16,10 +17,13 @@ const docker = new Docker();
  * @param {object} options
  * @param {string} options.projectDir - The project directory
  * @param {boolean} [options.build=false] - Whether to re-build images
- * @param {boolean} [options.detached=false] - Whether to run containers in background
+ * @param {boolean} [options.detach=false] - Whether to run containers in background
  * @param {boolean} [options.coreDev=false] - Whether to run in core development mode (requires access to core source)
+ * @param {boolean} [options.test=false] - Whether to run tests with ephemeral databases
+ * @param {string} [options.testService=''] - The service to run tests for (only applicable if test=true)
+ * @param {boolean} [options.removeOrphans=true] - Whether to remove orphaned containers
  */
-export async function up({ projectDir, build = false, detached = false, coreDev = false }) {
+export async function up({ projectDir, build = false, detach = false, coreDev = false, test = false, testService = '', removeOrphans = true }) {
     
     // core dev
     if (coreDev) {
@@ -40,8 +44,13 @@ export async function up({ projectDir, build = false, detached = false, coreDev
 
     // create nginx.conf dynamically from configPath
     const nginxConfPath = path.join(gnarHiddenDir, "nginx", "nginx.conf");
+    const serviceConfDir = path.join(gnarHiddenDir, "nginx", "service_conf")
+    await fs.mkdir(serviceConfDir, { recursive: true });
+
     const nginxConf = await createDynamicNginxConf({
-        config: parsedConfig.config
+        config: parsedConfig.config,
+        projectDir: projectDir,
+        serviceConfDir: serviceConfDir
     });
     await fs.writeFile(nginxConfPath, nginxConf);
 
@@ -52,7 +61,9 @@ export async function up({ projectDir, build = false, detached = false, coreDev
         secrets: parsedSecrets,
         gnarHiddenDir: gnarHiddenDir,
         projectDir: projectDir,
-        coreDev: coreDev
+        coreDev: coreDev,
+        test: test,
+        testService: testService
     });
     await fs.writeFile(dockerComposePath, yaml.dump(dockerCompose));
 
@@ -63,10 +74,14 @@ export async function up({ projectDir, build = false, detached = false, coreDev
         args.push("--build");
     }
 
-    if (detached) {
+    if (detach) {
         args.push("-d");
     }
 
+    if (removeOrphans) {
+        args.push("--remove-orphans")
+    }
+
     const processRef = spawn(
         "docker-compose",
         args,
@@ -96,11 +111,11 @@ export async function up({ projectDir, build = false, detached = false, coreDev
  */
 export async function down({ projectDir, allContainers = false }) {
     // list all containers
-    const containers = await docker.listContainers();
+    let containers = await docker.listContainers();
 
     // filter containers by image name
     if (!allContainers) {
-        const containers = containers.filter(c => c.Image.includes("ge-dev"));
+        containers = containers.filter(c => c.Image.includes("ge-localdev"));
     }
 
     if (containers.length === 0) {
@@ -122,15 +137,28 @@ export async function down({ projectDir, allContainers = false }) {
             });
         })
     );
+
+    // remove each container
+    await Promise.all(
+        containers.map(c => {
+            const container = docker.getContainer(c.Id);
+            return container.remove({ force: true }).catch(err => {
+                console.error(`Failed to remove ${c.Names[0]}: ${err.message}`);
+            });
+        })
+    );
+
+    console.log('Containers stopped and removed.');
 }
 
 /**
  * Create dynamic nginx.conf file for running application locally
  * 
  * @param {object} config
- * @param {string} outputPath - where to write nginx.conf
+ * @param {string} serviceConfDir
+ * @param {string} projectDir
  */
-export async function createDynamicNginxConf({ config, outputPath }) {
+export async function createDynamicNginxConf({ config, serviceConfDir, projectDir }) {
     // Start with the static parts of nginx.conf
     let nginxConf = `
         events { worker_connections 1024; }
@@ -139,10 +167,29 @@ export async function createDynamicNginxConf({ config, outputPath }) {
             server {
                 listen 80;
                 server_name ${config.namespace};
+                include /etc/nginx/service_conf/*.conf;
+
     `;
 
     // Loop over each service
     for (const service of config.services || []) {
+        // Check if override is present and add conf to service_conf dir
+        const serviceDir = path.join(projectDir, 'services', service.name);
+
+        if (await fs.stat(serviceDir).then(() => true).catch(() => false)) {
+            const overridePath = path.join(serviceDir, 'nginx.conf');
+            if (await fs.stat(overridePath).then(() => true).catch(() => false)) {
+                const overrideConf = await fs.readFile(overridePath, 'utf8');
+
+                // write to service_conf directory
+                const serviceConfPath = path.join(serviceConfDir, `${service.name}.conf`);
+                await fs.writeFile(serviceConfPath, overrideConf);
+
+                continue;
+            }
+        }
+
+        // Otherwise create generic conf block
         const serviceName = service.name;
         const paths = service.listener_rules?.paths || [];
         const containerPort = service.ports && service.ports.length > 0 ? service.ports[0].split(':')[1] : '3000';
@@ -179,12 +226,57 @@ export async function createDynamicNginxConf({ config, outputPath }) {
  * @param {string} gnarHiddenDir
  * @param {string} projectDir
  * @param {boolean} coreDev - Whether to volume mount the core source code
+ * @param {boolean} test - Whether to run tests with ephemeral databases
+ * @param {string} testService - The service to run tests for (only applicable if test=true)
+ * @param {boolean} attachAll - Whether to attach to all containers' stdio (otherwise databases and message queue are detached)
  */
-async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, projectDir, coreDev = false }) {
+async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, projectDir, coreDev = false, test = false, testService, attachAll = false }) {
     let mysqlPortsCounter = 3306;
     let mongoPortsCounter = 27017;
+    let mysqlHostsRequired = [];
+    let mongoHostsRequired = [];
     const services = {};
-    
+
+    // test mode env var adjustments
+    for (const svc of config.services) {
+        if (test) {
+            if (secrets.services?.[svc.name]?.MYSQL_HOST) {
+                secrets.services[svc.name].MYSQL_HOST = 'db-mysql-test';
+            }
+
+            if (secrets.services?.[svc.name]) {
+                secrets.services[svc.name].NODE_ENV = 'test';
+
+                console.log(testService, svc.name);
+                if (testService && svc.name === testService) {
+                    secrets.services[svc.name].RUN_TESTS = 'true';
+                }
+            }
+        }
+    }
+
+    // provision the provisioner service
+    services['provisioner'] = {
+        container_name: `ge-${config.environment}-${config.namespace}-provisioner`,
+        image: `ge-${config.environment}-${config.namespace}-provisioner`,
+        build: {
+            context: directories.provisioner,
+            dockerfile: `./Dockerfile`
+        },
+        environment: {
+            PROVISIONER_SECRETS: JSON.stringify(secrets)
+        },
+        volumes: [
+            `${directories.provisioner}/src:/usr/gnar_engine/app/src`
+        ],
+        restart: 'no',
+        attach: attachAll
+    }
+
+    if (coreDev) {
+        services['provisioner'].volumes.push(`../../../core/:${gnarEngineCliConfig.corePath}`);
+    }
+
     // nginx
     services['nginx'] = {
         image: 'nginx:latest',
@@ -194,9 +286,11 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
             "443:443"
         ],
         volumes: [
-            `${gnarHiddenDir}/nginx/nginx.conf:/etc/nginx/nginx.conf`
+            `${gnarHiddenDir}/nginx/nginx.conf:/etc/nginx/nginx.conf`,
+            `${gnarHiddenDir}/nginx/service_conf:/etc/nginx/service_conf`
         ],
-        restart: 'always'
+        restart: 'always',
+        attach: attachAll
     }
 
     // rabbit
@@ -211,7 +305,8 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
             RABBITMQ_DEFAULT_USER: secrets.global.RABBITMQ_USER || '',
             RABBITMQ_DEFAULT_PASS: secrets.global.RABBITMQ_PASS || ''
         },
-        restart: 'always'
+        restart: 'always',
+        attach: attachAll
     }
 
     // services
@@ -220,7 +315,7 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
         // env variables
         const serviceEnvVars = secrets.services?.[svc.name] || {};
         const localisedServiceEnvVars = {};
-        
+
         for (const [key, value] of Object.entries(serviceEnvVars)) {
             localisedServiceEnvVars[svc.name.toUpperCase() + '_' + key] = value;
         }
@@ -230,6 +325,14 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
             ...(localisedServiceEnvVars || {})
         };
 
+        // test mode adjustments
+        if (test) {
+            if (svc.depends_on && svc.depends_on.includes('db-mysql')) {
+                svc.depends_on = svc.depends_on.filter(d => d !== 'db-mysql');
+                svc.depends_on.push('db-mysql-test');
+            }
+        }
+
         // service block
         services[`${svc.name}-service`] = {
             container_name: `ge-${config.environment}-${config.namespace}-${svc.name}`,
@@ -248,69 +351,67 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
             restart: 'always'
         };
 
-        // add the core source code mount if in coreDeve mode
+        // add the core source code mount if in coreDev mode
         if (coreDev) {
             services[`${svc.name}-service`].volumes.push(`../../../core/:${gnarEngineCliConfig.corePath}`);
         }
 
-        // add a mysql instance if required
+        // check if mysql service required
         if (
             serviceEnvVars.MYSQL_HOST &&
-            serviceEnvVars.MYSQL_DATABASE &&
-            serviceEnvVars.MYSQL_USER &&
-            serviceEnvVars.MYSQL_PASSWORD &&
-            serviceEnvVars.MYSQL_RANDOM_ROOT_PASSWORD
+            secrets.provision?.MYSQL_ROOT_PASSWORD
+        ) {
+            mysqlHostsRequired.push(serviceEnvVars.MYSQL_HOST);
+        }
+
+        // add a mongodb instance if required
+        if (
+            serviceEnvVars.MONGO_HOST &&
+            secrets.provision?.MONGO_ROOT_PASSWORD
         ) {
-            services[`${svc.name}-db`] = {
-                container_name: `ge-${config.environment}-${config.namespace}-${svc.name}-db`,
+            mongoHostsRequired.push(serviceEnvVars.MONGO_HOST);
+        }
+    }
+
+    // add mysql if required
+    if (mysqlHostsRequired.length > 0) {
+        for (const host of mysqlHostsRequired) {
+            if (services[host]) {
+                continue;
+            }
+
+            services[host] = {
+                container_name: `ge-${config.environment}-${config.namespace}-${host}`,
                 image: 'mysql',
                 ports: [
                     `${mysqlPortsCounter}:${mysqlPortsCounter}`
                 ],
                 restart: 'always',
                 environment: {
-                    MYSQL_HOST: serviceEnvVars.MYSQL_HOST,
-                    MYSQL_DATABASE: serviceEnvVars.MYSQL_DATABASE,
-                    MYSQL_USER: serviceEnvVars.MYSQL_USER,
-                    MYSQL_PASSWORD: serviceEnvVars.MYSQL_PASSWORD,
-                    MYSQL_RANDOM_ROOT_PASSWORD: serviceEnvVars.MYSQL_RANDOM_ROOT_PASSWORD,
+                    MYSQL_HOST: host,
+                    MYSQL_ROOT_PASSWORD: secrets.provision.MYSQL_ROOT_PASSWORD
                 },
                 volumes: [
-                    `${gnarHiddenDir}/data/${svc.name}-db-data:/var/lib/mysql`
-                ]
-            }; 
+                    `${gnarHiddenDir}/data/${host}-data:/var/lib/mysql`
+                ],
+                attach: attachAll
+            };
 
-            // increment mysql port for next service as required
             mysqlPortsCounter++;
         }
 
-        // add a mongodb instance if required
-        if (
-            serviceEnvVars.MONGO_HOST &&
-            serviceEnvVars.MONGO_ROOT_PASSWORD &&
-            serviceEnvVars.MONGO_DATABASE &&
-            serviceEnvVars.MONGO_USER &&
-            serviceEnvVars.MONGO_PASSWORD
-        ) {
-            // add mongo init scripts to hidden dir
-            fs.mkdir(path.join(gnarHiddenDir, 'mongo-init-scripts'), { recursive: true });
-            const mongoInitScript = `
-                db = db.getSiblingDB("invoice_db");
-                db.createUser({
-                    user: "${serviceEnvVars.MONGO_USER}",
-                    pwd: "${serviceEnvVars.MONGO_PASSWORD}",
-                    roles: [{ role: "readWrite", db: "${serviceEnvVars.MONGO_DATABASE}" }]
-                });
-
-                print("Created user ${serviceEnvVars.MONGO_USER} with access to database ${serviceEnvVars.MONGO_DATABASE}");
-            `;
-            await fs.writeFile(path.join(gnarHiddenDir, 'mongo-init-scripts', `${svc.name}-init.js`), mongoInitScript);
+        services['provisioner'].depends_on = [...new Set(mysqlHostsRequired)];
+    }
+    
+    // add mongo hosts if required
+    if (mongoHostsRequired.length > 0) {
+        for (const host of mongoHostsRequired) {
+            if (services[host]) {
+                continue;
+            }
 
-            // create mongo service
-            const mongoUrl = `mongodb://${serviceEnvVars.MONGO_USER}:${serviceEnvVars.MONGO_PASSWORD}@${serviceEnvVars.MONGO_HOST}:27017/${serviceEnvVars.MONGO_DATABASE}`;
-            
-            services[`${svc.name}-db`] = {
-                container_name: `ge-${config.environment}-${config.namespace}-${svc.name}-mongo`,
+            services[host] = {
+                container_name: `ge-${config.environment}-${config.namespace}-${host}`,
                 image: 'mongo:latest',
                 ports: [
                     `${mongoPortsCounter}:27017`
@@ -318,15 +419,13 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
                 restart: 'always',
                 environment: {
                     MONGO_INITDB_ROOT_USERNAME: 'root',
-                    MONGO_INITDB_ROOT_PASSWORD: serviceEnvVars.MONGO_ROOT_PASSWORD,
-                    MONGO_INITDB_DATABASE: serviceEnvVars.MONGO_DATABASE,
-                    DB_USER: serviceEnvVars.MONGO_USER,
-                    DB_PASSWORD: serviceEnvVars.MONGO_PASSWORD,
+                    MONGO_INITDB_ROOT_PASSWORD: secrets.provision.MONGO_ROOT_PASSWORD
                 },
                 volumes: [
-                    `${gnarHiddenDir}/data/${svc.name}-mongo-data:/data/db`,
+                    `${gnarHiddenDir}/data/${host}-data:/data/db`,
                     './mongo-init-scripts:/docker-entrypoint-initdb.d'
-                ]
+                ],
+                attach: attachAll
             };
 
             // increment mongo port for next service as required
@@ -346,3 +445,4 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
 async function assertGnarEngineHiddenDir(gnarHiddenDir) {
     await fs.mkdir(gnarHiddenDir, { recursive: true });
 }
+

package/src/helpers/helpers.js

@@ -1,3 +1,6 @@
+import fs from 'fs/promises';
+import path from 'path';
+import yaml from 'js-yaml';
 
 /**
  * CLI helper functions
@@ -59,5 +62,26 @@ export const helpers = {
             result += chars.charAt(Math.floor(Math.random() * chars.length));
         }
         return result;
+    },
+
+    getDbTypeFromSecrets: async (serviceName, projectDir) => {
+        let dbType;
+        const secretsPath = path.join(projectDir, "secrets.localdev.yml");
+        const parsedSecrets = yaml.load(await fs.readFile(secretsPath, "utf8"));
+        const serviceSecrets = parsedSecrets.services[serviceName.toLowerCase()];
+
+        Object.keys(serviceSecrets).forEach(key => {
+            if (key.toLowerCase().includes('host')) {
+                const host = serviceSecrets[key].toLowerCase();
+
+                if (host.includes('mongo')) {
+                    dbType = 'mongodb';
+                } else if (host.includes('mysql')) {
+                    dbType = 'mysql';
+                }
+            }
+        });
+
+        return dbType;
     }
 }

package/src/profiles/command.js

@@ -166,5 +166,46 @@ export function registerProfileCommand(program) {
             });
         });
 
+    // delete profile
+    profile
+        .command('delete <profileName>')
+        .description('Delete an existing profile')
+        .action(async (profileName) => {
+            const config = profiles.getAllProfiles();
+            const activeProfileName = config.activeProfile;
+
+            if (activeProfileName === profileName) {
+                console.error(`Cannot delete active profile "${profileName}". Please set another profile as active first.`);
+                return;
+            }
+
+            if (!config.profiles[profileName]) {
+                console.error(`Profile "${profileName}" not found.`);
+                return;
+            }
+
+            try {
+                // confirm deletion with user
+                const confirmation = await inquirer.prompt([
+                    {
+                        type: 'confirm',
+                        name: 'confirmDelete',
+                        message: `Are you sure you want to delete profile "${profileName}"?`,
+                        default: false,
+                    },
+                ]);
+
+                if (!confirmation.confirmDelete) {
+                    console.log('❌ Deletion cancelled.');
+                    return;
+                }
+
+                profiles.deleteProfile({ profileName });
+                console.log(`✅ Profile "${profileName}" deleted successfully.`);
+            } catch (error) {
+                console.error(error.message);
+            }
+        });
+
     program.addCommand(profile);
 }

package/src/profiles/profiles.client.js

@@ -90,6 +90,29 @@ export const profiles = {
         this.saveProfiles(allProfiles);
     },
 
+    deleteProfile: function ({ profileName }) {
+        if (!profileName) {
+            throw new Error('Invalid profile name');
+        }
+
+        const allProfiles = this.getAllProfiles().profiles || {};
+
+        if (!allProfiles[profileName]) {
+            throw new Error(`Profile "${profileName}" not found`);
+        }
+
+        const activeProfileName = this.getActiveProfile()?.name;
+
+        if (activeProfileName === profileName) {
+            throw new Error(`Cannot delete active profile "${profileName}". Please set another profile as active first.`);
+        }
+
+        // Prompt user to confirm deletion in the console
+        delete allProfiles[profileName];
+
+        this.saveProfiles(allProfiles);
+    },
+
     saveProfiles: function (profilesObj) {
         const dir = path.dirname(this.configPath);
         if (!fs.existsSync(dir)) {

package/src/provisioner/Dockerfile

@@ -0,0 +1,27 @@
+# Dockerfile for the ephemeral provisioning service
+#
+# This service is responsible for asserting the databases 
+# & database users, or other single runtime provisioning tasks.
+
+FROM node:20-alpine
+
+# Set the working directory
+WORKDIR /usr/gnar_engine/app
+
+# Define a global env var
+ENV GLOBAL_SERVICE_BASE_DIR=/usr/gnar_engine/app/src/
+
+# Copy package.json and package-lock.json
+COPY ./package*.json ./
+
+# Copy source
+COPY ./src ./src
+
+# Install nodemon
+RUN npm install -g nodemon
+
+# Install app dependencies
+RUN npm install
+
+# Start the application
+CMD ["nodemon", "--watch", "./gnar_engine", "./gnar_engine/app.js"]

package/src/provisioner/package.json

@@ -0,0 +1,19 @@
+{
+    "name": "gnar_engine_user",
+    "version": "1.0.0",
+    "description": "Gnar Engine - User Service",
+    "type": "module",
+    "scripts": {
+        "start": "node ./src/app.js",
+        "start:dev": "nodemon --watch ./src ./src/app.js",
+        "test": "jest --watchAll --verbose"
+    },
+    "author": "Gnar Software Ltd.",
+    "license": "ISC",
+    "dependencies": {
+        "@gnar-engine/core": "^1.0.1",
+        "dotenv": "^16.4.7",
+        "mongodb": "^5.9.2",
+        "mysql2": "^3.12.0"
+    }
+}