@gnar-engine/cli 1.0.6 → 1.0.7

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gnar-engine/cli",
-    "version": "1.0.6",
+    "version": "1.0.7",
     "description": "Gnar Engine Development Framework CLI: Project bootstrap, scaffolder & control plane.",
     "type": "module",
     "bin": {

package/src/config.js

@@ -1,11 +1,15 @@
 import path from 'path';
+import { fileURLToPath } from 'url';
 
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
 export const gnarEngineCliConfig = {
 
     /**
      * The path the Gnar Engine service core should be found in the service containers
      */
-    corePath: '/usr/gnar_engine/app/node_modules/@gnar-engine/core'
+    corePath: '/usr/gnar_engine/app/node_modules/@gnar-engine/core',
+    coreDevPath: path.join(__dirname, '../../core'),
 
 }
 

package/src/dev/commands.js

@@ -12,9 +12,11 @@ export function registerDevCommands(program) {
 		.description('🛠️  Up Development Containers')
 		.option('-b, --build', 'Build without cache')
         .option('-d, --detach', 'Run containers in background')
+        .option('-a, --attach-all', 'Attach all services including database and message queues for debugging')
         .option('-t --test', 'Run the tests with ephemeral databases')
         .option('--test-service <service>', 'Run the tests for the specified service with ephemeral databases (e.g. --test-service user)')
         .addOption(new Option('--core-dev').hideHelp())
+        .addOption(new Option('--bootstrap-dev').hideHelp())
         .action(async (options) => {
 			let response = {};
 
@@ -39,8 +41,10 @@ export function registerDevCommands(program) {
                     build: options.build || false,
                     detach: options.detach || false,
                     coreDev: options.coreDev || false,
+                    bootstrapDev: options.bootstrapDev || false,
                     test: options.test || false,
-                    testService: options.testService || ''
+                    testService: options.testService || '',
+                    attachAll: options.attachAll || false
                  });
 			} catch (err) {
 				console.error("❌ Error running containers:", err.message);

package/src/dev/dev.service.js

@@ -3,10 +3,13 @@ import Docker from "dockerode";
 import process from "process";
 import fs from "fs/promises";
 import path from "path";
+import { fileURLToPath } from 'url';
 import yaml from "js-yaml";
 import { gnarEngineCliConfig } from "../config.js";
+import { buildImage, createContainer, createBridgeNetwork } from "../services/docker.js";
 import { directories } from "../config.js";
 
+
 const docker = new Docker();
 
 /**
@@ -19,14 +22,16 @@ const docker = new Docker();
  * @param {boolean} [options.build=false] - Whether to re-build images
  * @param {boolean} [options.detach=false] - Whether to run containers in background
  * @param {boolean} [options.coreDev=false] - Whether to run in core development mode (requires access to core source)
+ * @param {boolean} [options.bootstrapDev=false] - Whether to set the cli/src/bootstrap directory as the project directory
  * @param {boolean} [options.test=false] - Whether to run tests with ephemeral databases
  * @param {string} [options.testService=''] - The service to run tests for (only applicable if test=true)
  * @param {boolean} [options.removeOrphans=true] - Whether to remove orphaned containers
+ * @param {boolean} [options.attachAll=false] - Attach all services including database and message queues for debugging
  */
-export async function up({ projectDir, build = false, detach = false, coreDev = false, test = false, testService = '', removeOrphans = true }) {
+export async function up({ projectDir, build = false, detach = false, coreDev = false, bootstrapDev = false, test = false, testService = '', removeOrphans = true, attachAll = false}) {
     
-    // core dev
-    if (coreDev) {
+    // bootstrap dev
+    if (bootstrapDev) {
         const fileDir = path.dirname(new URL(import.meta.url).pathname);
         projectDir = path.resolve(fileDir, "../../bootstrap/");
     }
@@ -56,50 +61,51 @@ export async function up({ projectDir, build = false, detach = false, coreDev =
 
     // create docker-compose.yml dynamically from parsed config and secrets
     const dockerComposePath = path.join(gnarHiddenDir, "docker-compose.dev.yml");
-    const dockerCompose = await createDynamicDockerCompose({
+    const dockerCompose = await buildAndUpContainers({
         config: parsedConfig.config,
         secrets: parsedSecrets,
         gnarHiddenDir: gnarHiddenDir,
         projectDir: projectDir,
         coreDev: coreDev,
+        bootstrapDev: bootstrapDev,
         test: test,
-        testService: testService
+        testService: testService,
+        attachAll: attachAll
     });
-    await fs.writeFile(dockerComposePath, yaml.dump(dockerCompose));
-
-    // up docker-compose
-    const args = ["-f", dockerComposePath, "up"];
 
-    if (build) {
-        args.push("--build");
-    }
-
-    if (detach) {
-        args.push("-d");
-    }
-
-    if (removeOrphans) {
-        args.push("--remove-orphans")
-    }
-
-    const processRef = spawn(
-        "docker-compose",
-        args,
-        {
-            cwd: projectDir,
-            stdio: "inherit",
-            shell: "/bin/sh"
-        }
-    );
-
-    // handle exit
-    const exitCode = await new Promise((resolve) => {
-        processRef.on("close", resolve);
-    });
-
-    if (exitCode !== 0) {
-        throw new Error(`docker-compose up exited with code ${exitCode}`);
-    }
+    // // up docker-compose
+    // const args = ["-f", dockerComposePath, "up"];
+    //
+    // if (build) {
+    //     args.push("--build");
+    // }
+    //
+    // if (detach) {
+    //     args.push("-d");
+    // }
+    //
+    // if (removeOrphans) {
+    //     args.push("--remove-orphans")
+    // }
+
+    // const processRef = spawn(
+    //     "docker-compose",
+    //     args,
+    //     {
+    //         cwd: projectDir,
+    //         stdio: "inherit",
+    //         shell: "/bin/sh"
+    //     }
+    // );
+    //
+    // // handle exit
+    // const exitCode = await new Promise((resolve) => {
+    //     processRef.on("close", resolve);
+    // });
+    //
+    // if (exitCode !== 0) {
+    //     throw new Error(`docker-compose up exited with code ${exitCode}`);
+    // }
 }
 
 /**
@@ -226,17 +232,23 @@ export async function createDynamicNginxConf({ config, serviceConfDir, projectDi
  * @param {string} gnarHiddenDir
  * @param {string} projectDir
  * @param {boolean} coreDev - Whether to volume mount the core source code
+ * @param {boolean} bootstrapDev - Whether to set the cli/src/bootstrap directory as the project directory
+ * @param {boolean} build - Whether to re-build images
  * @param {boolean} test - Whether to run tests with ephemeral databases
  * @param {string} testService - The service to run tests for (only applicable if test=true)
  * @param {boolean} attachAll - Whether to attach to all containers' stdio (otherwise databases and message queue are detached)
  */
-async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, projectDir, coreDev = false, test = false, testService, attachAll = false }) {
+async function buildAndUpContainers({ config, secrets, gnarHiddenDir, projectDir, coreDev = false, bootstrapDev = false, build = false, test = false, testService, attachAll = false }) {
+    
     let mysqlPortsCounter = 3306;
     let mongoPortsCounter = 27017;
     let mysqlHostsRequired = [];
     let mongoHostsRequired = [];
     const services = {};
 
+    console.log('========   g n a r   e n g i n e   ========');
+    console.log('⛏️  Starting development environment...');
+    
     // test mode env var adjustments
     for (const svc of config.services) {
         if (test) {
@@ -247,7 +259,6 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
             if (secrets.services?.[svc.name]) {
                 secrets.services[svc.name].NODE_ENV = 'test';
 
-                console.log(testService, svc.name);
                 if (testService && svc.name === testService) {
                     secrets.services[svc.name].RUN_TESTS = 'true';
                 }
@@ -255,63 +266,92 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
         }
     }
 
+    // create bridge network
+    const networkName = `ge-${config.environment}-${config.namespace}`;
+    createBridgeNetwork({
+        name: networkName
+    })
+
     // provision the provisioner service
-    services['provisioner'] = {
-        container_name: `ge-${config.environment}-${config.namespace}-provisioner`,
-        image: `ge-${config.environment}-${config.namespace}-provisioner`,
-        build: {
+    const provisionerTag = `ge-${config.environment}-${config.namespace}-provisioner`;
+    
+    if (build) {
+        await buildImage({
             context: directories.provisioner,
-            dockerfile: `./Dockerfile`
-        },
-        environment: {
-            PROVISIONER_SECRETS: JSON.stringify(secrets)
-        },
-        volumes: [
-            `${directories.provisioner}/src:/usr/gnar_engine/app/src`
-        ],
-        restart: 'no',
-        attach: attachAll
+            dockerfile: 'Dockerfile',
+            imageTag: provisionerTag
+        });
     }
 
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = path.dirname(__filename);
+
+    const provisionerBinds = [
+        `${path.resolve(__dirname, '../provisioner', 'src')}:/usr/gnar_engine/app/src`
+    ];
+
     if (coreDev) {
-        services['provisioner'].volumes.push(`../../../core/:${gnarEngineCliConfig.corePath}`);
+        provisionerBinds.push(`${gnarEngineCliConfig.coreDevPath}:${gnarEngineCliConfig.corePath}`);
     }
 
-    // nginx
-    services['nginx'] = {
+    const provisioner = await createContainer({
+        name: provisionerTag,
+        image: provisionerTag,
+        env: {
+            PROVISIONER_SECRETS: JSON.stringify(secrets)
+        },
+        ports: {},
+        binds: provisionerBinds,
+        restart: 'no',
+        attach: attachAll,
+        network: networkName
+    });
+    services[provisionerTag] = provisioner;
+
+    // Nginx
+    const nginxName = `ge-${config.environment}-${config.namespace}-nginx`;
+    services[nginxName] = await createContainer({
+        name: nginxName,
         image: 'nginx:latest',
-        container_name: `ge-${config.environment}-${config.namespace}-nginx`,
-        ports: [
-            "80:80",
-            "443:443"
-        ],
-        volumes: [
+        ports: { 80: 80, 443: 443 },
+        binds: [
             `${gnarHiddenDir}/nginx/nginx.conf:/etc/nginx/nginx.conf`,
             `${gnarHiddenDir}/nginx/service_conf:/etc/nginx/service_conf`
         ],
-        restart: 'always',
-        attach: attachAll
-    }
+        attach: attachAll,
+        network: networkName
+    });
 
-    // rabbit
-    services['rabbitmq'] = {
+    // Rabbit MQ 
+    const rabbitMqName = `ge-${config.environment}-${config.namespace}-rabbitmq`;
+    services[rabbitMqName] = await createContainer({
+        name: rabbitMqName,
         image: 'rabbitmq:management',
-        container_name: `ge-${config.environment}-${config.namespace}-rabbitmq`,
-        ports: [
-            "5672:5672",
-            "15672:15672"
-        ],
-        environment: {
+        env: {
             RABBITMQ_DEFAULT_USER: secrets.global.RABBITMQ_USER || '',
             RABBITMQ_DEFAULT_PASS: secrets.global.RABBITMQ_PASS || ''
         },
-        restart: 'always',
-        attach: attachAll
-    }
+        ports: { 5672: 5672, 15672: 15672 },
+        binds: [],
+        attach: attachAll,
+        network: networkName,
+        aliases: ['rabbitmq']
+    });
 
     // services
     for (const svc of config.services) {
 
+        // build service image
+        const svcTag = `ge-${config.environment}-${config.namespace}-${svc.name}`;
+
+        if (build) {
+            await buildImage({
+                context: path.resolve(projectDir, 'services', svc.name),
+                dockerfile: 'Dockerfile',
+                imageTag: svcTag
+            });
+        }
+
         // env variables
         const serviceEnvVars = secrets.services?.[svc.name] || {};
         const localisedServiceEnvVars = {};
@@ -333,29 +373,35 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
             }
         }
 
-        // service block
-        services[`${svc.name}-service`] = {
-            container_name: `ge-${config.environment}-${config.namespace}-${svc.name}`,
-            image: `ge-${config.environment}-${config.namespace}-${svc.name}`,
-            build: {
-                context: projectDir,
-                dockerfile: `./services/${svc.name}/Dockerfile`
-            },
-            command: svc.command || [],
-            environment: env,
-            ports: svc.ports || [],
-            depends_on: svc.depends_on || [],
-            volumes: [
-                `${projectDir}/services/${svc.name}/src:/usr/gnar_engine/app/src`
-            ],
-            restart: 'always'
-        };
-
         // add the core source code mount if in coreDev mode
+        const serviceVolumes = [
+            `${path.resolve(projectDir, 'services', svc.name, 'src')}:/usr/gnar_engine/app/src`
+        ];
+
         if (coreDev) {
-            services[`${svc.name}-service`].volumes.push(`../../../core/:${gnarEngineCliConfig.corePath}`);
+            serviceVolumes.push(`${gnarEngineCliConfig.coreDevPath}:${gnarEngineCliConfig.corePath}`);
+        }
+
+        // split from "port:port" to { port: port }
+        const ports = {};
+        for (const portMapping of svc.ports || []) {
+            const [hostPort, containerPort] = portMapping.split(':').map(p => parseInt(p, 10));
+            ports[containerPort] = hostPort;
         }
 
+        services[svcTag] = await createContainer({
+            name: svcTag,
+            image: svcTag,
+            command: svc.command || [],
+            env: env,
+            ports: ports,
+            binds: serviceVolumes,
+            restart: 'always',
+            attach: true,
+            network: networkName,
+            aliases: [`${svc.name}-service`]
+        });
+
         // check if mysql service required
         if (
             serviceEnvVars.MYSQL_HOST &&
@@ -380,29 +426,30 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
                 continue;
             }
 
-            services[host] = {
-                container_name: `ge-${config.environment}-${config.namespace}-${host}`,
+            const mysqlContainerName = `ge-${config.environment}-${config.namespace}-${host}`;
+            services[mysqlContainerName] = await createContainer({
+                name: `ge-${config.environment}-${config.namespace}-${host}`,
                 image: 'mysql',
-                ports: [
-                    `${mysqlPortsCounter}:${mysqlPortsCounter}`
-                ],
-                restart: 'always',
-                environment: {
+                env: {
                     MYSQL_HOST: host,
                     MYSQL_ROOT_PASSWORD: secrets.provision.MYSQL_ROOT_PASSWORD
                 },
-                volumes: [
+                ports: {
+                    [mysqlPortsCounter]: mysqlPortsCounter
+                },
+                binds: [
                     `${gnarHiddenDir}/data/${host}-data:/var/lib/mysql`
                 ],
-                attach: attachAll
-            };
+                restart: 'always',
+                attach: attachAll,
+                network: networkName,
+                aliases: [host]
+            });
 
             mysqlPortsCounter++;
         }
-
-        services['provisioner'].depends_on = [...new Set(mysqlHostsRequired)];
     }
-    
+
     // add mongo hosts if required
     if (mongoHostsRequired.length > 0) {
         for (const host of mongoHostsRequired) {
@@ -410,33 +457,37 @@ async function createDynamicDockerCompose({ config, secrets, gnarHiddenDir, proj
                 continue;
             }
 
-            services[host] = {
-                container_name: `ge-${config.environment}-${config.namespace}-${host}`,
+            const mongoContainerName = `ge-${config.environment}-${config.namespace}-${host}`;
+            services[mongoContainerName] = await createContainer({
+                name: `ge-${config.environment}-${config.namespace}-${host}`,
                 image: 'mongo:latest',
-                ports: [
-                    `${mongoPortsCounter}:27017`
-                ],
-                restart: 'always',
-                environment: {
+                env: {
                     MONGO_INITDB_ROOT_USERNAME: 'root',
                     MONGO_INITDB_ROOT_PASSWORD: secrets.provision.MONGO_ROOT_PASSWORD
                 },
-                volumes: [
+                ports: {
+                    [mongoPortsCounter]: 27017
+                },
+                binds: [
                     `${gnarHiddenDir}/data/${host}-data:/data/db`,
-                    './mongo-init-scripts:/docker-entrypoint-initdb.d'
+                    //'./mongo-init-scripts:/docker-entrypoint-initdb.d'
                 ],
-                attach: attachAll
-            };
+                restart: 'always',
+                attach: attachAll,
+                network: networkName,
+                aliases: [host]
+            });
 
             // increment mongo port for next service as required
             mongoPortsCounter++;
         }
     }
 
-    return {
-        version: "3.9",
-        services
-    }
+    // start the containers
+    Object.keys(services).forEach(async (key) => {
+        const container = services[key];
+        container.start();
+    });
 }
 
 /**

package/src/services/docker.js

@@ -0,0 +1,173 @@
+import Docker from 'dockerode';
+import { Writable } from 'stream';
+import path from 'path';
+import fs from 'fs';
+
+const docker = new Docker();
+
+/**
+ * Build Docker image
+ *
+ * @param {Object} options
+ * @param {string} options.context - The build context directory
+ * @param {string} options.dockerfile - The Dockerfile path relative to the context
+ * @param {string} options.imageTag - The tag to assign to the built image
+ */
+export async function buildImage({ context, dockerfile, imageTag }) {
+
+    console.log('Building image...', imageTag);
+
+    const tarStream = await docker.buildImage(
+        {
+            context: context,
+            src: fs.readdirSync(context)
+        },
+        {
+            t: imageTag,
+            dockerfile
+        }
+    );
+
+    await new Promise((resolve, reject) => {
+        docker.modem.followProgress(tarStream, (err) => (err ? reject(err) : resolve()));
+    });
+
+    console.log('Built image:', imageTag);
+}
+
+/**
+ * Create docker container
+ *
+ * @param {Object} options
+ * @param {string} options.name - Container name
+ * @param {string} options.image - Docker image to use
+ * @param {Array} [options.command] - Command to run in the container
+ * @param {Object} [options.env] - Environment variables
+ * @param {Object} [options.ports] - Port mappings (containerPort: hostPort)
+ * @param {Array} [options.binds] - Volume bindings
+ * @param {string} [options.restart] - Restart policy
+ * @param {boolean} [options.attach] - Whether to attach to container output
+ * @param {string} [options.network] - Network name
+ * @returns {Promise<Docker.Container>} - The started container
+ */
+export async function createContainer({ name, image, command = [], env = {}, ports = {}, binds = [], restart = 'always', attach = true, network, aliases = [] }) {
+
+    // remove container first
+    try {
+        const existingContainer = docker.getContainer(name);
+        await existingContainer.inspect();
+        await existingContainer.remove({ force: true });
+    } catch (err) {
+        // Container does not exist, ignore
+    }
+
+    // create container
+    const containerConfig = {
+        name,
+        Image: image,
+        Cmd: command,
+        Env: Object.entries(env).map(([k, v]) => `${k}=${v}`),
+        HostConfig: {
+            RestartPolicy: { Name: restart },
+            Binds: binds,
+            PortBindings: Object.fromEntries(
+                Object.entries(ports).map(([cPort, hPort]) => [
+                    `${cPort}/tcp`,
+                    [{ HostPort: String(hPort) }]
+                ])
+            ),
+        },
+        ExposedPorts: Object.fromEntries(
+            Object.keys(ports).map(p => [`${p}/tcp`, {}])
+        )
+    };
+
+    const container = await docker.createContainer(containerConfig);
+
+    // Attach logs before starting the container
+    if (attach) {
+        const stream = await container.attach({ stream: true, stdout: true, stderr: true, logs: true });
+        const stdoutStream = createPrefixStream(name, process.stdout);
+        const stderrStream = createPrefixStream(name, process.stderr);
+
+        container.modem.demuxStream(stream, stdoutStream, stderrStream);
+    }
+
+    await docker.getNetwork(network).connect({
+        Container: container.id,
+        EndpointConfig: {
+            Aliases: aliases
+        }
+    });
+
+    return container;
+}
+
+/**
+ * Create network
+ *
+ * @param {String} name - Network name
+ */
+export async function createBridgeNetwork({ name }) {
+    try {
+        await docker.createNetwork({
+            Name: name,
+            Driver: 'bridge',
+            CheckDuplicate: true
+        });
+    } catch (err) {
+        if (err.statusCode === 409) {
+            // network already exists, ignore
+            return;
+        }
+        console.error(err);
+    }
+}
+
+/**
+ * Tansform stream
+ *
+ * @param {String} name - Container name
+ * @returns {Transform} - Transform stream
+ */
+function createPrefixStream(name, targetStream) {
+    const color = colorForName(name);
+    const labelWidth = 38;
+    name = '[' + name + ']';
+    const paddedName = name.padEnd(labelWidth, ' ');
+
+    return new Writable({
+        write(chunk, encoding, callback) {
+            const lines = chunk.toString().split(/\r?\n/);
+            for (const line of lines) {
+                if (line.trim()) {
+                    targetStream.write(`${color}${paddedName}${RESET} ${line}\n`);
+                }
+            }
+            callback();
+        }
+    });
+}
+
+/**
+ * Colours for container name
+ */
+const COLORS = [
+    '\x1b[31m', // red
+    '\x1b[32m', // green
+    '\x1b[33m', // yellow
+    '\x1b[34m', // blue
+    '\x1b[35m', // magenta
+    '\x1b[36m', // cyan
+    '\x1b[37m', // white
+];
+
+const RESET = '\x1b[0m';
+
+function colorForName(name) {
+    let hash = 0;
+    for (let i = 0; i < name.length; i++) {
+        hash = name.charCodeAt(i) + ((hash << 5) - hash);
+    }
+    return COLORS[Math.abs(hash) % COLORS.length];
+}