@enspirit/emb 0.0.5 → 0.0.7

package/dist/src/monorepo/config.js

@@ -1,21 +1,33 @@
 import deepMerge from '@fastify/deepmerge';
 import { deepMergeArray } from '../utils/index.js';
 export class MonorepoConfig {
-    components;
+    // the flavor we come from
+    currentFlavor;
+    //
     defaults;
     env;
     flavors;
     plugins;
     project;
     vars;
+    tasks;
+    _components;
     constructor(config) {
-        this.components = config.components;
+        this._components = config.components.reduce((map, cmp) => {
+            map.set(cmp.name, cmp);
+            return map;
+        }, new Map());
         this.defaults = config.defaults || {};
         this.project = config.project;
         this.vars = config.vars || {};
         this.flavors = config.flavors || [];
         this.env = config.env || {};
         this.plugins = config.plugins || [];
+        this.tasks = config.tasks || [];
+        this.currentFlavor = config.currentFlavor || 'default';
+    }
+    get components() {
+        return [...this._components.values()];
     }
     component(name) {
         const config = this.components.find((c) => c.name === name);
@@ -33,6 +45,7 @@ export class MonorepoConfig {
     }
     toJSON() {
         return {
+            currentFlavor: this.currentFlavor,
             components: this.components,
             defaults: this.defaults,
             env: this.env,
@@ -40,11 +53,13 @@ export class MonorepoConfig {
             plugins: this.plugins,
             project: this.project,
             vars: this.vars,
+            tasks: this.tasks,
         };
     }
     with(overrides) {
         const newConfig = {
             ...this.toJSON(),
+            ...overrides,
             components: deepMerge({
                 mergeArray() {
                     // Merge components by identifying them by name
@@ -53,7 +68,7 @@ export class MonorepoConfig {
                     });
                 },
             })(this.components, overrides?.components || []),
-            defaults: deepMerge()(this.defaults, overrides.defaults),
+            defaults: deepMerge()(this.defaults || {}, overrides.defaults || {}),
             env: deepMerge()(this.env, overrides?.env || {}),
             project: deepMerge()(this.project, overrides?.project || {}),
             vars: deepMerge()(this.vars, overrides?.vars || {}),
@@ -61,6 +76,9 @@ export class MonorepoConfig {
         return new MonorepoConfig(newConfig);
     }
     withFlavor(name) {
-        return this.with(this.flavor(name));
+        return this.with({
+            ...this.flavor(name),
+            currentFlavor: name,
+        });
     }
 }

package/dist/src/monorepo/index.d.ts

@@ -1,7 +1,9 @@
 export * from './component.js';
 export * from './config.js';
 export * from './monorepo.js';
+export * from './operations/index.js';
 export * from './plugins/index.js';
 export * from './store/index.js';
+export * from './taskManagerFactory.js';
 export * from './types.js';
 export * from './utils/index.js';

package/dist/src/monorepo/index.js

@@ -1,7 +1,9 @@
 export * from './component.js';
 export * from './config.js';
 export * from './monorepo.js';
+export * from './operations/index.js';
 export * from './plugins/index.js';
 export * from './store/index.js';
+export * from './taskManagerFactory.js';
 export * from './types.js';
 export * from './utils/index.js';

package/dist/src/monorepo/monorepo.d.ts

@@ -1,12 +1,11 @@
 import { IMonorepoConfig } from '../config/index.js';
-import { IOperation } from '../operations/types.js';
+import { IOperation } from '../operations/index.js';
 import { TemplateExpander } from '../utils/index.js';
 import { Component } from './component.js';
 import { EMBStore } from './store/index.js';
 import { TaskInfo } from './types.js';
 export declare class Monorepo {
     private _config;
-    private _env;
     private _store;
     private initialized;
     constructor(config: IMonorepoConfig);
@@ -16,12 +15,15 @@ export declare class Monorepo {
     get flavors(): string[];
     get name(): string;
     get rootDir(): string;
+    get currentFlavor(): string;
     get store(): EMBStore;
     get tasks(): TaskInfo[];
     get vars(): Record<string, unknown>;
     component(name: string): Component;
+    task(nameOrId: string): TaskInfo;
     expand(str: string, expander?: TemplateExpander): Promise<string>;
     expand<R extends Record<string, unknown>>(record: R, expander?: TemplateExpander): Promise<R>;
+    private installStore;
     init(): Promise<Monorepo>;
     join(...paths: string[]): string;
     run<I, O>(operation: IOperation<I, O>, args: I): Promise<O>;

package/dist/src/monorepo/monorepo.js

@@ -6,12 +6,10 @@ import { getPlugin } from './plugins/index.js';
 import { EMBStore } from './store/index.js';
 export class Monorepo {
     _config;
-    _env;
     _store;
     initialized = false;
     constructor(config) {
         this._config = new MonorepoConfig(config);
-        this._env = config.env || {};
     }
     // TODO: cache/improve
     get components() {
@@ -32,13 +30,22 @@ export class Monorepo {
     get rootDir() {
         return this._config.project.rootDir;
     }
+    get currentFlavor() {
+        return this._config.currentFlavor;
+    }
     get store() {
         return this._store;
     }
     get tasks() {
+        const globalTasks = (this._config.tasks || [])?.map((t) => {
+            return {
+                ...t,
+                id: `global:${t.name}`,
+            };
+        });
         return this.components.reduce((tasks, cmp) => {
             return [...tasks, ...cmp.tasks];
-        }, []);
+        }, globalTasks);
     }
     get vars() {
         return this._config.vars;
@@ -46,6 +53,20 @@ export class Monorepo {
     component(name) {
         return new Component(this._config.component(name), this);
     }
+    task(nameOrId) {
+        const byId = this.tasks.find((t) => t.id === nameOrId);
+        if (byId) {
+            return byId;
+        }
+        const found = this.tasks.filter((t) => t.name === nameOrId);
+        if (found.length > 1) {
+            throw new Error(`Task name ambigous, found multiple matches: ${nameOrId}`);
+        }
+        if (found.length === 0) {
+            throw new Error(`Task not found: ${nameOrId}`);
+        }
+        return found[0];
+    }
     async expand(strOrRecord, expander = new TemplateExpander()) {
         const options = {
             default: 'vars',
@@ -59,13 +80,16 @@ export class Monorepo {
         }
         return expander.expandRecord(strOrRecord, options);
     }
+    async installStore(store) {
+        this._store = store || new EMBStore(this);
+        await this._store.init();
+    }
     // Initialize
     async init() {
         if (this.initialized) {
             throw new Error('Monorepo already initialized');
         }
-        this._store = new EMBStore(this);
-        await this._store.init();
+        await this.installStore();
         const plugins = this._config.plugins.map((p) => {
             const PluginClass = getPlugin(p.name);
             return new PluginClass(p.config, this);
@@ -100,7 +124,7 @@ export class Monorepo {
     }
     async withFlavor(name) {
         const repo = new Monorepo(this._config.withFlavor(name));
-        await repo.init();
+        await repo.installStore();
         return repo;
     }
 }

package/dist/src/monorepo/operations/components/BuildComponentsOperation.d.ts

@@ -1,11 +1,20 @@
 import * as z from 'zod';
+import { DockerComponentBuild } from '../../../docker/index.js';
 import { AbstractOperation } from '../../../operations/index.js';
+export type BuildComponentMeta = {
+    dryRun?: boolean;
+    cacheHit?: boolean;
+    build: DockerComponentBuild;
+    preBuildMeta?: string;
+    sentinelFile: string;
+};
 declare const schema: z.ZodObject<{
     components: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    dryRun: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
-export declare class BuildComponentsOperation extends AbstractOperation<typeof schema, Array<unknown>> {
+export declare class BuildComponentsOperation extends AbstractOperation<typeof schema, Record<string, BuildComponentMeta>> {
     constructor();
-    protected _run(input: z.input<typeof schema>): Promise<Array<unknown>>;
+    protected _run(input: z.input<typeof schema>): Promise<Record<string, BuildComponentMeta>>;
     private buildComponent;
 }
 export {};

package/dist/src/monorepo/operations/components/BuildComponentsOperation.js

@@ -1,79 +1,58 @@
-import { Manager } from '@listr2/manager';
-import { createColors } from 'colorette';
-import { delay, ListrDefaultRendererLogLevels, PRESET_TIMER, } from 'listr2';
+import { getContext } from '../../../index.js';
 import * as z from 'zod';
 import { BuildImageOperation, getSentinelFile, } from '../../../docker/index.js';
-import { findBuildOrder } from '../../utils/findBuildOrder.js';
+import { EMBCollection, findRunOrder, taskManagerFactory, } from '../../index.js';
 import { AbstractOperation } from '../../../operations/index.js';
-import { FilePrerequisitePlugin } from '../../../prerequisites/FilePrerequisitePlugin.js';
-import { PrerequisiteType } from '../../../prerequisites/types.js';
+import { FilePrerequisitePlugin, PrerequisiteType } from '../../../prerequisites/index.js';
 const schema = z.object({
     components: z
         .array(z.string())
         .describe('The list of components to build')
         .optional(),
+    dryRun: z
+        .boolean()
+        .optional()
+        .describe('Do not build but return the config that would be used to build the images'),
 });
 export class BuildComponentsOperation extends AbstractOperation {
     constructor() {
         super(schema);
     }
     async _run(input) {
-        const ordered = findBuildOrder(this.context.monorepo.components, input.components);
+        const { monorepo } = getContext();
+        const manager = taskManagerFactory();
+        const selection = (input.components || []).map((t) => monorepo.component(t));
+        const collection = new EMBCollection(this.context.monorepo.components, {
+            idField: 'name',
+            depField: 'dependencies',
+            forbidIdNameCollision: true,
+        });
+        const ordered = findRunOrder(selection.map((s) => s.name), collection);
         const tasks = await Promise.all(ordered.map((cmp) => {
             return {
                 task: async (context, task) => {
-                    return this.buildComponent(cmp, task);
+                    return this.buildComponent(cmp, task, context, input.dryRun);
                 },
                 title: `Building ${cmp.name}`,
             };
         }));
-        const manager = new Manager({
-            collectErrors: 'minimal',
-            concurrent: false,
-            exitOnError: true,
-            rendererOptions: {
-                collapseErrors: false,
-                collapseSubtasks: false,
-                color: {
-                    // @ts-expect-error not sure why
-                    [ListrDefaultRendererLogLevels.SKIPPED_WITH_COLLAPSE]: createColors().green,
-                },
-                icon: {
-                    [ListrDefaultRendererLogLevels.SKIPPED_WITH_COLLAPSE]: '♺',
-                },
-                timer: {
-                    ...PRESET_TIMER,
-                },
-            },
+        const list = manager.newListr([...tasks], {
+            rendererOptions: { persistentOutput: true },
+            ctx: {},
         });
-        manager.add([
-            {
-                async task(_context, task) {
-                    return task.newListr([...tasks], {
-                        rendererOptions: {
-                            collapseSubtasks: false,
-                        },
-                    });
-                },
-                title: 'Building components',
-            },
-        ]);
-        await manager.runAll();
-        return ordered;
+        const results = await list.run();
+        return results;
     }
-    async buildComponent(cmp, parentTask) {
-        return parentTask.newListr([
+    async buildComponent(cmp, parentTask, parentContext, dryRun = false) {
+        const prereqPlugin = new FilePrerequisitePlugin();
+        const list = parentTask.newListr([
             // Collect all the prerequisites and other build infos
             // (This is when variables are expanded etc)
             {
                 async task(ctx) {
-                    // Reset the context to defaults (as apparently the context is shared amongst branches??)
-                    // TODO understand and fix
-                    ctx.skip = false;
-                    //
-                    ctx.parentTask = parentTask;
+                    // Install the context for this specific component build chain
+                    ctx.cacheHit = false;
                     ctx.sentinelFile = getSentinelFile(cmp);
-                    ctx.plugin = new FilePrerequisitePlugin();
                     ctx.build = await cmp.toDockerBuild();
                 },
                 title: 'Prepare build context',
@@ -81,7 +60,7 @@ export class BuildComponentsOperation extends AbstractOperation {
             // Check for sentinal information to see if the build can be skipped
             {
                 task: async (ctx) => {
-                    ctx.preBuildMeta = await ctx.plugin.meta(cmp, ctx.build.prerequisites, 'pre');
+                    ctx.preBuildMeta = await prereqPlugin.meta(cmp, ctx.build.prerequisites, 'pre');
                     let lastValue;
                     try {
                         lastValue = (await this.context.monorepo.store.readFile(ctx.sentinelFile)).toString();
@@ -90,10 +69,10 @@ export class BuildComponentsOperation extends AbstractOperation {
                         lastValue = undefined;
                     }
                     if (lastValue) {
-                        const diff = await ctx.plugin.diff(cmp, ctx.build.prerequisites, lastValue, ctx.preBuildMeta);
+                        const diff = await prereqPlugin.diff(cmp, ctx.build.prerequisites, lastValue, ctx.preBuildMeta);
                         if (!diff) {
-                            ctx.skip = true;
-                            ctx.parentTask.skip(`${ctx.parentTask.title} (cache hit)`);
+                            ctx.cacheHit = true;
+                            // parentTask.skip(`${parentTask.title} (cache hit)`);
                         }
                     }
                 },
@@ -101,10 +80,9 @@ export class BuildComponentsOperation extends AbstractOperation {
             },
             {
                 task: async (ctx, task) => {
-                    if (ctx.skip) {
+                    if (ctx.cacheHit || ctx.dryRun) {
                         return task.skip();
                     }
-                    await delay(500);
                     const title = `Building image ${ctx.build.name}:${ctx.build.tag}`;
                     task.title = title;
                     const op = new BuildImageOperation((progress) => {
@@ -127,18 +105,26 @@ export class BuildComponentsOperation extends AbstractOperation {
             // Update sentinel file
             {
                 task: async (ctx, task) => {
-                    if (ctx.skip) {
+                    if (ctx.cacheHit || ctx.dryRun) {
                         return task.skip();
                     }
-                    const sentinelValue = await ctx.plugin.meta(cmp, ctx.build.prerequisites, 'post');
+                    const sentinelValue = await prereqPlugin.meta(cmp, ctx.build.prerequisites, 'post');
                     await this.context.monorepo.store.writeFile(ctx.sentinelFile, sentinelValue);
                 },
                 title: 'Dumping cache info',
             },
-        ], {
-            rendererOptions: {
-                collapseSubtasks: true,
+            {
+                // Return build meta data
+                async task(ctx) {
+                    parentContext[cmp.name] = ctx;
+                    if (ctx.dryRun) {
+                        // parentTask.skip(`${parentTask.title} (dry run)`);
+                    }
+                },
             },
+        ], {
+            ctx: { dryRun },
         });
+        return list;
     }
 }

package/dist/src/monorepo/operations/components/GetComponentContainerOperation.d.ts

@@ -0,0 +1,6 @@
+import { ContainerInfo } from 'dockerode';
+import { Component } from '../../index.js';
+import { IOperation } from '../../../operations/index.js';
+export declare class GetComponentContainerOperation implements IOperation<Component, ContainerInfo> {
+    run(component: Component | string): Promise<ContainerInfo>;
+}

package/dist/src/monorepo/operations/components/GetComponentContainerOperation.js

@@ -0,0 +1,21 @@
+import { getContext } from '../../../index.js';
+import { ListContainersOperation } from '../../../docker/index.js';
+import { Component } from '../../index.js';
+export class GetComponentContainerOperation {
+    async run(component) {
+        const { monorepo } = getContext();
+        const cmpName = component instanceof Component ? component.name : component;
+        const matching = await monorepo.run(new ListContainersOperation(), {
+            filters: {
+                label: [`emb/project=${monorepo.name}`, `emb/component=${cmpName}`],
+            },
+        });
+        if (matching.length === 0) {
+            throw new Error(`Could not find a running container for '${cmpName}'`);
+        }
+        if (matching.length > 1) {
+            throw new Error(`More than one running container found for '${cmpName}'`);
+        }
+        return matching[0];
+    }
+}

package/dist/src/monorepo/operations/components/index.d.ts

@@ -1 +1,2 @@
 export * from './BuildComponentsOperation.js';
+export * from './GetComponentContainerOperation.js';

package/dist/src/monorepo/operations/components/index.js

@@ -1 +1,2 @@
 export * from './BuildComponentsOperation.js';
+export * from './GetComponentContainerOperation.js';

package/dist/src/monorepo/operations/index.d.ts

@@ -1 +1,3 @@
 export * from './components/index.js';
+export * from './shell/index.js';
+export * from './tasks/index.js';

package/dist/src/monorepo/operations/index.js

@@ -1 +1,3 @@
 export * from './components/index.js';
+export * from './shell/index.js';
+export * from './tasks/index.js';

package/dist/src/monorepo/operations/shell/ExecuteLocalCommandOperation.d.ts

@@ -0,0 +1,16 @@
+import { Readable } from 'node:stream';
+import * as z from 'zod';
+import { AbstractOperation } from '../../../operations/index.js';
+/**
+ * https://docs.docker.com/reference/api/engine/version/v1.37/#tag/Exec/operation/ContainerExec
+ */
+declare const schema: z.ZodObject<{
+    env: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
+    script: z.ZodString;
+    workingDir: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+export declare class ExecuteLocalCommandOperation extends AbstractOperation<typeof schema, Readable> {
+    constructor();
+    protected _run(input: z.input<typeof schema>): Promise<Readable>;
+}
+export {};

package/dist/src/monorepo/operations/shell/ExecuteLocalCommandOperation.js

@@ -0,0 +1,30 @@
+import { execa } from 'execa';
+import * as z from 'zod';
+import { AbstractOperation } from '../../../operations/index.js';
+/**
+ * https://docs.docker.com/reference/api/engine/version/v1.37/#tag/Exec/operation/ContainerExec
+ */
+const schema = z.object({
+    env: z
+        .record(z.string(), z.string())
+        .optional()
+        .describe('A list of environment variables in the form'),
+    script: z.string().describe('Command to run, as a string'),
+    workingDir: z
+        .string()
+        .optional()
+        .describe('The working directory for the exec process inside the container'),
+});
+export class ExecuteLocalCommandOperation extends AbstractOperation {
+    constructor() {
+        super(schema);
+    }
+    async _run(input) {
+        const process = execa(input.script, {
+            all: true,
+            cwd: input.workingDir,
+            shell: true,
+        });
+        return process.all;
+    }
+}

package/dist/src/monorepo/operations/shell/index.d.ts

@@ -0,0 +1 @@
+export * from './ExecuteLocalCommandOperation.js';

package/dist/src/monorepo/operations/shell/index.js

@@ -0,0 +1 @@
+export * from './ExecuteLocalCommandOperation.js';

package/dist/src/monorepo/operations/tasks/RunTasksOperation.d.ts

@@ -0,0 +1,21 @@
+import { Writable } from 'node:stream';
+import { TaskInfo } from '../../index.js';
+import { IOperation } from '../../../operations/index.js';
+export declare enum ExecutorType {
+    container = "container",
+    local = "local"
+}
+export type RunTasksOperationParams = {
+    tasks: Array<string>;
+    executor?: ExecutorType | undefined;
+};
+export type TaskWithScript = TaskInfo & {
+    script: string;
+};
+export declare class RunTasksOperation implements IOperation<RunTasksOperationParams, Array<TaskInfo>> {
+    protected out?: Writable | undefined;
+    constructor(out?: Writable | undefined);
+    run(params: RunTasksOperationParams): Promise<Array<TaskInfo>>;
+    protected runDocker(task: TaskWithScript, out?: Writable): Promise<void>;
+    protected runLocal(task: TaskWithScript): Promise<import("stream").Readable>;
+}

package/dist/src/monorepo/operations/tasks/RunTasksOperation.js

@@ -0,0 +1,84 @@
+import { getContext } from '../../../index.js';
+import { Manager } from '@listr2/manager';
+import { ContainerExecOperation } from '../../../docker/index.js';
+import { EMBCollection, findRunOrder } from '../../index.js';
+import { ExecuteLocalCommandOperation, GetComponentContainerOperation, } from '../index.js';
+export var ExecutorType;
+(function (ExecutorType) {
+    ExecutorType["container"] = "container";
+    ExecutorType["local"] = "local";
+})(ExecutorType || (ExecutorType = {}));
+export class RunTasksOperation {
+    out;
+    constructor(out) {
+        this.out = out;
+    }
+    async run(params) {
+        const { monorepo } = getContext();
+        // First ensure the selection is valid (user can use task IDs or names)
+        const collection = new EMBCollection(monorepo.tasks, {
+            idField: 'id',
+            depField: 'pre',
+        });
+        const ordered = findRunOrder(params.tasks, collection);
+        const runner = new Manager({
+            concurrent: false,
+            exitOnError: false,
+            rendererOptions: {
+                collapseSubtasks: false,
+                collapseSkips: false,
+            },
+        });
+        const list = runner.newListr(ordered.map((task) => {
+            return {
+                rendererOptions: {
+                    persistentOutput: true,
+                },
+                task: async (context, listrTask) => {
+                    if (!task.script) {
+                        return;
+                    }
+                    const executor = params.executor ??
+                        (task.component ? ExecutorType.container : ExecutorType.local);
+                    if (executor === ExecutorType.container && !task.component) {
+                        throw new Error('Cannot use the container executor with global tasks');
+                    }
+                    switch (executor) {
+                        case ExecutorType.container: {
+                            return this.runDocker(task, listrTask.stdout());
+                        }
+                        case ExecutorType.local: {
+                            return this.runLocal(task);
+                        }
+                        default: {
+                            throw new Error(`Unssuported executor type: ${executor}`);
+                        }
+                    }
+                },
+                title: `Running ${task.id}`,
+            };
+        }));
+        await list.run();
+        return ordered;
+    }
+    async runDocker(task, out) {
+        const { monorepo } = getContext();
+        const containerInfo = await monorepo.run(new GetComponentContainerOperation(), task.component);
+        return monorepo.run(new ContainerExecOperation(out), {
+            attachStderr: true,
+            attachStdout: true,
+            container: containerInfo.Id,
+            script: task.script,
+        });
+    }
+    async runLocal(task) {
+        const { monorepo } = getContext();
+        const cwd = task.component
+            ? monorepo.component(task.component).rootdir
+            : monorepo.rootDir;
+        return monorepo.run(new ExecuteLocalCommandOperation(), {
+            script: task.script,
+            workingDir: cwd,
+        });
+    }
+}

package/dist/src/monorepo/operations/tasks/index.d.ts

@@ -0,0 +1 @@
+export * from './RunTasksOperation.js';

package/dist/src/monorepo/operations/tasks/index.js

@@ -0,0 +1 @@
+export * from './RunTasksOperation.js';

package/dist/src/monorepo/plugins/ComponentDiscoverPlugin.d.ts

@@ -0,0 +1,15 @@
+import { Monorepo, MonorepoConfig } from '../index.js';
+import { AbstractPlugin } from './plugin.js';
+export type ComponentDiscoverPluginOptions = {
+    glob?: string;
+    ignore?: string | string[];
+};
+export declare const ComponentDiscoverPluginDefaultOptions: {
+    glob: string;
+};
+export declare class ComponentDiscoverPlugin extends AbstractPlugin<ComponentDiscoverPluginOptions> {
+    protected monorepo: Monorepo;
+    static name: string;
+    constructor(config: Partial<ComponentDiscoverPluginOptions>, monorepo: Monorepo);
+    extendConfig(config: MonorepoConfig): Promise<MonorepoConfig>;
+}