@enspirit/emb 0.0.6 → 0.0.8

package/dist/src/monorepo/monorepo.js

@@ -6,12 +6,10 @@ import { getPlugin } from './plugins/index.js';
 import { EMBStore } from './store/index.js';
 export class Monorepo {
     _config;
-    _env;
     _store;
     initialized = false;
     constructor(config) {
         this._config = new MonorepoConfig(config);
-        this._env = config.env || {};
     }
     // TODO: cache/improve
     get components() {
@@ -32,13 +30,22 @@ export class Monorepo {
     get rootDir() {
         return this._config.project.rootDir;
     }
+    get currentFlavor() {
+        return this._config.currentFlavor;
+    }
     get store() {
         return this._store;
     }
     get tasks() {
+        const globalTasks = (this._config.tasks || [])?.map((t) => {
+            return {
+                ...t,
+                id: `global:${t.name}`,
+            };
+        });
         return this.components.reduce((tasks, cmp) => {
             return [...tasks, ...cmp.tasks];
-        }, []);
+        }, globalTasks);
     }
     get vars() {
         return this._config.vars;
@@ -46,6 +53,20 @@ export class Monorepo {
     component(name) {
         return new Component(this._config.component(name), this);
     }
+    task(nameOrId) {
+        const byId = this.tasks.find((t) => t.id === nameOrId);
+        if (byId) {
+            return byId;
+        }
+        const found = this.tasks.filter((t) => t.name === nameOrId);
+        if (found.length > 1) {
+            throw new Error(`Task name ambigous, found multiple matches: ${nameOrId}`);
+        }
+        if (found.length === 0) {
+            throw new Error(`Task not found: ${nameOrId}`);
+        }
+        return found[0];
+    }
     async expand(strOrRecord, expander = new TemplateExpander()) {
         const options = {
             default: 'vars',
@@ -59,13 +80,29 @@ export class Monorepo {
         }
         return expander.expandRecord(strOrRecord, options);
     }
+    async installStore(store) {
+        this._store = store || new EMBStore(this);
+        await this._store.init();
+    }
+    async installEnv() {
+        // Expand env vars at the init and then we don't expand anymore
+        // The only available source for them is the existing env
+        const expander = new TemplateExpander();
+        const options = {
+            default: 'env',
+            sources: {
+                env: process.env,
+            },
+        };
+        const expanded = await expander.expandRecord(this._config.env, options);
+        Object.assign(process.env, expanded);
+    }
     // Initialize
     async init() {
         if (this.initialized) {
             throw new Error('Monorepo already initialized');
         }
-        this._store = new EMBStore(this);
-        await this._store.init();
+        await this.installStore();
         const plugins = this._config.plugins.map((p) => {
             const PluginClass = getPlugin(p.name);
             return new PluginClass(p.config, this);
@@ -74,17 +111,7 @@ export class Monorepo {
             const newConfig = await plugin.extendConfig?.(await pConfig);
             return newConfig ?? pConfig;
         }, Promise.resolve(this._config));
-        // Expand env vars at the init and then we don't expand anymore
-        // The only available source for them is the existing env
-        const expander = new TemplateExpander();
-        const options = {
-            default: 'env',
-            sources: {
-                env: process.env,
-            },
-        };
-        const expanded = await expander.expandRecord(this._config.env, options);
-        Object.assign(process.env, expanded);
+        await this.installEnv();
         this.initialized = true;
         await Promise.all(plugins.map(async (p) => {
             await p.init?.();
@@ -100,7 +127,8 @@ export class Monorepo {
     }
     async withFlavor(name) {
         const repo = new Monorepo(this._config.withFlavor(name));
-        await repo.init();
+        await repo.installStore();
+        await repo.installEnv();
         return repo;
     }
 }

package/dist/src/monorepo/operations/components/BuildComponentsOperation.d.ts

@@ -1,11 +1,21 @@
 import * as z from 'zod';
+import { DockerComponentBuild } from '../../../docker/index.js';
 import { AbstractOperation } from '../../../operations/index.js';
+export type BuildComponentMeta = {
+    dryRun?: boolean;
+    cacheHit?: boolean;
+    build: DockerComponentBuild;
+    preBuildMeta?: string;
+    sentinelFile: string;
+};
 declare const schema: z.ZodObject<{
     components: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    dryRun: z.ZodOptional<z.ZodBoolean>;
+    silent: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
-export declare class BuildComponentsOperation extends AbstractOperation<typeof schema, Array<unknown>> {
+export declare class BuildComponentsOperation extends AbstractOperation<typeof schema, Record<string, BuildComponentMeta>> {
     constructor();
-    protected _run(input: z.input<typeof schema>): Promise<Array<unknown>>;
+    protected _run(input: z.input<typeof schema>): Promise<Record<string, BuildComponentMeta>>;
     private buildComponent;
 }
 export {};

package/dist/src/monorepo/operations/components/BuildComponentsOperation.js

@@ -1,79 +1,63 @@
-import { Manager } from '@listr2/manager';
-import { createColors } from 'colorette';
-import { delay, ListrDefaultRendererLogLevels, PRESET_TIMER, } from 'listr2';
+import { getContext } from '../../../index.js';
 import * as z from 'zod';
 import { BuildImageOperation, getSentinelFile, } from '../../../docker/index.js';
-import { findBuildOrder } from '../../utils/findBuildOrder.js';
+import { EMBCollection, findRunOrder, taskManagerFactory, } from '../../index.js';
 import { AbstractOperation } from '../../../operations/index.js';
-import { FilePrerequisitePlugin } from '../../../prerequisites/FilePrerequisitePlugin.js';
-import { PrerequisiteType } from '../../../prerequisites/types.js';
+import { FilePrerequisitePlugin, PrerequisiteType } from '../../../prerequisites/index.js';
 const schema = z.object({
     components: z
         .array(z.string())
         .describe('The list of components to build')
         .optional(),
+    dryRun: z
+        .boolean()
+        .optional()
+        .describe('Do not build but return the config that would be used to build the images'),
+    silent: z
+        .boolean()
+        .optional()
+        .describe('Do not produce any output on the terminal'),
 });
 export class BuildComponentsOperation extends AbstractOperation {
     constructor() {
         super(schema);
     }
     async _run(input) {
-        const ordered = findBuildOrder(this.context.monorepo.components, input.components);
+        const { monorepo } = getContext();
+        const manager = taskManagerFactory();
+        const selection = (input.components || []).map((t) => monorepo.component(t));
+        const collection = new EMBCollection(this.context.monorepo.components, {
+            idField: 'name',
+            depField: 'dependencies',
+            forbidIdNameCollision: true,
+        });
+        const ordered = findRunOrder(selection.map((s) => s.name), collection);
         const tasks = await Promise.all(ordered.map((cmp) => {
             return {
                 task: async (context, task) => {
-                    return this.buildComponent(cmp, task);
+                    return this.buildComponent(cmp, task, context, input.dryRun);
                 },
                 title: `Building ${cmp.name}`,
             };
         }));
-        const manager = new Manager({
-            collectErrors: 'minimal',
-            concurrent: false,
-            exitOnError: true,
-            rendererOptions: {
-                collapseErrors: false,
-                collapseSubtasks: false,
-                color: {
-                    // @ts-expect-error not sure why
-                    [ListrDefaultRendererLogLevels.SKIPPED_WITH_COLLAPSE]: createColors().green,
-                },
-                icon: {
-                    [ListrDefaultRendererLogLevels.SKIPPED_WITH_COLLAPSE]: '♺',
-                },
-                timer: {
-                    ...PRESET_TIMER,
-                },
-            },
+        const list = manager.newListr([...tasks], {
+            renderer: input.silent ? 'silent' : 'default',
+            rendererOptions: { persistentOutput: true },
+            ctx: {},
         });
-        manager.add([
-            {
-                async task(_context, task) {
-                    return task.newListr([...tasks], {
-                        rendererOptions: {
-                            collapseSubtasks: false,
-                        },
-                    });
-                },
-                title: 'Building components',
-            },
-        ]);
-        await manager.runAll();
-        return ordered;
+        const results = await list.run();
+        return results;
     }
-    async buildComponent(cmp, parentTask) {
-        return parentTask.newListr([
+    async buildComponent(cmp, parentTask, parentContext, dryRun = false) {
+        const prereqPlugin = new FilePrerequisitePlugin();
+        const list = parentTask.newListr([
             // Collect all the prerequisites and other build infos
             // (This is when variables are expanded etc)
             {
                 async task(ctx) {
-                    // Reset the context to defaults (as apparently the context is shared amongst branches??)
-                    // TODO understand and fix
-                    ctx.skip = false;
-                    //
-                    ctx.parentTask = parentTask;
+                    // Install the context for this specific component build chain
+                    ctx.cacheHit = false;
                     ctx.sentinelFile = getSentinelFile(cmp);
-                    ctx.plugin = new FilePrerequisitePlugin();
                     ctx.build = await cmp.toDockerBuild();
                 },
                 title: 'Prepare build context',
@@ -81,7 +65,7 @@ export class BuildComponentsOperation extends AbstractOperation {
             // Check for sentinal information to see if the build can be skipped
             {
                 task: async (ctx) => {
-                    ctx.preBuildMeta = await ctx.plugin.meta(cmp, ctx.build.prerequisites, 'pre');
+                    ctx.preBuildMeta = await prereqPlugin.meta(cmp, ctx.build.prerequisites, 'pre');
                     let lastValue;
                     try {
                         lastValue = (await this.context.monorepo.store.readFile(ctx.sentinelFile)).toString();
@@ -90,10 +74,10 @@ export class BuildComponentsOperation extends AbstractOperation {
                         lastValue = undefined;
                     }
                     if (lastValue) {
-                        const diff = await ctx.plugin.diff(cmp, ctx.build.prerequisites, lastValue, ctx.preBuildMeta);
+                        const diff = await prereqPlugin.diff(cmp, ctx.build.prerequisites, lastValue, ctx.preBuildMeta);
                         if (!diff) {
-                            ctx.skip = true;
-                            ctx.parentTask.skip(`${ctx.parentTask.title} (cache hit)`);
+                            ctx.cacheHit = true;
+                            parentTask.skip(`${parentTask.title} (cache hit)`);
                         }
                     }
                 },
@@ -101,10 +85,9 @@ export class BuildComponentsOperation extends AbstractOperation {
             },
             {
                 task: async (ctx, task) => {
-                    if (ctx.skip) {
+                    if (ctx.cacheHit || ctx.dryRun) {
                         return task.skip();
                     }
-                    await delay(500);
                     const title = `Building image ${ctx.build.name}:${ctx.build.tag}`;
                     task.title = title;
                     const op = new BuildImageOperation((progress) => {
@@ -127,18 +110,26 @@ export class BuildComponentsOperation extends AbstractOperation {
             // Update sentinel file
             {
                 task: async (ctx, task) => {
-                    if (ctx.skip) {
+                    if (ctx.cacheHit || ctx.dryRun) {
                         return task.skip();
                     }
-                    const sentinelValue = await ctx.plugin.meta(cmp, ctx.build.prerequisites, 'post');
+                    const sentinelValue = await prereqPlugin.meta(cmp, ctx.build.prerequisites, 'post');
                     await this.context.monorepo.store.writeFile(ctx.sentinelFile, sentinelValue);
                 },
                 title: 'Dumping cache info',
             },
-        ], {
-            rendererOptions: {
-                collapseSubtasks: true,
+            {
+                // Return build meta data
+                async task(ctx) {
+                    parentContext[cmp.name] = ctx;
+                    if (ctx.dryRun) {
+                        parentTask.skip(`${parentTask.title} (dry run)`);
+                    }
+                },
             },
+        ], {
+            ctx: { dryRun },
         });
+        return list;
     }
 }

package/dist/src/monorepo/operations/components/GetComponentContainerOperation.d.ts

@@ -1,5 +1,5 @@
 import { ContainerInfo } from 'dockerode';
-import { Component } from '../../component.js';
+import { Component } from '../../index.js';
 import { IOperation } from '../../../operations/index.js';
 export declare class GetComponentContainerOperation implements IOperation<Component, ContainerInfo> {
     run(component: Component | string): Promise<ContainerInfo>;

package/dist/src/monorepo/operations/components/GetComponentContainerOperation.js

@@ -1,6 +1,6 @@
 import { getContext } from '../../../index.js';
 import { ListContainersOperation } from '../../../docker/index.js';
-import { Component } from '../../component.js';
+import { Component } from '../../index.js';
 export class GetComponentContainerOperation {
     async run(component) {
         const { monorepo } = getContext();

package/dist/src/monorepo/operations/shell/ExecuteLocalCommandOperation.d.ts

@@ -1,5 +1,4 @@
-import { ResultPromise } from 'execa';
-import { Writable } from 'node:stream';
+import { Readable } from 'node:stream';
 import * as z from 'zod';
 import { AbstractOperation } from '../../../operations/index.js';
 /**
@@ -10,9 +9,8 @@ declare const schema: z.ZodObject<{
     script: z.ZodString;
     workingDir: z.ZodOptional<z.ZodString>;
 }, z.core.$strip>;
-export declare class ExecuteLocalCommandOperation extends AbstractOperation<typeof schema, ResultPromise> {
-    protected out?: Writable | undefined;
-    constructor(out?: Writable | undefined);
-    protected _run(input: z.input<typeof schema>): Promise<ResultPromise>;
+export declare class ExecuteLocalCommandOperation extends AbstractOperation<typeof schema, Readable> {
+    constructor();
+    protected _run(input: z.input<typeof schema>): Promise<Readable>;
 }
 export {};

package/dist/src/monorepo/operations/shell/ExecuteLocalCommandOperation.js

@@ -16,10 +16,8 @@ const schema = z.object({
         .describe('The working directory for the exec process inside the container'),
 });
 export class ExecuteLocalCommandOperation extends AbstractOperation {
-    out;
-    constructor(out) {
+    constructor() {
         super(schema);
-        this.out = out;
     }
     async _run(input) {
         const process = execa(input.script, {
@@ -27,9 +25,6 @@ export class ExecuteLocalCommandOperation extends AbstractOperation {
             cwd: input.workingDir,
             shell: true,
         });
-        if (this.out) {
-            process.all?.pipe(this.out);
-        }
-        return process;
+        return process.all;
     }
 }

package/dist/src/monorepo/operations/tasks/RunTasksOperation.d.ts

@@ -0,0 +1,20 @@
+import { Writable } from 'node:stream';
+import { TaskInfo } from '../../index.js';
+import { IOperation } from '../../../operations/index.js';
+export declare enum ExecutorType {
+    container = "container",
+    local = "local"
+}
+export type RunTasksOperationParams = {
+    tasks: Array<string>;
+    executor?: ExecutorType | undefined;
+    allMatching?: boolean;
+};
+export type TaskWithScript = TaskInfo & {
+    script: string;
+};
+export declare class RunTasksOperation implements IOperation<RunTasksOperationParams, Array<TaskInfo>> {
+    run(params: RunTasksOperationParams): Promise<Array<TaskInfo>>;
+    protected runDocker(task: TaskWithScript, out?: Writable): Promise<void>;
+    protected runLocal(task: TaskWithScript): Promise<import("stream").Readable>;
+}

package/dist/src/monorepo/operations/tasks/RunTasksOperation.js

@@ -0,0 +1,82 @@
+import { getContext } from '../../../index.js';
+import { Manager } from '@listr2/manager';
+import { ContainerExecOperation } from '../../../docker/index.js';
+import { EMBCollection, findRunOrder } from '../../index.js';
+import { ExecuteLocalCommandOperation, GetComponentContainerOperation, } from '../index.js';
+export var ExecutorType;
+(function (ExecutorType) {
+    ExecutorType["container"] = "container";
+    ExecutorType["local"] = "local";
+})(ExecutorType || (ExecutorType = {}));
+export class RunTasksOperation {
+    async run(params) {
+        const { monorepo } = getContext();
+        // First ensure the selection is valid (user can use task IDs or names)
+        const collection = new EMBCollection(monorepo.tasks, {
+            idField: 'id',
+            depField: 'pre',
+        });
+        const ordered = findRunOrder(params.tasks, collection, {
+            onAmbiguous: params.allMatching ? 'runAll' : 'error',
+        });
+        const runner = new Manager({
+            concurrent: false,
+            exitOnError: false,
+            rendererOptions: {
+                collapseSubtasks: false,
+                collapseSkips: false,
+            },
+        });
+        const list = runner.newListr(ordered.map((task) => {
+            return {
+                rendererOptions: {
+                    persistentOutput: true,
+                },
+                task: async (context, listrTask) => {
+                    if (!task.script) {
+                        return;
+                    }
+                    const executor = params.executor ??
+                        (task.component ? ExecutorType.container : ExecutorType.local);
+                    if (executor === ExecutorType.container && !task.component) {
+                        throw new Error('Cannot use the container executor with global tasks');
+                    }
+                    switch (executor) {
+                        case ExecutorType.container: {
+                            return this.runDocker(task, listrTask.stdout());
+                        }
+                        case ExecutorType.local: {
+                            return this.runLocal(task);
+                        }
+                        default: {
+                            throw new Error(`Unssuported executor type: ${executor}`);
+                        }
+                    }
+                },
+                title: `Running ${task.id}`,
+            };
+        }));
+        await list.run();
+        return ordered;
+    }
+    async runDocker(task, out) {
+        const { monorepo } = getContext();
+        const containerInfo = await monorepo.run(new GetComponentContainerOperation(), task.component);
+        return monorepo.run(new ContainerExecOperation(out), {
+            attachStderr: true,
+            attachStdout: true,
+            container: containerInfo.Id,
+            script: task.script,
+        });
+    }
+    async runLocal(task) {
+        const { monorepo } = getContext();
+        const cwd = task.component
+            ? monorepo.component(task.component).rootdir
+            : monorepo.rootDir;
+        return monorepo.run(new ExecuteLocalCommandOperation(), {
+            script: task.script,
+            workingDir: cwd,
+        });
+    }
+}

package/dist/src/monorepo/operations/tasks/index.d.ts

@@ -1 +1 @@
-export * from './RunTaskOperation.js';
+export * from './RunTasksOperation.js';

package/dist/src/monorepo/operations/tasks/index.js

@@ -1 +1 @@
-export * from './RunTaskOperation.js';
+export * from './RunTasksOperation.js';

package/dist/src/monorepo/plugins/ComponentDiscoverPlugin.js

@@ -25,8 +25,10 @@ export class ComponentDiscoverPlugin extends AbstractPlugin {
             const name = dirname(path);
             const component = config.components.find((cmp) => cmp.name === name);
             const cfg = {
-                context: name,
                 name,
+                docker: {
+                    context: name,
+                },
             };
             return component ? deepmerge()(component, cfg) : cfg;
         });

package/dist/src/monorepo/plugins/EmbfileLoaderPlugin.d.ts

@@ -1,14 +1,14 @@
 import { Monorepo, MonorepoConfig } from '../index.js';
 import { AbstractPlugin } from './plugin.js';
 export type EmbfileLoaderPluginOptions = {
-    glob?: string;
+    glob?: string | string[];
 };
 export declare const EmbfileLoaderPluginDefaultOptions: {
     glob: string;
 };
-export declare class EmbfileLoaderPlugin extends AbstractPlugin<EmbfileLoaderPluginOptions> {
+export declare class EmbfileLoaderPlugin extends AbstractPlugin<Required<EmbfileLoaderPluginOptions>> {
     protected monorepo: Monorepo;
     static name: string;
-    constructor(config: Partial<EmbfileLoaderPluginOptions>, monorepo: Monorepo);
+    constructor(cfg: Partial<EmbfileLoaderPluginOptions>, monorepo: Monorepo);
     extendConfig(config: MonorepoConfig): Promise<MonorepoConfig>;
 }

package/dist/src/monorepo/plugins/EmbfileLoaderPlugin.js

@@ -8,15 +8,19 @@ export const EmbfileLoaderPluginDefaultOptions = {
 export class EmbfileLoaderPlugin extends AbstractPlugin {
     monorepo;
     static name = 'embfiles';
-    constructor(config, monorepo) {
-        super({
+    constructor(cfg, monorepo) {
+        const config = {
             ...EmbfileLoaderPluginDefaultOptions,
-            ...config,
-        }, monorepo);
+            ...cfg,
+        };
+        if (!Array.isArray(config.glob)) {
+            config.glob = [config.glob];
+        }
+        super(config, monorepo);
         this.monorepo = monorepo;
     }
     async extendConfig(config) {
-        const files = await glob(this.config.glob || EmbfileLoaderPluginDefaultOptions.glob, {
+        const files = await glob(this.config.glob, {
             ...this.config,
             cwd: config.project.rootDir,
         });

package/dist/src/monorepo/store/index.js

@@ -10,9 +10,11 @@ import { dirname, join, normalize } from 'node:path';
 export class EMBStore {
     monorepo;
     path;
-    constructor(monorepo, dirname = '.emb') {
+    constructor(monorepo, dirname) {
         this.monorepo = monorepo;
-        this.path = this.monorepo.join(dirname);
+        // By default, we use the flavor name to build that root of the store
+        // so that logs and sentinel files for different flavors are keps separate
+        this.path = this.monorepo.join(dirname || '.emb');
     }
     async createReadStream(path) {
         await this.mkdirp(dirname(path));
@@ -45,7 +47,7 @@ export class EMBStore {
         }
     }
     join(path) {
-        return join(this.path, path);
+        return join(this.path, this.monorepo.currentFlavor, path);
     }
     async mkdirp(path) {
         // Avoid getting out of the store by ensuring nothing goes past ../

package/dist/src/monorepo/taskManagerFactory.d.ts

@@ -0,0 +1,3 @@
+import { Manager } from '@listr2/manager';
+import { ListrBaseClassOptions } from 'listr2';
+export declare function taskManagerFactory<T extends Record<PropertyKey, unknown>>(override?: ListrBaseClassOptions): Manager<T>;

package/dist/src/monorepo/taskManagerFactory.js

@@ -0,0 +1,20 @@
+import { Manager } from '@listr2/manager';
+import { ListrDefaultRendererLogLevels, PRESET_TIMER, } from 'listr2';
+export function taskManagerFactory(override) {
+    return new Manager({
+        collectErrors: 'minimal',
+        concurrent: false,
+        exitOnError: true,
+        rendererOptions: {
+            collapseErrors: false,
+            collapseSubtasks: false,
+            icon: {
+                [ListrDefaultRendererLogLevels.SKIPPED_WITH_COLLAPSE]: '♺',
+            },
+            timer: {
+                ...PRESET_TIMER,
+            },
+        },
+        ...override,
+    });
+}

package/dist/src/monorepo/types.d.ts

@@ -3,5 +3,6 @@ export type TaskInfo = {
     description?: string;
     id: string;
     name: string;
-    script: string;
+    script?: string;
+    pre?: Array<string>;
 };

package/dist/src/monorepo/utils/findRunOrder.d.ts

@@ -0,0 +1,34 @@
+type DepList = readonly string[] | string[] | undefined;
+type CollectionConfig<IDK extends PropertyKey, DPK extends PropertyKey> = {
+    idField: IDK;
+    depField: DPK;
+    /** If true, throw when an item's id equals some other item's name (or vice versa). */
+    forbidIdNameCollision?: boolean;
+};
+type AmbiguityPolicy = 'error' | 'runAll';
+export declare class EMBCollection<T extends Partial<Record<DPK, DepList>> & Record<IDK, string> & {
+    name: string;
+}, IDK extends keyof T, DPK extends keyof T> {
+    private items;
+    readonly idField: IDK;
+    readonly depField: DPK;
+    private byId;
+    private byName;
+    constructor(items: Iterable<T>, cfg: CollectionConfig<IDK, DPK>);
+    /** All items (stable array iteration) */
+    get all(): Iterable<T>;
+    idOf(t: T): string;
+    depsOf(t: T): readonly string[];
+    matches(ref: string, opts?: {
+        multiple?: false;
+    }): T;
+    matches(ref: string, opts: {
+        multiple: true;
+    }): T[];
+}
+export declare function findRunOrder<T extends Partial<Record<DPK, DepList>> & Record<IDK, string> & {
+    name: string;
+}, IDK extends keyof T, DPK extends keyof T>(selection: readonly string[], collection: EMBCollection<T, IDK, DPK>, { onAmbiguous }?: {
+    onAmbiguous?: AmbiguityPolicy | undefined;
+}): T[];
+export {};