@enspirit/emb 0.0.1 → 0.0.3

package/dist/src/docker/protobuf/index.js

@@ -0,0 +1,29 @@
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import protobuf from 'protobufjs';
+let control;
+export const loadControl = async () => {
+    if (control) {
+        return control;
+    }
+    const path = join(dirname(fileURLToPath(import.meta.url)), './control.proto');
+    control = await protobuf.load(path);
+    return control;
+};
+export const decodeBuildkitStatusResponse = async (str) => {
+    const control = await loadControl();
+    const buffer = typeof str === 'string' ? Buffer.from(str, 'base64') : str;
+    const uint8 = new Uint8Array(buffer);
+    const Trace = control.lookupType('moby.buildkit.v1.StatusResponse');
+    const message = Trace.decode(uint8);
+    const object = Trace.toObject(message, {
+        arrays: true,
+        bytes: String,
+        defaults: true,
+        enums: String,
+        longs: String,
+        objects: true,
+        oneofs: true,
+    });
+    return object;
+};

package/dist/src/docker/types.d.ts

@@ -0,0 +1,14 @@
+import { FilePrerequisite } from '../prerequisites/index.js';
+export type EnvVariable = {
+    name: string;
+};
+export interface DockerComponentBuild {
+    buildArgs?: Record<string, string>;
+    context: string;
+    dockerfile: string;
+    labels?: Record<string, string>;
+    name: string;
+    prerequisites: Array<FilePrerequisite>;
+    tag: string;
+    target?: string;
+}

package/dist/src/docker/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/docker/utils.d.ts

@@ -0,0 +1,7 @@
+import { Component } from '../monorepo/index.js';
+export declare const shortId: (id: string) => string;
+/**
+ * This is too naive and will need better logic to ensure
+ * we take care of flavors etc
+ */
+export declare const getSentinelFile: (component: Component) => string;

package/dist/src/docker/utils.js

@@ -0,0 +1,10 @@
+export const shortId = (id) => {
+    return id.slice(0, 12);
+};
+/**
+ * This is too naive and will need better logic to ensure
+ * we take care of flavors etc
+ */
+export const getSentinelFile = (component) => {
+    return `sentinels/docker/build/${component.name}.built`;
+};

package/dist/src/executors/docker.d.ts

@@ -0,0 +1,6 @@
+import { Container } from 'dockerode';
+import { Executor, ExecutorRunOptions } from './types.js';
+export type DockerExecutorRunOptions = ExecutorRunOptions & {
+    container: Container;
+};
+export declare const dockerExecutor: Executor<DockerExecutorRunOptions>;

package/dist/src/executors/docker.js

@@ -0,0 +1,14 @@
+export const dockerExecutor = {
+    async run(script, options) {
+        const exec = await options.container.exec({
+            AttachStderr: true,
+            AttachStdout: true,
+            Cmd: ['bash', '-c', script],
+        });
+        const stream = await exec.start({});
+        if (options.out) {
+            options.container.modem.demuxStream(stream, options.out, options.out);
+        }
+        return stream;
+    },
+};

package/dist/src/executors/index.d.ts

@@ -0,0 +1,6 @@
+export * from './docker.js';
+export * from './shell.js';
+export declare enum ExecutorType {
+    container = "container",
+    local = "local"
+}

package/dist/src/executors/index.js

@@ -0,0 +1,7 @@
+export * from './docker.js';
+export * from './shell.js';
+export var ExecutorType;
+(function (ExecutorType) {
+    ExecutorType["container"] = "container";
+    ExecutorType["local"] = "local";
+})(ExecutorType || (ExecutorType = {}));

package/dist/src/executors/shell.d.ts

@@ -0,0 +1,2 @@
+import { Executor } from './types.js';
+export declare const shellExecutor: Executor;

package/dist/src/executors/shell.js

@@ -0,0 +1,14 @@
+import { execa } from 'execa';
+export const shellExecutor = {
+    async run(script, options) {
+        const process = execa(script, {
+            all: true,
+            cwd: options.cwd,
+            shell: true,
+        });
+        if (options.out) {
+            process.all?.pipe(options.out);
+        }
+        return process;
+    },
+};

package/dist/src/executors/types.d.ts

@@ -0,0 +1,8 @@
+import { Writable } from 'node:stream';
+export type ExecutorRunOptions = {
+    cwd?: string;
+    out?: Writable;
+};
+export type Executor<RO extends ExecutorRunOptions = ExecutorRunOptions, T = unknown> = {
+    run(script: string, options?: RO): Promise<T>;
+};

package/dist/src/executors/types.js

@@ -0,0 +1 @@
+export {};

package/dist/src/index.d.ts

@@ -0,0 +1,3 @@
+export * from './context.js';
+export * from './types.js';
+export { run } from '@oclif/core';

package/dist/src/index.js

@@ -0,0 +1,3 @@
+export * from './context.js';
+export * from './types.js';
+export { run } from '@oclif/core';

package/dist/src/monorepo/component.d.ts

@@ -0,0 +1,24 @@
+import { ComponentConfig } from '../config/index.js';
+import { DockerComponentBuild } from '../docker/index.js';
+import { Monorepo, TaskInfo } from './index.js';
+import { FilePrerequisite } from '../prerequisites/index.js';
+export declare class Component {
+    protected _config: ComponentConfig;
+    protected monorepo: Monorepo;
+    constructor(_config: ComponentConfig, monorepo: Monorepo);
+    get config(): ComponentConfig;
+    get context(): string | undefined;
+    get dependencies(): Component[];
+    get imageName(): string;
+    get imageTag(): string;
+    get labels(): {
+        'emb/component': string;
+    };
+    get name(): string;
+    get rootdir(): string;
+    get tasks(): Array<TaskInfo>;
+    cloneWith(config: Partial<ComponentConfig>): Component;
+    getPrerequisites(): Promise<Array<FilePrerequisite>>;
+    join(path: string): string;
+    toDockerBuild(): Promise<DockerComponentBuild>;
+}

package/dist/src/monorepo/component.js

@@ -0,0 +1,77 @@
+import deepmerge from '@fastify/deepmerge';
+import { join } from 'node:path';
+import { GitPrerequisitePlugin } from '../prerequisites/index.js';
+export class Component {
+    _config;
+    monorepo;
+    constructor(_config, monorepo) {
+        this._config = _config;
+        this.monorepo = monorepo;
+    }
+    get config() {
+        return structuredClone(this._config);
+    }
+    get context() {
+        return this.config.context;
+    }
+    get dependencies() {
+        return this.monorepo.components.filter((c) => this.config.dependencies?.includes(c.name));
+    }
+    get imageName() {
+        return join(this.monorepo.name, this.name);
+    }
+    get imageTag() {
+        return this.monorepo.defaults.docker?.tag || 'latest';
+    }
+    get labels() {
+        return {
+            'emb/component': this.name,
+            ...this._config.labels,
+        };
+    }
+    get name() {
+        return this.config.name;
+    }
+    get rootdir() {
+        return this.monorepo.join(this.context || this.name);
+    }
+    get tasks() {
+        return (this.config.tasks || [])?.map((t) => {
+            return {
+                ...t,
+                component: this.name,
+                id: `${this.name}:${t.name}`,
+            };
+        });
+    }
+    cloneWith(config) {
+        return new Component({
+            ...this.config,
+            ...config,
+        }, this.monorepo);
+    }
+    async getPrerequisites() {
+        // TODO: move this to config with potential overridzs
+        const plugin = new GitPrerequisitePlugin();
+        return plugin.collect(this);
+    }
+    join(path) {
+        return this.monorepo.join(this.context || this.name, path);
+    }
+    async toDockerBuild() {
+        return {
+            buildArgs: await this.monorepo.expand(deepmerge()(this.monorepo.defaults.docker?.buildArgs || {}, this.config.buildArgs || {})),
+            context: this.rootdir,
+            dockerfile: this.config.dockerfile || 'Dockerfile',
+            labels: deepmerge()({
+                ...this.monorepo.defaults.docker?.labels,
+            }, this.labels),
+            name: this.imageName,
+            prerequisites: await this.getPrerequisites(),
+            tag: this.imageTag
+                ? await this.monorepo.expand(this.imageTag)
+                : 'latest',
+            target: this.config.target || this.monorepo.defaults?.docker?.target,
+        };
+    }
+}

package/dist/src/monorepo/config.d.ts

@@ -0,0 +1,16 @@
+import { ComponentConfig, DefaultSettings, FlavorConfig, IMonorepoConfig, IProjectConfig, PluginConfig } from '../config/index.js';
+export declare class MonorepoConfig implements IMonorepoConfig {
+    components: ComponentConfig[];
+    defaults: DefaultSettings;
+    env: Record<string, string>;
+    flavors: Array<FlavorConfig>;
+    plugins: Array<PluginConfig>;
+    project: IProjectConfig;
+    vars: Record<string, unknown>;
+    constructor(config: IMonorepoConfig);
+    component(name: string): ComponentConfig;
+    flavor(name: string): FlavorConfig;
+    toJSON(): Required<IMonorepoConfig>;
+    with(overrides: Partial<IMonorepoConfig>): MonorepoConfig;
+    withFlavor(name: string): MonorepoConfig;
+}

package/dist/src/monorepo/config.js

@@ -0,0 +1,66 @@
+import deepMerge from '@fastify/deepmerge';
+import { deepMergeArray } from '../utils/index.js';
+export class MonorepoConfig {
+    components;
+    defaults;
+    env;
+    flavors;
+    plugins;
+    project;
+    vars;
+    constructor(config) {
+        this.components = config.components;
+        this.defaults = config.defaults || {};
+        this.project = config.project;
+        this.vars = config.vars || {};
+        this.flavors = config.flavors || [];
+        this.env = config.env || {};
+        this.plugins = config.plugins || [];
+    }
+    component(name) {
+        const config = this.components.find((c) => c.name === name);
+        if (!config) {
+            throw new Error(`Unknown component ${name}`);
+        }
+        return config;
+    }
+    flavor(name) {
+        const flavor = this.flavors.find((f) => f.name === name);
+        if (!flavor) {
+            throw new Error(`Unknown flavor: ${name}`);
+        }
+        return flavor;
+    }
+    toJSON() {
+        return {
+            components: this.components,
+            defaults: this.defaults,
+            env: this.env,
+            flavors: this.flavors,
+            plugins: this.plugins,
+            project: this.project,
+            vars: this.vars,
+        };
+    }
+    with(overrides) {
+        const newConfig = {
+            ...this.toJSON(),
+            components: deepMerge({
+                mergeArray() {
+                    // Merge components by identifying them by name
+                    return (target, source) => deepMergeArray(target, source, (item) => {
+                        return item.name;
+                    });
+                },
+            })(this.components, overrides?.components || []),
+            defaults: deepMerge()(this.defaults, overrides.defaults),
+            env: deepMerge()(this.env, overrides?.env || {}),
+            project: deepMerge()(this.project, overrides?.project || {}),
+            vars: deepMerge()(this.vars, overrides?.vars || {}),
+        };
+        return new MonorepoConfig(newConfig);
+    }
+    withFlavor(name) {
+        return this.with(this.flavor(name));
+    }
+}

package/dist/src/monorepo/index.d.ts

@@ -0,0 +1,7 @@
+export * from './component.js';
+export * from './config.js';
+export * from './monorepo.js';
+export * from './plugins/index.js';
+export * from './store/index.js';
+export * from './types.js';
+export * from './utils/index.js';

package/dist/src/monorepo/index.js

@@ -0,0 +1,7 @@
+export * from './component.js';
+export * from './config.js';
+export * from './monorepo.js';
+export * from './plugins/index.js';
+export * from './store/index.js';
+export * from './types.js';
+export * from './utils/index.js';

package/dist/src/monorepo/monorepo.d.ts

@@ -0,0 +1,29 @@
+import { IMonorepoConfig } from '../config/index.js';
+import { IOperation } from '../operations/types.js';
+import { TemplateExpander } from '../utils/index.js';
+import { Component } from './component.js';
+import { EMBStore } from './store/index.js';
+import { TaskInfo } from './types.js';
+export declare class Monorepo {
+    private _config;
+    private _env;
+    private _store;
+    private initialized;
+    constructor(config: IMonorepoConfig);
+    get components(): Component[];
+    get config(): IMonorepoConfig;
+    get defaults(): import("../config/index.js").DefaultSettings;
+    get flavors(): string[];
+    get name(): string;
+    get rootDir(): string;
+    get store(): EMBStore;
+    get tasks(): TaskInfo[];
+    get vars(): Record<string, unknown>;
+    component(name: string): Component;
+    expand(str: string, expander?: TemplateExpander): Promise<string>;
+    expand<R extends Record<string, unknown>>(record: R, expander?: TemplateExpander): Promise<R>;
+    init(): Promise<Monorepo>;
+    join(...paths: string[]): string;
+    run<I, O>(operation: IOperation<I, O>, args: I): Promise<O>;
+    withFlavor(name: string): Promise<Monorepo>;
+}

package/dist/src/monorepo/monorepo.js

@@ -0,0 +1,106 @@
+import { join } from 'node:path';
+import { TemplateExpander } from '../utils/index.js';
+import { Component } from './component.js';
+import { MonorepoConfig } from './config.js';
+import { getPlugin } from './plugins/index.js';
+import { EMBStore } from './store/index.js';
+export class Monorepo {
+    _config;
+    _env;
+    _store;
+    initialized = false;
+    constructor(config) {
+        this._config = new MonorepoConfig(config);
+        this._env = config.env || {};
+    }
+    // TODO: cache/improve
+    get components() {
+        return this._config.components.map((c) => new Component(c, this));
+    }
+    get config() {
+        return this._config.toJSON();
+    }
+    get defaults() {
+        return this._config.defaults;
+    }
+    get flavors() {
+        return this._config.flavors.map((f) => f.name);
+    }
+    get name() {
+        return this._config.project.name;
+    }
+    get rootDir() {
+        return this._config.project.rootDir;
+    }
+    get store() {
+        return this._store;
+    }
+    get tasks() {
+        return this.components.reduce((tasks, cmp) => {
+            return [...tasks, ...cmp.tasks];
+        }, []);
+    }
+    get vars() {
+        return this._config.vars;
+    }
+    component(name) {
+        return new Component(this._config.component(name), this);
+    }
+    async expand(strOrRecord, expander = new TemplateExpander()) {
+        const options = {
+            default: 'vars',
+            sources: {
+                env: process.env,
+                vars: this.vars,
+            },
+        };
+        if (typeof strOrRecord === 'string') {
+            return expander.expand(strOrRecord, options);
+        }
+        return expander.expandRecord(strOrRecord, options);
+    }
+    // Initialize
+    async init() {
+        if (this.initialized) {
+            throw new Error('Monorepo already initialized');
+        }
+        this._store = new EMBStore(this);
+        await this._store.init();
+        const plugins = this._config.plugins.map((p) => {
+            const PluginClass = getPlugin(p.name);
+            return new PluginClass(p.config, this);
+        });
+        this._config = await plugins.reduce(async (pConfig, plugin) => {
+            const newConfig = await plugin.extendConfig?.(await pConfig);
+            return newConfig ?? pConfig;
+        }, Promise.resolve(this._config));
+        // Expand env vars at the init and then we don't expand anymore
+        // The only available source for them is the existing env
+        const expander = new TemplateExpander();
+        const options = {
+            default: 'env',
+            sources: {
+                env: process.env,
+            },
+        };
+        const expanded = await expander.expandRecord(this._config.env, options);
+        Object.assign(process.env, expanded);
+        this.initialized = true;
+        await Promise.all(plugins.map(async (p) => {
+            await p.init?.();
+        }));
+        return this;
+    }
+    // Helper to build relative path to the root dir
+    join(...paths) {
+        return join(this._config.project.rootDir, ...paths);
+    }
+    async run(operation, args) {
+        return operation.run(args);
+    }
+    async withFlavor(name) {
+        const repo = new Monorepo(this._config.withFlavor(name));
+        await repo.init();
+        return repo;
+    }
+}

package/dist/src/monorepo/operations/components/BuildComponentsOperation.d.ts

@@ -0,0 +1,11 @@
+import * as z from 'zod';
+import { AbstractOperation } from '../../../operations/index.js';
+declare const schema: z.ZodObject<{
+    components: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export declare class BuildComponentsOperation extends AbstractOperation<typeof schema, Array<unknown>> {
+    constructor();
+    protected _run(input: z.input<typeof schema>): Promise<Array<unknown>>;
+    private buildComponent;
+}
+export {};

package/dist/src/monorepo/operations/components/BuildComponentsOperation.js

@@ -0,0 +1,144 @@
+import { Manager } from '@listr2/manager';
+import { createColors } from 'colorette';
+import { delay, ListrDefaultRendererLogLevels, PRESET_TIMER, } from 'listr2';
+import * as z from 'zod';
+import { BuildImageOperation, getSentinelFile, } from '../../../docker/index.js';
+import { findBuildOrder } from '../../utils/findBuildOrder.js';
+import { AbstractOperation } from '../../../operations/index.js';
+import { FilePrerequisitePlugin } from '../../../prerequisites/FilePrerequisitePlugin.js';
+import { PrerequisiteType } from '../../../prerequisites/types.js';
+const schema = z.object({
+    components: z
+        .array(z.string())
+        .describe('The list of components to build')
+        .optional(),
+});
+export class BuildComponentsOperation extends AbstractOperation {
+    constructor() {
+        super(schema);
+    }
+    async _run(input) {
+        const ordered = findBuildOrder(this.context.monorepo.components, input.components);
+        const tasks = await Promise.all(ordered.map((cmp) => {
+            return {
+                task: async (context, task) => {
+                    return this.buildComponent(cmp, task);
+                },
+                title: `Building ${cmp.name}`,
+            };
+        }));
+        const manager = new Manager({
+            collectErrors: 'minimal',
+            concurrent: false,
+            exitOnError: true,
+            rendererOptions: {
+                collapseErrors: false,
+                collapseSubtasks: false,
+                color: {
+                    // @ts-expect-error not sure why
+                    [ListrDefaultRendererLogLevels.SKIPPED_WITH_COLLAPSE]: createColors().green,
+                },
+                icon: {
+                    [ListrDefaultRendererLogLevels.SKIPPED_WITH_COLLAPSE]: '♺',
+                },
+                timer: {
+                    ...PRESET_TIMER,
+                },
+            },
+        });
+        manager.add([
+            {
+                async task(_context, task) {
+                    return task.newListr([...tasks], {
+                        rendererOptions: {
+                            collapseSubtasks: false,
+                        },
+                    });
+                },
+                title: 'Building components',
+            },
+        ]);
+        await manager.runAll();
+        return ordered;
+    }
+    async buildComponent(cmp, parentTask) {
+        return parentTask.newListr([
+            // Collect all the prerequisites and other build infos
+            // (This is when variables are expanded etc)
+            {
+                async task(ctx) {
+                    // Reset the context to defaults (as apparently the context is shared amongst branches??)
+                    // TODO understand and fix
+                    ctx.skip = false;
+                    //
+                    ctx.parentTask = parentTask;
+                    ctx.sentinelFile = getSentinelFile(cmp);
+                    ctx.plugin = new FilePrerequisitePlugin();
+                    ctx.build = await cmp.toDockerBuild();
+                },
+                title: 'Prepare build context',
+            },
+            // Check for sentinal information to see if the build can be skipped
+            {
+                task: async (ctx) => {
+                    ctx.preBuildMeta = await ctx.plugin.meta(cmp, ctx.build.prerequisites, 'pre');
+                    let lastValue;
+                    try {
+                        lastValue = (await this.context.monorepo.store.readFile(ctx.sentinelFile)).toString();
+                    }
+                    catch {
+                        lastValue = undefined;
+                    }
+                    if (lastValue) {
+                        const diff = await ctx.plugin.diff(cmp, ctx.build.prerequisites, lastValue, ctx.preBuildMeta);
+                        if (!diff) {
+                            ctx.skip = true;
+                            ctx.parentTask.skip(`${ctx.parentTask.title} (cache hit)`);
+                        }
+                    }
+                },
+                title: 'Checking prerequisites',
+            },
+            {
+                task: async (ctx, task) => {
+                    if (ctx.skip) {
+                        return task.skip();
+                    }
+                    await delay(500);
+                    const title = `Building image ${ctx.build.name}:${ctx.build.tag}`;
+                    task.title = title;
+                    const op = new BuildImageOperation((progress) => {
+                        task.title = progress;
+                    });
+                    await this.context.monorepo.run(op, {
+                        ...ctx.build,
+                        src: ctx.build.prerequisites
+                            .filter((p) => {
+                            return p.type === PrerequisiteType.file;
+                        })
+                            .map((p) => p.path),
+                        tag: ctx.build.name + ':' + ctx.build.tag,
+                    });
+                    // Restore title
+                    task.title = title;
+                },
+                title: 'Building image',
+            },
+            // Update sentinel file
+            {
+                task: async (ctx, task) => {
+                    if (ctx.skip) {
+                        return task.skip();
+                    }
+                    const sentinelValue = await ctx.plugin.meta(cmp, ctx.build.prerequisites, 'post');
+                    await this.context.monorepo.store.writeFile(ctx.sentinelFile, sentinelValue);
+                },
+                title: 'Dumping cache info',
+            },
+        ], {
+            rendererOptions: {
+                collapseSubtasks: true,
+            },
+        });
+    }
+}

package/dist/src/monorepo/operations/components/index.d.ts

@@ -0,0 +1 @@
+export * from './BuildComponentsOperation.js';

package/dist/src/monorepo/operations/components/index.js

@@ -0,0 +1 @@
+export * from './BuildComponentsOperation.js';

package/dist/src/monorepo/operations/index.d.ts

@@ -0,0 +1 @@
+export * from './components/index.js';

package/dist/src/monorepo/operations/index.js

@@ -0,0 +1 @@
+export * from './components/index.js';

package/dist/src/monorepo/plugins/ComponentsDiscover.d.ts

@@ -0,0 +1,6 @@
+import { MonorepoConfig } from '../index.js';
+import { AbstractPlugin } from './plugin.js';
+export declare class ComponentDiscoverPlugin extends AbstractPlugin {
+    static name: string;
+    extendConfig(config: MonorepoConfig): Promise<MonorepoConfig>;
+}