hologit 0.43.2 → 0.45.0

package/index.d.ts

@@ -1,6 +1,15 @@
+declare module 'git-client' {
+    export class Git {
+        constructor(options: { gitDir: string; workTree?: string });
+    }
+}
+
+declare module '@iarna/toml' {
+    export function parse(content: string): any;
+}
+
 declare module 'hologit' {
     import { Git as GitClient } from 'git-client';
-    import { Docker } from 'dockerode';
 
     export interface GitOptions {
         gitDir: string;
@@ -59,16 +68,40 @@ declare module 'hologit' {
         cacheTo?: string | null;
     }
 
+    export interface DockerExecOptions {
+        $relayStderr?: boolean;
+        $relayStdout?: boolean;
+    }
+
+    export interface StudioContainer {
+        id?: string;
+        type?: 'studio';
+        env?: { [key: string]: string };
+        defaultUser?: string;
+    }
+
+    export interface HoloSpec {
+        holospec: {
+            lens: {
+                input: string;
+                container?: string;
+                package?: string;
+                command?: string;
+                [key: string]: any;
+            };
+        };
+    }
+
     export class Git {
-        static async get(): Promise<typeof GitClient>;
+        static get(): Promise<typeof GitClient>;
         constructor(options: GitOptions);
         gitDir: string;
         workTree?: string;
     }
 
     export class BlobObject {
-        static async write(repo: Repo, content: string): Promise<BlobObject>;
-        static async writeFromFile(repo: Repo, filePath: string): Promise<BlobObject>;
+        static write(repo: Repo, content: string): Promise<BlobObject>;
+        static writeFromFile(repo: Repo, filePath: string): Promise<BlobObject>;
 
         constructor(repo: Repo, options: GitObjectOptions);
 
@@ -78,12 +111,12 @@ declare module 'hologit' {
         isBlob: boolean;
         type: 'blob';
 
-        async read(): Promise<string>;
+        read(): Promise<string>;
     }
 
     export class TreeObject {
         static getEmptyTreeHash(): string;
-        static async createFromRef(repo: Repo, ref: string): Promise<TreeObject>;
+        static createFromRef(repo: Repo, ref: string): Promise<TreeObject>;
 
         constructor(repo: Repo, options?: { hash?: string; parent?: TreeObject | null });
 
@@ -95,19 +128,19 @@ declare module 'hologit' {
         type: 'tree';
         mode: '040000';
 
-        async getHash(): Promise<string>;
+        getHash(): Promise<string>;
         getWrittenHash(): string | null;
         markDirty(): void;
-        async getChild(childPath: string): Promise<TreeObject | BlobObject | CommitObject | null>;
-        async writeChild(childPath: string, content: string | BlobObject): Promise<BlobObject>;
-        async getChildren(): Promise<{ [key: string]: TreeObject | BlobObject | CommitObject }>;
-        async getBlobMap(): Promise<{ [key: string]: BlobObject }>;
-        async deleteChild(childPath: string): Promise<void>;
-        async getSubtree(subtreePath: string, create?: boolean): Promise<TreeObject | null>;
-        async getSubtreeStack(subtreePath: string, create?: boolean): Promise<TreeObject[] | null>;
-        async write(): Promise<string>;
-        async merge(input: TreeObject, options?: MergeOptions, basePath?: string, preloadChildren?: boolean): Promise<void>;
-        async clone(): Promise<TreeObject>;
+        getChild(childPath: string): Promise<TreeObject | BlobObject | CommitObject | null>;
+        writeChild(childPath: string, content: string | BlobObject): Promise<BlobObject>;
+        getChildren(): Promise<{ [key: string]: TreeObject | BlobObject | CommitObject }>;
+        getBlobMap(): Promise<{ [key: string]: BlobObject }>;
+        deleteChild(childPath: string): Promise<void>;
+        getSubtree(subtreePath: string, create?: boolean): Promise<TreeObject | null>;
+        getSubtreeStack(subtreePath: string, create?: boolean): Promise<TreeObject[] | null>;
+        write(): Promise<string>;
+        merge(input: TreeObject, options?: MergeOptions, basePath?: string, preloadChildren?: boolean): Promise<void>;
+        clone(): Promise<TreeObject>;
     }
 
     export class CommitObject {
@@ -128,10 +161,10 @@ declare module 'hologit' {
 
         getWorkspace(): Workspace;
         getRepo(): Repo;
-        async readConfig(): Promise<any>;
-        async writeConfig(config?: any): Promise<void>;
-        async getConfig(): Promise<any>;
-        async getCachedConfig(): Promise<any>;
+        readConfig(): Promise<any>;
+        writeConfig(config?: any): Promise<void>;
+        getConfig(): Promise<any>;
+        getCachedConfig(): Promise<any>;
     }
 
     export class Branch extends Configurable {
@@ -141,17 +174,17 @@ declare module 'hologit' {
 
         getKind(): 'holobranch';
         getConfigPath(): string;
-        async isDefined(): Promise<boolean>;
+        isDefined(): Promise<boolean>;
         getMapping(key: string): Mapping;
-        async getMappings(): Promise<Map<string, Mapping>>;
-        async composite(options: {
+        getMappings(): Promise<Map<string, Mapping>>;
+        composite(options: {
             outputTree?: TreeObject;
             fetch?: boolean | string[];
             cacheFrom?: string | null;
             cacheTo?: string | null;
         }): Promise<TreeObject>;
         getLens(name: string): Lens;
-        async getLenses(): Promise<Map<string, Lens>>;
+        getLenses(): Promise<Map<string, Lens>>;
     }
 
     export class Source extends Configurable {
@@ -163,21 +196,21 @@ declare module 'hologit' {
 
         getKind(): 'holosource';
         getConfigPath(): string;
-        async getSpec(): Promise<{ hash: string; ref: string; data: any }>;
-        async getCachedSpec(): Promise<{ hash: string; ref: string; data: any }>;
-        async queryRef(): Promise<{ hash: string; ref: string } | null>;
-        async hashWorkTree(): Promise<string | null>;
-        async getOutputTree(options?: {
+        getSpec(): Promise<{ hash: string; ref: string; data: any }>;
+        getCachedSpec(): Promise<{ hash: string; ref: string; data: any }>;
+        queryRef(): Promise<{ hash: string; ref: string } | null>;
+        hashWorkTree(): Promise<string | null>;
+        getOutputTree(options?: {
             working?: boolean | null;
             fetch?: boolean | string[];
             cacheFrom?: string | null;
             cacheTo?: string | null;
         }): Promise<string>;
-        async getHead(options?: { required?: boolean; working?: boolean | null }): Promise<string | null>;
-        async getCachedHead(): Promise<string | null>;
-        async getBranch(): Promise<string | null>;
-        async fetch(options?: { depth?: number; unshallow?: boolean | null }, ...refs: string[]): Promise<{ refs: string[] }>;
-        async checkout(options?: { submodule?: boolean }): Promise<{
+        getHead(options?: { required?: boolean; working?: boolean | null }): Promise<string | null>;
+        getCachedHead(): Promise<string | null>;
+        getBranch(): Promise<string | null>;
+        fetch(options?: { depth?: number; unshallow?: boolean | null }, ...refs: string[]): Promise<{ refs: string[] }>;
+        checkout(options?: { submodule?: boolean }): Promise<{
             path: string;
             head: string;
             branch: string | null;
@@ -195,13 +228,13 @@ declare module 'hologit' {
 
         getKind(): 'hololens';
         getConfigPath(): string;
-        async buildInputTree(inputRoot?: TreeObject): Promise<TreeObject>;
-        async buildSpec(inputTree: TreeObject): Promise<{
+        buildInputTree(inputRoot?: TreeObject): Promise<TreeObject>;
+        buildSpec(inputTree: TreeObject): Promise<{
             hash: string;
             ref: string;
             data: any;
         }>;
-        async executeSpec(specHash: string, options: {
+        executeSpec(specHash: string, options: {
             refresh?: boolean;
             save?: boolean;
             repo?: Repo | null;
@@ -209,7 +242,7 @@ declare module 'hologit' {
             cacheTo?: string | null;
         }): Promise<string>;
 
-        static async executeSpec(specHash: string, options: {
+        static executeSpec(specHash: string, options: {
             refresh?: boolean;
             save?: boolean;
             repo?: Repo | null;
@@ -226,18 +259,18 @@ declare module 'hologit' {
         getWorkspace(): Workspace;
         getKind(): 'holospace';
         getConfigPath(): string;
-        async writeWorkingChanges(): Promise<void>;
+        writeWorkingChanges(): Promise<void>;
         getBranch(name: string): Branch;
-        async getBranches(): Promise<Map<string, Branch>>;
+        getBranches(): Promise<Map<string, Branch>>;
         getSource(name: string): Source;
-        async getSources(): Promise<Map<string, Source>>;
-        async getLayers(): Promise<Map<string, Map<string, Mapping>>>;
+        getSources(): Promise<Map<string, Source>>;
+        getLayers(): Promise<Map<string, Map<string, Mapping>>>;
         getLens(name: string): Lens;
-        async getLenses(): Promise<Map<string, Lens>>;
+        getLenses(): Promise<Map<string, Lens>>;
     }
 
     export class Repo {
-        static async getFromEnvironment(options?: { ref?: string; working?: boolean }): Promise<Repo>;
+        static getFromEnvironment(options?: { ref?: string; working?: boolean }): Promise<Repo>;
 
         constructor(options: RepoOptions);
 
@@ -245,27 +278,27 @@ declare module 'hologit' {
         ref: string;
         workTree: string | null;
 
-        async getWorkspace(): Promise<Workspace>;
-        async createWorkspaceFromRef(ref: string): Promise<Workspace>;
-        async createWorkspaceFromTreeHash(hash: string): Promise<Workspace>;
-        async getGit(): Promise<GitClient>;
-        async resolveRef(ref?: string | null): Promise<string | null>;
+        getWorkspace(): Promise<Workspace>;
+        createWorkspaceFromRef(ref: string): Promise<Workspace>;
+        createWorkspaceFromTreeHash(hash: string): Promise<Workspace>;
+        getGit(): Promise<GitClient>;
+        resolveRef(ref?: string | null): Promise<string | null>;
         createBlob(options: GitObjectOptions): BlobObject;
-        async writeBlob(content: string): Promise<BlobObject>;
-        async writeBlobFromFile(filePath: string): Promise<BlobObject>;
+        writeBlob(content: string): Promise<BlobObject>;
+        writeBlobFromFile(filePath: string): Promise<BlobObject>;
         createTree(options?: { hash?: string; parent?: TreeObject | null }): TreeObject;
-        async createTreeFromRef(ref: string): Promise<TreeObject>;
+        createTreeFromRef(ref: string): Promise<TreeObject>;
         createCommit(options: GitObjectOptions): CommitObject;
-        async hasCommit(commit: string): Promise<boolean>;
-        async hashWorkTree(): Promise<string>;
-        async watch(options: { callback: (treeHash: string, commitHash?: string) => void }): Promise<{
+        hasCommit(commit: string): Promise<boolean>;
+        hashWorkTree(): Promise<string>;
+        watch(options: { callback: (treeHash: string, commitHash?: string) => void }): Promise<{
             watching: Promise<void>;
             cancel: () => void;
         }>;
     }
 
     export class Projection {
-        static async projectBranch(branch: Branch, options?: ProjectionOptions): Promise<string>;
+        static projectBranch(branch: Branch, options?: ProjectionOptions): Promise<string>;
 
         constructor(options: { branch: Branch });
 
@@ -273,39 +306,39 @@ declare module 'hologit' {
         workspace: Workspace;
         output: Workspace;
 
-        async composite(options: {
+        composite(options: {
             fetch?: boolean | string[];
             cacheFrom?: string | null;
             cacheTo?: string | null;
         }): Promise<void>;
-        async lens(options: {
+        lens(options: {
             cacheFrom?: string | null;
             cacheTo?: string | null;
         }): Promise<void>;
-        async commit(ref: string, options?: {
+        commit(ref: string, options?: {
             parentCommit?: string | null;
             commitMessage?: string | null;
         }): Promise<string>;
     }
 
     export class Studio {
-        static async cleanup(): Promise<void>;
-        static async getHab(): Promise<any>;
-        static async getDocker(): Promise<Docker>;
-        static async isEnvironmentStudio(): Promise<boolean>;
-        static async get(gitDir: string): Promise<Studio>;
+        static cleanup(): Promise<void>;
+        static getHab(): Promise<any>;
+        static isEnvironmentStudio(): Promise<boolean>;
+        static get(gitDir: string): Promise<Studio>;
+        static execDocker(args: string[], options?: DockerExecOptions): Promise<string>;
 
-        constructor(options: { gitDir: string; container: any });
+        constructor(options: { gitDir: string; container: StudioContainer });
 
-        container: any;
+        container: StudioContainer;
         gitDir: string;
 
         isLocal(): boolean;
-        async habExec(...command: any[]): Promise<string>;
-        async habPkgExec(pkg: string, bin: string, ...args: any[]): Promise<string>;
-        async holoExec(...command: any[]): Promise<string>;
-        async holoLensExec(spec: string): Promise<string>;
-        async getPackage(query: string, options?: { install?: boolean }): Promise<string | null>;
+        habExec(...command: any[]): Promise<string>;
+        habPkgExec(pkg: string, bin: string, ...args: any[]): Promise<string>;
+        holoExec(...command: any[]): Promise<string>;
+        holoLensExec(spec: string): Promise<string>;
+        getPackage(query: string, options?: { install?: boolean }): Promise<string | null>;
     }
 
     export class Mapping extends Configurable {
@@ -319,13 +352,17 @@ declare module 'hologit' {
         getConfigPath(): string;
     }
 
-    export class SpecObject extends BlobObject {
-        static async write(repo: Repo, kind: string, data: any): Promise<{
+    export class SpecObject {
+        constructor(repo: Repo, options: GitObjectOptions);
+
+        repo: Repo;
+        hash: string;
+        isSpec: boolean;
+
+        static write(repo: Repo, kind: string, data: any): Promise<{
             hash: string;
             ref: string;
         }>;
         static buildRef(kind: string, hash: string): string;
-
-        isSpec: boolean;
     }
 }

package/lib/Lens.js

@@ -44,12 +44,10 @@ class Lens extends Configurable {
         const config = await super.getConfig();
 
         // process lens configuration
-        if (!config.package) {
-            throw new Error(`hololens has no package defined: ${this.name}`);
+        if (config.package) {
+            config.command = config.command || 'lens-tree {{ input }}';
         }
 
-        config.command = config.command || 'lens-tree {{ input }}';
-
         if (config.before) {
             config.before =
                 typeof config.before == 'string'
@@ -103,7 +101,62 @@ class Lens extends Configurable {
     async buildSpec (inputTree) {
         const config = await this.getCachedConfig();
 
+        if (config.container) {
+            return this.buildSpecForContainer(inputTree, config);
+        } else if (config.package) {
+            return this.buildSpecForHabitatPackage(inputTree, config);
+        } else {
+            throw new Error(`hololens has no package or container defined: ${this.name}`);
+        }
+    }
+
+    async buildSpecForContainer (inputTree, config) {
+        const { container: containerQuery } = config;
+
+        // check if image exists locally first
+        let imageHash;
+        try {
+            const inspectOutput = await Studio.execDocker(['inspect', containerQuery]);
+            const imageInfo = JSON.parse(inspectOutput)[0];
+            imageHash = imageInfo.Id;
+            logger.info(`found local image: ${containerQuery}@${imageHash}`);
+        } catch (err) {
+            // image doesn't exist locally or can't be inspected, try pulling
+            logger.info(`pulling image: ${containerQuery}`);
+
+            try {
+                await Studio.execDocker(['pull', containerQuery], { $relayStdout: true });
+                const inspectOutput = await Studio.execDocker(['inspect', containerQuery]);
+                const imageInfo = JSON.parse(inspectOutput)[0];
+                imageHash = imageInfo.Id;
+            } catch (err) {
+                throw new Error(`failed to pull container image ${containerQuery}: ${err.message}`);
+            }
+        }
+
+        if (!imageHash) {
+            throw new Error(`failed to get hash for container image ${containerQuery}`);
+        }
+
+        // build spec
+        const data = {
+            ...config,
+            container: `${containerQuery.replace(/:.*$/, '')}@${imageHash}`,
+            input: await inputTree.write(),
+            output: null,
+            before: null,
+            after: null
+        };
 
+        // write spec and return packet
+        return {
+            ...await SpecObject.write(this.workspace.getRepo(), 'lens', data),
+            data,
+            type: 'container'
+        };
+    }
+
+    async buildSpecForHabitatPackage (inputTree, config) {
         // determine current package version
         const { package: packageQuery } = config;
         const [pkgOrigin, pkgName, pkgVersion, pkgBuild] = packageQuery.split('/');
@@ -163,14 +216,6 @@ class Lens extends Configurable {
         }
 
 
-        // old studio method that might be useful as fallback/debug option
-        // const setupOutput = await studio.exec('hab', 'pkg', 'install', 'core/hab-plan-build');
-        // const originOutput = await studio.exec('hab', 'origin', 'key', 'generate', 'holo');
-        // const buildOutput = await studio.habPkgExec('core/hab-plan-build', 'hab-plan-build', '/src/lenses/compass');
-        // const studio = await Studio.get(this.workspace.getRepo().gitDir);
-        // let packageIdent = await studio.getPackage(packageQuery);
-
-
         // build spec
         const data = {
             ...config,
@@ -185,21 +230,21 @@ class Lens extends Configurable {
         // write spec and return packet
         return {
             ...await SpecObject.write(this.workspace.getRepo(), 'lens', data),
-            data
+            data,
+            type: 'habitat'
         };
     }
 
-    async executeSpec (specHash, options) {
-        return Lens.executeSpec(specHash, {...options, repo: this.workspace.getRepo()});
+    async executeSpec (specType, specHash, options) {
+        return Lens.executeSpec(specType, specHash, {...options, repo: this.workspace.getRepo()});
     }
 
-    static async executeSpec (specHash, { refresh=false, save=true, repo=null, cacheFrom=null, cacheTo=null }) {
+    static async executeSpec (specType, specHash, options) {
+        const { refresh=false, cacheFrom=null, cacheTo=null, save=true } = options;
 
-        // load holorepo
-        if (!repo) {
-            repo = await Repo.getFromEnvironment();
-        }
 
+        // load holorepo
+        const repo = options.repo || await Repo.getFromEnvironment();
         const git = await repo.getGit();
 
 
@@ -228,9 +273,161 @@ class Lens extends Configurable {
         }
 
 
-        // ensure the rest runs inside a studio environment
+        // execute lens in container or with habitat package:
+        let lensedTreeHash;
+        if (specType == 'container') {
+            lensedTreeHash = await Lens.executeSpecForContainer(repo, specHash);
+        } else if (specType == 'habitat') {
+            lensedTreeHash = await Lens.executeSpecForHabitatPackage(repo, specHash);
+        }
+
+        // save ref to accelerate next projection
+        if (save) {
+            await git.updateRef(specRef, lensedTreeHash);
+
+            if (cacheTo) {
+                await _cacheResultTo(repo, specRef, cacheTo);
+            }
+        }
+
+        return lensedTreeHash;
+    }
+
+    static async executeSpecForContainer (repo, specHash) {
+        const git = await repo.getGit();
+
+        // read and parse spec file
+        const specToml = await git.catFile({ p: true }, specHash);
+        const {
+            holospec: {
+                lens: spec
+            }
+        } = TOML.parse(specToml);
+
+        // write commit with input tree and spec content
+        const commitHash = await git.commitTree(spec.input, {
+            p: [],
+            m: specToml
+        });
+
+        // extract repository and hash from container string
+        const containerMatch = spec.container.match(/^.+@sha256:([a-f0-9]{64})$/);
+        if (!containerMatch) {
+            throw new Error(`Invalid container format: ${spec.container}`);
+        }
+        const [, sha256Hash] = containerMatch;
+
+        // create and start container
+        const persistentDebugContainer = process.env.HOLO_DEBUG_PERSIST_CONTAINER;
+        let containerId;
+        try {
+            if (persistentDebugContainer) {
+                try {
+                    const containerInfo = await Studio.execDocker(['inspect', persistentDebugContainer]);
+                    const containerState = JSON.parse(containerInfo)[0].State;
+
+                    if (containerState.Running) {
+                        logger.info(`Found running debug container: ${persistentDebugContainer}`);
+                        containerId = persistentDebugContainer;
+                    }
+                } catch (error) {
+                    containerId = null;
+                }
+            }
+
+            // create container
+            if (!containerId) {
+                containerId = await Studio.execDocker([
+                    'create',
+                    '-p', '9000:9000',
+                    ...(persistentDebugContainer ? ['--name', persistentDebugContainer] : []),
+                    ...(process.env.DEBUG ? ['-e', 'DEBUG=1'] : []),
+                    sha256Hash
+                ]);
+                containerId = containerId.trim();
+
+                logger.info('starting container');
+                await Studio.execDocker(['start', containerId]);
+            }
+
+            // wait for port 9000 to be available
+            let attempts = 0;
+            const maxAttempts = 30;
+            const waitTime = 1000; // 1 second
+
+            while (attempts < maxAttempts) {
+                try {
+                    const containerInfo = await Studio.execDocker(['inspect', containerId]);
+                    const containerState = JSON.parse(containerInfo)[0].State;
+
+                    if (containerState.Running) {
+                        // check if port 9000 is listening
+                        try {
+                            await Studio.execDocker([
+                                'exec',
+                                containerId,
+                                'nc',
+                                '-z',
+                                'localhost',
+                                '9000'
+                            ]);
+                            break;
+                        } catch (err) {
+                            // ignore error and continue waiting
+                        }
+                    }
+                } catch (err) {
+                    // ignore error and continue waiting
+                }
+
+                await new Promise(resolve => setTimeout(resolve, waitTime));
+                attempts++;
+            }
+
+            if (attempts >= maxAttempts) {
+                throw new Error('Timeout waiting for git server to be ready');
+            }
+
+            // push commit to git server
+            logger.info(`pushing and executing job: ${commitHash}`);
+            await git.push(`http://localhost:9000/`, `${commitHash}:refs/heads/lens-input`, {
+                force: true,
+                $wait: true,
+                $onStderr: (line) => process.stderr.write(`\x1b[90m${line}\x1b[0m\n`)
+            });
+
+            // fetch and verify output commit
+            const outputRef = `refs/lens-jobs/${specHash}`;
+            logger.info('fetching result');
+            await git.fetch('http://localhost:9000/', `+refs/heads/lens-input:${outputRef}`);
+
+            // verify the output commit's parent matches our input commit
+            const outputParent = await git.revParse(`${outputRef}^`);
+            if (outputParent !== commitHash) {
+                throw new Error(`Output commit parent ${outputParent} does not match input commit ${commitHash}`);
+            }
+
+            return await git.getTreeHash(outputRef);
+
+        } finally {
+            // cleanup
+            if (containerId && !persistentDebugContainer) {
+                try {
+                    await Studio.execDocker(['stop', containerId]);
+                    await Studio.execDocker(['rm', containerId]);
+                } catch (err) {
+                    logger.warn(`Failed to cleanup container: ${err.message}`);
+                }
+            }
+        }
+    }
+
+    static async executeSpecForHabitatPackage (repo, specHash) {
+        const git = await repo.getGit();
+
         let lensedTreeHash;
 
+        // ensure the rest runs inside a studio environment
         if (!await Studio.isEnvironmentStudio()) {
             const studio = await Studio.get(repo.gitDir);
             lensedTreeHash = await studio.holoLensExec(specHash);
@@ -307,16 +504,6 @@ class Lens extends Configurable {
         }
 
 
-        // save ref to accelerate next projection
-        if (save) {
-            await git.updateRef(specRef, lensedTreeHash);
-
-            if (cacheTo) {
-                await _cacheResultTo(repo, specRef, cacheTo);
-            }
-        }
-
-
         // return tree hash
         return lensedTreeHash;
     }

package/lib/Projection.js

@@ -162,11 +162,11 @@ class Projection {
 
             // build tree of matching files to input to lens
             logger.info(`building input tree for lens ${lens.name} from ${inputRoot == '.' ? '' : (path.join(inputRoot, '.')+'/')}{${inputFiles}}`);
-            const { hash: specHash } = await lens.buildSpec(await lens.buildInputTree(this.output.root));
+            const { hash: specHash, type: specType } = await lens.buildSpec(await lens.buildInputTree(this.output.root));
 
 
             // check for existing output tree
-            const outputTreeHash = await lens.executeSpec(specHash, { cacheFrom, cacheTo });
+            const outputTreeHash = await lens.executeSpec(specType, specHash, { cacheFrom, cacheTo });
 
 
             // verify output