cloudron 5.11.3 → 5.11.5

package/bin/cloudron

@@ -22,12 +22,6 @@ if (!semver.satisfies(process.version, require('../package.json').engines.node))
 }
 
 const program = new Command();
-
-function collectArgs(value, collected) {
-    collected.push(value);
-    return collected;
-}
-
 program.version(version);
 
 // global options
@@ -39,6 +33,7 @@ program.option('--server <server>', 'Cloudron domain')
 // these are separate binaries since global options are not applicable
 program.command('appstore', 'Cloudron appstore commands');
 program.command('repo', 'Cloudron repo commands');
+program.command('build', 'Cloudron build commands');
 
 const backupCommand = program.command('backup')
     .description('App backup commands');
@@ -83,21 +78,6 @@ program.command('completion')
     .description('Shows completion for your shell')
     .action(completion);
 
-// should probably move to separate binary since globals don't apply
-program.command('build')
-    .description('Build an app')
-    .option('--build-arg <namevalue>', 'Build arg passed to docker. Can be used multiple times', collectArgs, [])
-    .option('--build-service-token <token>', 'Build service token')
-    .option('-f, --file <dockerfile>', 'Name of the Dockerfile')
-    .option('--set-repository [repository url]', 'Change the repository')
-    .option('--set-build-service [buildservice url]', 'Set build service app URL')
-    .option('--local', 'Build docker images locally')
-    .option('--no-cache', 'Do not use cache')
-    .option('--no-push', 'Do not push built image to registry')
-    .option('--raw', 'Raw output build log')
-    .option('--tag <docker image tag>', 'Docker image tag. Note that this does not include the repository name')
-    .action(buildActions.build);
-
 program.command('cancel')
     .description('Cancels any active or pending app task')
     .option('--app <id/location>', 'App id or location')

package/bin/cloudron-build

@@ -0,0 +1,55 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const { program } = require('commander'),
+    buildActions = require('../src/build-actions.js');
+
+program.version(require('../package.json').version);
+
+function collectArgs(value, collected) {
+    collected.push(value);
+    return collected;
+}
+
+// global options
+program.option('--server <server>', 'Cloudron domain')
+    .option('--token, --build-service-token <token>', 'Build service token')
+    .option('--url, --set-build-service [buildservice url]', 'Set build service URL. This build service is automatically used for future calls from this project');
+
+program.command('build', { isDefault: true })
+    .description('Build an app. This is the default subcommand')
+    .option('--build-arg <namevalue>', 'Build arg passed to docker. Can be used multiple times', collectArgs, [])
+    .option('-f, --file <dockerfile>', 'Name of the Dockerfile')
+    .option('--set-repository [repository url]', 'Change the repository. This url is stored for future builds for this project. e.g registry/username/projectname')
+    .option('--local', 'Build docker images locally')
+    .option('--no-cache', 'Do not use cache')
+    .option('--no-push', 'Do not push built image to registry')
+    .option('--raw', 'Raw output build log')
+    .option('--tag <docker image tag>', 'Docker image tag. Note that this does not include the repository name')
+    .action(buildActions.build);
+
+program.command('login')
+    .description('Login to the build service')
+    .option('-t, --token <token>', 'Build service token')
+    .action(buildActions.login);
+
+program.command('logs')
+    .description('Build logs. This works only when using the Build Service')
+    .option('--id <buildid>', 'Build ID')
+    .option('--raw', 'Raw output build log')
+    .action(buildActions.logs);
+
+program.command('push')
+    .description('Push the build image')
+    .option('--id <buildid>', 'Build ID')
+    .option('--repository [repository url]', 'Set repository to push to. e.g registry/username/projectname')
+    .option('--tag <docker image tag>', 'Docker image tag. Note that this does not include the repository name')
+    .action(buildActions.push);
+
+program.command('status')
+    .description('Build status. This works only when using the Build Service')
+    .option('--id <buildid>', 'Build ID')
+    .action(buildActions.status);
+
+program.parse(process.argv);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cloudron",
-  "version": "5.11.3",
+  "version": "5.11.5",
   "license": "MIT",
   "description": "Cloudron Commandline Tool",
   "main": "main.js",
@@ -25,7 +25,6 @@
     "ejs": "^3.1.10",
     "eventsource": "^2.0.2",
     "micromatch": "^4.0.7",
-    "once": "^1.4.0",
     "open": "^8.4.0",
     "progress": "^2.0.3",
     "progress-stream": "^2.0.0",

package/src/build-actions.js

@@ -1,7 +1,11 @@
 'use strict';
 
 exports = module.exports = {
-    build
+    login,
+    build,
+    logs,
+    status,
+    push
 };
 
 const assert = require('assert'),
@@ -11,7 +15,6 @@ const assert = require('assert'),
     execSync = require('child_process').execSync,
     exit = require('./helper.js').exit,
     fs = require('fs'),
-    once = require('once'),
     helper = require('./helper.js'),
     manifestFormat = require('cloudron-manifestformat'),
     micromatch = require('micromatch'),
@@ -20,78 +23,80 @@ const assert = require('assert'),
     os = require('os'),
     path = require('path'),
     safe = require('safetydance'),
+    stream = require('stream/promises'),
     tar = require('tar-fs'),
-    url = require('url'),
-    util = require('util');
-
-function superagentEndBuildService(requestFactory, callback) {
-    assert.strictEqual(typeof requestFactory, 'function');
-    assert.strictEqual(typeof callback, 'function');
-
-    requestFactory().end(function (error, result) {
-        if (error && !error.response) return callback(error);
-        if (result.statusCode === 400) return authenticateBuildService({ error: true }, superagentEndBuildService.bind(null, requestFactory, callback));
-        if (result.statusCode === 401) return authenticateBuildService({ error: true }, superagentEndBuildService.bind(null, requestFactory, callback));
-        if (result.statusCode === 403) return callback(new Error(result.type === 'application/javascript' ? JSON.stringify(result.body) : result.text));
-        callback(error, result);
-    });
+    url = require('url');
+
+function requestError(response) {
+    if (response.statusCode === 401 || response.statusCode === 403) return 'Invalid token. Use cloudron build login again.';
+
+    return `${response.statusCode} message: ${response.body?.message || null}`;
 }
 
-function authenticateBuildService(options, callback) {
-    assert.strictEqual(typeof options, 'object');
-    assert.strictEqual(typeof callback, 'function');
+// analyzes options and merges with any existing build service config
+function getBuildServiceConfig(options) {
+    const buildService = config.getBuildServiceConfig();
+    if (!buildService.type) buildService.type = 'local'; // default
 
-    var buildServiceConfig = config.getBuildServiceConfig();
+    if (options.local) {
+        buildService.type = 'local';
+    } else if (options.setBuildService) { // stash for future use
+        buildService.token = null;
+        buildService.url = null;
+        buildService.type = 'remote';
 
-    if (!options.hideBanner) console.log('Build Service login' + ` (${buildServiceConfig.url}):`);
+        let url;
+        if (typeof options.setBuildService === 'string') {
+            url = options.setBuildService;
+        } else {
+            url = readlineSync.question('Enter build service URL: ', { });
+        }
 
-    var username = options.username || readlineSync.question('Username: ', {});
-    var password = options.password || readlineSync.question('Password: ', { noEchoBack: true });
+        if (url.indexOf('://') === -1) url = `https://${url}`;
+        buildService.url = url;
+    }
 
-    // reset token
-    buildServiceConfig.token = null;
-    config.setBuildServiceConfig(buildServiceConfig);
+    if (options.buildServiceToken) buildService.token = options.buildServiceToken;
 
-    superagent.post(`${buildServiceConfig.url}/api/v1/login`).send({ username: username, password: password }).end(function (error, result) {
-        if (error && !error.response) exit(error);
+    config.setBuildServiceConfig(buildService);
 
-        if (result.statusCode !== 200 && result.statusCode !== 201) {
-            console.log('Login failed.');
+    return buildService;
+}
 
-            options.hideBanner = true;
-            options.username = '';
-            options.password = '';
+async function login(options) {
+    assert.strictEqual(typeof options, 'object');
 
-            return authenticateBuildService(options, callback);
-        }
+    const buildServiceConfig = getBuildServiceConfig(options);
 
-        buildServiceConfig.token = result.body.accessToken;
-        config.setBuildServiceConfig(buildServiceConfig);
+    console.log('Build Service login' + ` (${buildServiceConfig.url}):`);
 
-        console.log('Login successful.');
+    const token = options.buildServiceToken || readlineSync.question('Token: ', {});
 
-        if (typeof callback === 'function') callback();
-    });
+    const response = await superagent.get(`${buildServiceConfig.url}/api/v1/profile`).query({ accessToken: token }).ok(() => true);
+    if (response.statusCode === 401 || response.statusCode === 403) return exit(`Authentication error: ${requestError(response)}`);
+    if (response.statusCode !== 200) return exit(`Unexpected response: ${requestError(response)}`);
+
+    buildServiceConfig.token = token;
+    config.setBuildServiceConfig(buildServiceConfig);
+
+    console.log('Login successful.');
 }
 
-function followBuildLog(buildId, raw, callback) {
+async function followBuildLog(buildId, raw) {
     assert.strictEqual(typeof buildId, 'string');
     assert.strictEqual(typeof raw, 'boolean');
 
-    // ensure callback is only ever called once
-    callback = once(callback);
-
     // EventSource always requires http
     let tmp = url.parse(config.getBuildServiceConfig().url);
     if (tmp.protocol !== 'https:' && tmp.protocol !== 'http:') tmp = url.parse('http://' + config.getBuildServiceConfig().url);
 
-    var es = new EventSource(`${tmp.href}api/v1/builds/${buildId}/logstream?accessToken=${config.getBuildServiceConfig().token}`);
-    var prevId = null, prevWasStatus = false;
+    const es = new EventSource(`${tmp.href}api/v1/builds/${buildId}/logstream?accessToken=${config.getBuildServiceConfig().token}`);
+    let prevId = null, prevWasStatus = false;
 
     es.on('message', function (e) {
         if (raw) return console.dir(e);
 
-        var data = safe.JSON.parse(e.data);
+        const data = safe.JSON.parse(e.data);
         if (!data) return; // this is a bug in docker or our build server
 
         if (data.status) { // image push log
@@ -131,25 +136,39 @@ function followBuildLog(buildId, raw, callback) {
             console.error(data.error);
         }
     });
-    es.on('error', function (error) {
-        if (raw) console.dir(error);
 
-        // We sometimes get { type: 'error' } from es module when the server closes the socket. not clear why
-        if (error && !error.status && error.type === 'error') error = null;
+    let didConnect = false;
+    es.once('open', () => didConnect = true);
+
+    return new Promise((resolve, reject) => {
+        es.once('error', function (error) { // server close or network error or some interruption
+            if (raw) console.dir(error);
 
-        callback(error && error.status ? error : null); // eventsource module really needs to give us better errors
+            es.close();
+            if (didConnect) resolve(); else reject(new Error('Failed to connect'));
+        });
     });
 }
 
+async function getStatus(buildId) {
+    const buildServiceConfig = config.getBuildServiceConfig();
+
+    const response2 = await superagent.get(`${buildServiceConfig.url}/api/v1/builds/${buildId}`)
+        .query({ accessToken: buildServiceConfig.token })
+        .ok(() => true);
+    if (response2.statusCode !== 200) throw new Error(`Failed to get status: ${requestError(response2)}`);
+    return response2.body.status;
+}
+
 function dockerignoreMatcher(dockerignorePath) {
-    var patterns = [];
+    let patterns = [];
 
     if (fs.existsSync(dockerignorePath)) {
         patterns = fs.readFileSync(dockerignorePath, 'utf8').split('\n');
 
         patterns = patterns.filter(function (line) { return line[0] !== '#'; });
         patterns = patterns.map(function (line) {
-            var l = line.trim();
+            let l = line.trim();
 
             while (l[0] === '/') l = l.slice(1);
             while (l[l.length-1] === '/') l = l.slice(0, -1);
@@ -164,7 +183,7 @@ function dockerignoreMatcher(dockerignorePath) {
     };
 }
 
-function buildLocal(manifest, sourceDir, appConfig, options) {
+async function buildLocal(manifest, sourceDir, appConfig, options) {
     let tag;
     if (options.tag) {
         tag = options.tag;
@@ -180,7 +199,7 @@ function buildLocal(manifest, sourceDir, appConfig, options) {
     console.log('Building locally as %s', dockerImage);
     console.log();
 
-    let buildArgsCmdLine = options.buildArg.map(function (a) { return `--build-arg "${a}"`; }).join(' ');
+    const buildArgsCmdLine = options.buildArg.map(function (a) { return `--build-arg "${a}"`; }).join(' ');
 
     let dockerfile = 'Dockerfile';
     if (options.file) dockerfile = options.file;
@@ -196,9 +215,9 @@ function buildLocal(manifest, sourceDir, appConfig, options) {
         if (safe.error) exit('Failed to push image (are you logged in? if not, use "docker login")');
     }
 
-    let result = safe.child_process.execSync(`docker inspect --format="{{index .RepoDigests 0}}" ${dockerImage}`, { encoding: 'utf8' });
+    const result = safe.child_process.execSync(`docker inspect --format="{{index .RepoDigests 0}}" ${dockerImage}`, { encoding: 'utf8' });
     if (safe.error) exit('Failed to inspect image');
-    let match = /.*@sha256:(.*)/.exec(result.trim());
+    const match = /.*@sha256:(.*)/.exec(result.trim());
     if (!match) exit('Failed to detect sha256');
 
     appConfig.dockerImage = `${dockerImage}`;
@@ -206,7 +225,7 @@ function buildLocal(manifest, sourceDir, appConfig, options) {
     config.setAppConfig(sourceDir, appConfig);
 }
 
-function buildRemote(manifest, sourceDir, appConfig, options) {
+async function buildRemote(manifest, sourceDir, appConfig, options) {
     console.log('Using build service', config.getBuildServiceConfig().url);
 
     let tag;
@@ -223,21 +242,21 @@ function buildRemote(manifest, sourceDir, appConfig, options) {
 
     console.log('Building %s', dockerImage);
 
-    var sourceArchiveFilePath = path.join(os.tmpdir(), path.basename(sourceDir) + '.tar.gz');
-    var dockerignoreFilePath = path.join(sourceDir, '.dockerignore');
-    var ignoreMatcher = dockerignoreMatcher(dockerignoreFilePath);
+    const sourceArchiveFilePath = path.join(os.tmpdir(), path.basename(sourceDir) + '.tar.gz');
+    const dockerignoreFilePath = path.join(sourceDir, '.dockerignore');
+    const ignoreMatcher = dockerignoreMatcher(dockerignoreFilePath);
 
     console.log('Uploading source tarball...');
 
-    var stream = tar.pack(sourceDir, {
+    const tarStream = tar.pack(sourceDir, {
         ignore: function (name) {
             return ignoreMatcher(name.slice(sourceDir.length + 1)); // make name as relative path
         }
-    }).pipe(fs.createWriteStream(sourceArchiveFilePath));
-
-    stream.on('error', function (error) {
-        exit('Failed to create application source archive: ' + error);
     });
+    const sourceArchiveStream = fs.createWriteStream(sourceArchiveFilePath);
+
+    const [tarError] = await safe(stream.pipeline(tarStream, sourceArchiveStream));
+    if (tarError) return exit(`Could not tar: ${tarError.message}`);
 
     let dockerfile = 'Dockerfile';
     if (options.file) dockerfile = options.file;
@@ -251,49 +270,40 @@ function buildRemote(manifest, sourceDir, appConfig, options) {
         buildArgsObject[key] = value;
     });
 
-    stream.on('finish', function () {
-        superagentEndBuildService(function () {
-            let buildServiceConfig = config.getBuildServiceConfig();
-            return superagent.post(`${buildServiceConfig.url}/api/v1/builds`)
-                .query({ accessToken: buildServiceConfig.token, noCache: !options.cache, dockerfile: dockerfile, noPush: !options.push })
-                .field('dockerImageRepo', appConfig.repository)
-                .field('dockerImageTag', tag)
-                .field('buildArgs', JSON.stringify(buildArgsObject))
-                .attach('sourceArchive', sourceArchiveFilePath);
-        }, function (error, result) {
-            if (error && !error.response) return exit(util.format('Failed to build app: %s', error.message));
-            if (result.statusCode === 413) exit('Failed to build app. The app source is too large.\nPlease adjust your .dockerignore file to only include neccessary files.');
-            if (result.statusCode !== 201) exit(util.format('Failed to build app (statusCode %s): \n%s', result.statusCode, result.body && result.body.message ? result.body.message : result.text));
-
-            var buildId = result.body.id;
-
-            followBuildLog(buildId, false, function (error) {
-                if (error) return console.error(error);
-
-                superagentEndBuildService(function () {
-                    let buildServiceConfig = config.getBuildServiceConfig();
-                    return superagent.get(`${buildServiceConfig.url}/api/v1/builds/${buildId}`)
-                        .query({ accessToken: buildServiceConfig.token });
-                }, function (error, result) {
-                    if (error && !error.response) return exit(util.format('Failed to build app: %s', error.message));
-                    if (result.statusCode !== 200) exit(util.format('Failed to build app (statusCode %s): \n%s', result.statusCode, result.body && result.body.message ? result.body.message : result.text));
-                    if (result.body.status !== 'success') exit('Failed to build app. See log output above.');
-
-                    appConfig.dockerImage = dockerImage;
-                    // appConfig.dockerImageSha256 = match[1]; // stash this separately for now
-                    config.setAppConfig(sourceDir, appConfig);
-
-                    console.log(dockerImage);
-                    console.log('\nBuild successful');
-
-                    exit();
-                });
-            });
-        });
-    });
+    const buildServiceConfig = config.getBuildServiceConfig();
+    const response = await superagent.post(`${buildServiceConfig.url}/api/v1/builds`)
+        .query({ accessToken: buildServiceConfig.token, noCache: !options.cache, dockerfile: dockerfile, noPush: !options.push })
+        .field('dockerImageRepo', appConfig.repository)
+        .field('dockerImageTag', tag)
+        .field('buildArgs', JSON.stringify(buildArgsObject))
+        .attach('sourceArchive', sourceArchiveFilePath)
+        .ok(() => true);
+    if (response.statusCode === 413) return exit('Failed to build app. The app source is too large.\nPlease adjust your .dockerignore file to only include neccessary files.');
+    if (response.statusCode !== 201) return exit(`Failed to upload app for building: ${requestError(response)}`);
+
+    const buildId = response.body.id;
+    console.log(`BuildId: ${buildId}`);
+
+    const [logsError] = await safe(followBuildLog(buildId, !!options.raw));
+    if (logsError) console.log(`Failed to get logs: ${logsError.message}`);
+
+    const [statusError, status] = await safe(getStatus(buildId));
+    if (statusError) return exit(`Failed to get status: ${statusError.message}`);
+    if (status !== 'success') return exit('Failed to build app. See log output above.');
+
+    appConfig.dockerImage = dockerImage;
+    // appConfig.dockerImageSha256 = match[1]; // stash this separately for now
+    config.setAppConfig(sourceDir, appConfig);
+
+    console.log(`Docker image: ${dockerImage}`);
+    console.log('\nBuild successful');
+
+    exit();
 }
 
-function build(options) {
+async function build(localOptions, cmd) {
+    const options = cmd.optsWithGlobals();
+
     // try to find the manifest of this project
     const manifestFilePath = helper.locateManifest();
     if (!manifestFilePath) return exit('No CloudronManifest.json found');
@@ -301,35 +311,11 @@ function build(options) {
     const result = manifestFormat.parseFile(manifestFilePath);
     if (result.error) return exit('Error in CloudronManifest.json: ' + result.error.message);
 
-    let manifest = result.manifest;
+    const manifest = result.manifest;
     const sourceDir = path.dirname(manifestFilePath);
 
     const appConfig = config.getAppConfig(sourceDir);
-
-    let buildService = config.getBuildServiceConfig();
-    if (!buildService.type) buildService.type = 'local'; // default
-
-    if (options.local) {
-        buildService.type = 'local';
-    } else if (options.setBuildService) {
-        buildService.token = null;
-        buildService.url = null;
-        buildService.type = 'remote';
-
-        let url;
-        if (typeof options.setBuildService === 'string') {
-            url = options.setBuildService;
-        } else {
-            url = readlineSync.question('Enter build service URL: ', { });
-        }
-
-        if (url.indexOf('://') === -1) url = `https://${url}`;
-        buildService.url = url;
-    }
-
-    if (options.buildServiceToken) buildService.token = options.buildServiceToken;
-
-    config.setBuildServiceConfig(buildService);
+    const buildServiceConfig = getBuildServiceConfig(options);
 
     let repository = appConfig.repository;
     if (!repository || options.setRepository) {
@@ -345,11 +331,53 @@ function build(options) {
         config.setAppConfig(sourceDir, appConfig);
     }
 
-    if (buildService.type === 'local') {
-        buildLocal(manifest, sourceDir, appConfig, options);
-    } else if (buildService.type === 'remote' && buildService.url) {
-        buildRemote(manifest, sourceDir, appConfig, options);
+    if (buildServiceConfig.type === 'local') {
+        await buildLocal(manifest, sourceDir, appConfig, options);
+    } else if (buildServiceConfig.type === 'remote' && buildServiceConfig.url) {
+        await buildRemote(manifest, sourceDir, appConfig, options);
     } else {
         exit('Unknown build service type or missing build service url. Rerun with --reset-build-service');
     }
 }
+
+async function logs(localOptions, cmd) {
+    const options = cmd.optsWithGlobals();
+
+    if (!options.id) return exit('buildId is required');
+
+    const [logsError] = await safe(followBuildLog(options.id, !!options.raw));
+    if (logsError) console.log(`Failed to get logs: ${logsError.message}`);
+}
+
+async function status(localOptions, cmd) {
+    const options = cmd.optsWithGlobals();
+
+    if (!options.id) return exit('buildId is required');
+
+    const [statusError, status] = await safe(getStatus(options.id));
+    if (statusError) return exit(`Failed to get status: ${statusError.message}`);
+    console.log(status);
+}
+
+async function push(localOptions, cmd) {
+    const options = cmd.optsWithGlobals();
+
+    if (!options.id) return exit('buildId is required');
+    if (!options.repository) return exit('repository is required');
+    if (!options.tag) return exit('tag is required');
+
+    const buildServiceConfig = getBuildServiceConfig(options);
+
+    const response = await superagent.post(`${buildServiceConfig.url}/api/v1/builds/${options.id}/push`)
+        .query({ accessToken: buildServiceConfig.token })
+        .send({ dockerImageRepo: options.repository, dockerImageTag: options.tag })
+        .ok(() => true);
+    if (response.statusCode !== 201) return exit(`Failed to push: ${requestError(response)}`);
+
+    const [logsError] = await safe(followBuildLog(options.id, !!options.raw));
+    if (logsError) console.log(`Failed to get logs: ${logsError.message}`);
+
+    const [statusError, status] = await safe(getStatus(options.id));
+    if (statusError) return exit(`Failed to get status: ${statusError.message}`);
+    if (status !== 'success') return exit('Failed to push app. See log output above.');
+}