cloudron 7.0.5 → 7.1.0

package/CHANGELOG.md

@@ -49,3 +49,48 @@
 * Various changes to accomodate Cloudron 9 release API changes
 * add --siteId to cloudron backup to specify site id explicitly
 
+[7.0.0]
+* Port from CommonJS to ESM
+* Add initial OIDC login support
+* Add `cloudron versions` subcommand
+* `versions`: init with publisher info, add `--version` flag, add publishes by default
+* Add `cloudron build info` subcommand
+* Add `cloudron build clear` subcommand
+* Add `cloudron build logout` subcommand
+* Rework the build command
+* Support source tarball upload with install and update subcommands
+* Remove calling the /repair route in update subcommand
+* Use POST /api/v1/apps for installations
+* Remove unused superagent module
+* Do not print stack if app isn't found
+
+[7.0.1]
+* Fix `--no-wait` flag
+
+[7.0.2]
+* Rename `--set-repository` to `--repository`
+* Fix broken build command
+
+[7.0.3]
+* Implement `versionsUrl` for testing
+* Fix app detection to be based on appId
+* Save appId on update
+
+[7.0.4]
+* Rename `build clear` to `build reset`
+* `versions`: add fields in init
+* Set minBoxVersion
+* Print the build type
+
+[7.0.5]
+* Gzip the source tarball for upload
+* Print where the image is coming from
+
+[7.0.6]
+* Remove duplicate build info output
+* Do not error on EROFS
+
+[7.1.0]
+* Add `cloudron sync push` and `cloudron sync pull` subcommands
+* Merge all binaries into single `cloudron` command
+

package/bin/cloudron

@@ -1,10 +1,13 @@
 #!/usr/bin/env node
 
 import actions from '../src/actions.js';
+import appstoreActions from '../src/appstore-actions.js';
 import backupTools from '../src/backup-tools.js';
+import buildActions from '../src/build-actions.js';
 import completion from '../src/completion.js';
 import { Command } from 'commander';
 import semver from 'semver';
+import versionsActions from '../src/versions-actions.js';
 import pkg from '../package.json' with { type: 'json' };
 
 // ensure node version
@@ -23,13 +26,116 @@ program.option('--server <server>', 'Cloudron domain')
     .option('--accept-selfsigned', 'Accept self signed SSL certificate')
     .option('--no-wait', 'Do not wait for the operation to finish');
 
-// these are separate binaries since global options are not applicable
-program.command('appstore', 'Cloudron appstore commands');
-program.command('build', 'Cloudron build commands');
-program.command('versions', 'Cloudron versions commands');
+const appstoreCommand = program.command('appstore').description('Commands for publishing to the Appstore')
+    .option('--appstore-token <token>', 'AppStore token');
+
+appstoreCommand.command('login')
+    .description('Login to the appstore')
+    .option('-e, --email <email>', 'Email address')
+    .option('-p, --password <password>', 'Password (unsafe)')
+    .action(appstoreActions.login);
+
+appstoreCommand.command('logout')
+    .description('Logout from the appstore')
+    .action(appstoreActions.logout);
+
+appstoreCommand.command('info')
+    .description('List info of published app')
+    .option('--appstore-id <appid@version>', 'Appstore id and version')
+    .action(appstoreActions.info);
+
+appstoreCommand.command('approve')
+    .description('Approve a submitted app version')
+    .option('--appstore-id <appid@version>', 'Appstore id and version')
+    .option('--no-git-push', 'Do not attempt to push to git repo')
+    .action(appstoreActions.approve);
+
+appstoreCommand.command('notify')
+    .description('Notify forum about successful app submission')
+    .action(appstoreActions.notify);
+
+appstoreCommand.command('revoke')
+    .description('Revoke a published app version')
+    .option('--appstore-id <appid@version>', 'Appstore id and version')
+    .action(appstoreActions.revoke);
+
+appstoreCommand.command('submit')
+    .description('Submit app to the store for review')
+    .action(appstoreActions.submit);
+
+appstoreCommand.command('upload')
+    .description('Upload app to the store for testing')
+    .option('-i, --image <image>', 'Docker image')
+    .option('-f, --force', 'Update existing version')
+    .action(appstoreActions.upload);
+
+appstoreCommand.command('verify-manifest')
+    .description('Verify if manifest is ready to be upload/published')
+    .action(appstoreActions.verifyManifest);
+
+appstoreCommand.command('versions')
+    .alias('list')
+    .description('List published versions')
+    .option('--appstore-id <id>', 'Appstore id')
+    .option('--raw', 'Dump versions as json')
+    .action(appstoreActions.listVersions);
+
+const backupCommand = program.command('backup').description('App backup commands');
+
+function collectBuildArgs(value, collected) {
+    collected.push(value);
+    return collected;
+}
 
-const backupCommand = program.command('backup')
-    .description('App backup commands');
+const buildCommand = program.command('build').description('Build using the build service')
+    .option('--build-service-url <url>', 'Build service URL')
+    .option('--build-service-token <token>', 'Build service token');
+
+buildCommand.command('build', { isDefault: true })
+    .description('Build an app')
+    .option('--build-arg <namevalue>', 'Build arg passed to docker. Can be used multiple times', collectBuildArgs, [])
+    .option('-f, --file <dockerfile>', 'Name of the Dockerfile')
+    .option('--repository [repository url]', 'Change the repository. This url is stored for future builds for this project. e.g registry/username/projectname')
+    .option('--no-cache', 'Do not use cache')
+    .option('--no-push', 'Do not push built image to registry')
+    .option('--raw', 'Raw output build log')
+    .option('--tag <docker image tag>', 'Docker image tag. Note that this does not include the repository name')
+    .action(buildActions.build);
+
+buildCommand.command('reset')
+    .description('Reset build configuration for this directory')
+    .action(buildActions.reset);
+
+buildCommand.command('info')
+    .description('Print build information')
+    .action(buildActions.info);
+
+buildCommand.command('login')
+    .description('Login to the build service')
+    .action(buildActions.login);
+
+buildCommand.command('logout')
+    .description('Logout from the build service')
+    .action(buildActions.logout);
+
+buildCommand.command('logs')
+    .description('Build logs. This works only when using the Build Service')
+    .option('--id <buildid>', 'Build ID')
+    .option('--raw', 'Raw output build log')
+    .action(buildActions.logs);
+
+buildCommand.command('push')
+    .description('Push the build image')
+    .option('--id <buildid>', 'Build ID')
+    .option('--repository [repository url]', 'Set repository to push to. e.g registry/username/projectname')
+    .option('--tag <docker image tag>', 'Docker image tag. Note that this does not include the repository name')
+    .option('--image <docker image>', 'Docker image of the form registry/repo:tag')
+    .action(buildActions.push);
+
+buildCommand.command('status')
+    .description('Build status. This works only when using the Build Service')
+    .option('--id <buildid>', 'Build ID')
+    .action(buildActions.status);
 
 backupCommand.command('create')
     .description('Create new app backup')
@@ -223,6 +329,23 @@ program.command('push <local> <remote>')
         console.log();
     });
 
+const sync = program.command('sync')
+    .description('Sync files between local and remote (only transfers changed files)');
+
+sync.command('push <local> <remote>')
+    .description('Upload changed files to remote directory. A trailing slash on <local> syncs its contents; without it, the directory itself is placed inside <remote>.')
+    .option('--app <id/location>', 'App id or location')
+    .option('--delete', 'Delete remote files not present locally')
+    .option('--force', 'Remove files blocking directory creation at destination')
+    .action(actions.syncPush);
+
+sync.command('pull <remote> <local>')
+    .description('Download changed files from remote directory. A trailing slash on <remote> syncs its contents; without it, the directory itself is placed inside <local>.')
+    .option('--app <id/location>', 'App id or location')
+    .option('--delete', 'Delete local files not present on remote')
+    .option('--force', 'Remove files blocking directory creation at destination')
+    .action(actions.syncPull);
+
 program.command('repair')
     .description('Repair an installed application (re-configure)')
     .option('--app <id/location>', 'App id or location')
@@ -279,4 +402,29 @@ program.command('update')
     .option('--no-force', 'Match appstore id and manifest id before updating', true)
     .action(actions.update);
 
+const versionsCommand = program.command('versions').description('Commands for publishing community packages');
+
+versionsCommand.command('add')
+    .description('Add the current build to version file')
+    .option('--state <state>', 'Publish state (published or testing)')
+    .action(versionsActions.addOrUpdate);
+
+versionsCommand.command('init')
+    .description('Create versions file')
+    .action(versionsActions.init);
+
+versionsCommand.command('list')
+    .description('List existing versions')
+    .action(versionsActions.list);
+
+versionsCommand.command('revoke')
+    .description('Revoke the latest version')
+    .action(versionsActions.revoke);
+
+versionsCommand.command('update')
+    .description('Update existing version')
+    .option('--version <version>', 'Version to update')
+    .option('--state <state>', 'Publish state (published or testing)')
+    .action(versionsActions.addOrUpdate);
+
 program.parse(process.argv);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cloudron",
-  "version": "7.0.5",
+  "version": "7.1.0",
   "license": "MIT",
   "description": "Cloudron Commandline Tool",
   "type": "module",
@@ -17,26 +17,26 @@
   },
   "author": "Cloudron Developers <support@cloudron.io>",
   "dependencies": {
-    "@cloudron/manifest-format": "^6.0.2",
+    "@cloudron/manifest-format": "^6.1.0",
     "@cloudron/safetydance": "^3.0.1",
     "@cloudron/superagent": "^2.1.1",
     "commander": "^14.0.3",
     "debug": "^4.4.3",
     "easy-table": "^1.2.0",
-    "ejs": "^4.0.1",
+    "ejs": "^5.0.1",
     "eventsource": "^4.1.0",
     "micromatch": "^4.0.8",
     "open": "^11.0.0",
     "semver": "^7.7.4",
-    "tar-fs": "^3.1.1"
+    "tar-fs": "^3.1.2"
   },
   "engines": {
     "node": ">= 20.11.0"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
-    "eslint": "^10.0.0",
-    "globals": "^17.3.0",
+    "eslint": "^10.0.3",
+    "globals": "^17.4.0",
     "mocha": "^11.7.5"
   }
 }

package/src/actions.js

@@ -13,6 +13,7 @@ import * as readline from './readline.js';
 import safe from '@cloudron/safetydance';
 import { spawn } from 'child_process';
 import semver from 'semver';
+import { PassThrough } from 'stream';
 import stream from 'stream/promises';
 import superagent from '@cloudron/superagent';
 import Table from 'easy-table';
@@ -106,10 +107,14 @@ async function stopActiveTask(app, options) {
 function saveCwdAppId(appId, manifestFilePath) {
     if (!manifestFilePath) return;
 
-    const sourceDir = path.dirname(manifestFilePath);
-    const cwdConfig = config.getCwdConfig(sourceDir);
-    cwdConfig.appId = appId;
-    config.setCwdConfig(sourceDir, cwdConfig);
+    try {
+        const sourceDir = path.dirname(manifestFilePath);
+        const cwdConfig = config.getCwdConfig(sourceDir);
+        cwdConfig.appId = appId;
+        config.setCwdConfig(sourceDir, cwdConfig);
+    } catch (e) { // will happen when run on EROFS (set HOME to avoid)
+        console.log(`Warning: Could not save app id to config: ${e.message} . Try setting $HOME`);
+    }
 }
 
 // appId may be the appId or the location
@@ -1422,6 +1427,187 @@ function demuxStream(inputStream, stdout, stderr) {
     });
 }
 
+async function createExecSession(appId, args, { tty, lang }, options) {
+    const request = createRequest('POST', `/api/v1/apps/${appId}/exec`, options);
+    const response = await request.send({ cmd: args, tty, lang });
+    if (response.status !== 200) exit(`Failed to create exec: ${requestError(response)}`);
+    return response.body.id;
+}
+
+async function getExecSession(appId, execId, options) {
+    const response = await createRequest('GET', `/api/v1/apps/${appId}/exec/${execId}`, options);
+    if (response.status !== 200) exit(`Failed to get exec session: ${requestError(response)}`);
+    return response.body;
+}
+
+function runExecSession(appId, execId, { stdin, stdout, stderr, tty }, options) {
+    return new Promise((resolve, reject) => {
+        const { adminFqdn, token: reqToken, rejectUnauthorized } = requestOptions(options);
+
+        const searchParams = new URLSearchParams({
+            rows: stdout.rows || 24,
+            columns: stdout.columns || 80,
+            access_token: reqToken,
+            tty
+        });
+
+        const req = https.request({
+            hostname: adminFqdn,
+            path: `/api/v1/apps/${appId}/exec/${execId}/start?${searchParams.toString()}`,
+            method: 'GET',
+            headers: {
+                'Connection': 'Upgrade',
+                'Upgrade': 'tcp'
+            },
+            rejectUnauthorized
+        }, function handler(res) {
+            if (res.statusCode === 403) return reject(new Error('Unauthorized.'));
+            reject(new Error(`Could not upgrade connection to tcp. http status: ${res.statusCode}`));
+        });
+
+        req.on('upgrade', function (resThatShouldNotBeUsed, socket) {
+            socket.on('error', reject);
+
+            socket.setNoDelay(true);
+            socket.setKeepAlive(true);
+
+            if (tty) {
+                stdin.setRawMode(true);
+                stdin.pipe(socket, { end: false });
+                socket.pipe(stdout);
+                socket.on('end', () => resolve());
+            } else {
+                if (typeof stdin === 'function') stdin = stdin();
+                let socketEnded = false;
+
+                stdin.on('data', function (d) {
+                    if (socketEnded) return;
+                    var buf = Buffer.alloc(4);
+                    buf.writeUInt32BE(d.length, 0);
+                    socket.write(buf);
+                    socket.write(d);
+                });
+                stdin.on('end', function () {
+                    if (socketEnded) return;
+                    var buf = Buffer.alloc(4);
+                    buf.writeUInt32BE(0, 0);
+                    socket.write(buf);
+                });
+
+                demuxStream(socket, stdout, stderr);
+                socket.on('end', function () {
+                    socketEnded = true;
+                    if (typeof stdin.end === 'function') stdin.end();
+                    socket.end();
+
+                    if (stdout !== process.stdout && typeof stdout.end === 'function') {
+                        stdout.on('close', () => resolve());
+                        stdout.end();
+                        stdout.resume(); // drain readable side of Duplex/Transform streams so autoDestroy emits 'close'
+                    } else {
+                        setImmediate(resolve);
+                    }
+                });
+            }
+        });
+
+        req.on('error', reject);
+        req.end();
+    });
+}
+
+async function runExecCommand(appId, args, options) {
+    const stdoutChunks = [];
+    const stderrChunks = [];
+
+    const stdoutStream = new PassThrough();
+    const stderrStream = new PassThrough();
+
+    stdoutStream.on('data', (chunk) => stdoutChunks.push(chunk));
+    stderrStream.on('data', (chunk) => stderrChunks.push(chunk));
+
+    const execId = await createExecSession(appId, args, { tty: false }, options);
+
+    await runExecSession(appId, execId, {
+        stdin: () => new PassThrough(),
+        stdout: stdoutStream,
+        stderr: stderrStream,
+        tty: false
+    }, options);
+
+    const session = await getExecSession(appId, execId, options);
+
+    return {
+        stdout: Buffer.concat(stdoutChunks),
+        stderr: Buffer.concat(stderrChunks),
+        exitCode: session.exitCode
+    };
+}
+
+function getLocalFileList(localPath) {
+    const resolved = path.resolve(localPath);
+    const files = new Map();
+
+    if (fs.lstatSync(resolved).isFile()) {
+        const stat = fs.statSync(resolved);
+        files.set(path.basename(resolved), { size: stat.size, mtimeSec: Math.floor(stat.mtimeMs / 1000) });
+        return { baseDir: path.dirname(resolved), files };
+    }
+
+    const entries = fs.readdirSync(resolved, { withFileTypes: true, recursive: true });
+    for (const entry of entries) {
+        if (!entry.isFile()) continue;
+        const relativePath = path.relative(resolved, path.join(entry.parentPath || entry.path, entry.name));
+        const stat = fs.statSync(path.join(resolved, relativePath));
+        files.set(relativePath, { size: stat.size, mtimeSec: Math.floor(stat.mtimeMs / 1000) });
+    }
+
+    return { baseDir: resolved, files };
+}
+
+async function getRemoteFileList(appId, dir, options) {
+    const files = new Map();
+
+    let result;
+    try {
+        result = await runExecCommand(appId, ['find', dir, '-type', 'f', '-printf', '%s\\t%T@\\t%P\\n'], options);
+    } catch (e) {
+        console.error(`Warning: could not list remote files (${e.message}), syncing all files`);
+        return files;
+    }
+
+    if (result.exitCode !== 0) return files;
+
+    const output = result.stdout.toString().trim();
+    if (!output) return files;
+
+    for (const line of output.split('\n')) {
+        const [size, mtime, ...pathParts] = line.split('\t');
+        const relativePath = pathParts.join('\t');
+        files.set(relativePath, { size: parseInt(size, 10), mtimeSec: Math.floor(parseFloat(mtime)) });
+    }
+
+    return files;
+}
+
+function diffFileLists(srcList, destList) {
+    const changed = [];
+    const extraneous = [];
+
+    for (const [filePath, src] of srcList) {
+        const dest = destList.get(filePath);
+        if (!dest || dest.size !== src.size || src.mtimeSec > dest.mtimeSec) {
+            changed.push(filePath);
+        }
+    }
+
+    for (const filePath of destList.keys()) {
+        if (!srcList.has(filePath)) extraneous.push(filePath);
+    }
+
+    return { changed, extraneous };
+}
+
 // cloudron exec - must work interactively. needs tty.
 // cloudron exec -- ls asdf  - must work
 // cloudron exec -- cat /home/cloudron/start.sh > /tmp/start.sh - must work (test with binary files). should disable tty
@@ -1429,12 +1615,9 @@ function demuxStream(inputStream, stdout, stderr) {
 // cat ~/tmp/fantome.tar.gz | cloudron exec -- bash -c "tar xvf - -C /tmp" - must show an error
 // cat ~/tmp/fantome.tar.gz | cloudron exec -- bash -c "tar zxf - -C /tmp" - must extrack ok
 async function execApp(args, localOptions, cmd) {
-    let stdin = localOptions._stdin || process.stdin; // hack for 'push', 'pull' to reuse this function
-    const stdout = localOptions._stdout || process.stdout; // hack for 'push', 'pull' to reuse this function
-
     const options = cmd.optsWithGlobals();
     let tty = !!options.tty;
-    let lang; // used as body param later
+    let lang;
 
     const [error, app] = await safe(getApp(options));
     if (error) return exit(error);
@@ -1443,150 +1626,288 @@ async function execApp(args, localOptions, cmd) {
     if (app.installationState !== 'installed') exit('App is not yet running. Try again later.');
 
     if (args.length === 0) {
-        args = [ '/bin/bash' ];
-        tty = true; // override
+        args = ['/bin/bash'];
+        tty = true;
         lang = 'C.UTF-8';
     }
 
-    if (tty && !stdin.isTTY) exit('stdin is not tty');
-
-    const request = createRequest('POST', `/api/v1/apps/${app.id}/exec`, options);
-    const response = await request.send({ cmd: args, tty, lang });
-    if (response.status !== 200) return exit(`Failed to create exec: ${requestError(response)}`);
-    const execId = response.body.id;
-
-    async function exitWithCode() {
-        const response2 = await createRequest('GET', `/api/v1/apps/${app.id}/exec/${execId}`, options);
-        if (response2.status !== 200) return exit(`Failed to get exec code: ${requestError(response2)}`);
-
-        process.exit(response2.body.exitCode);
-    }
+    if (tty && !process.stdin.isTTY) exit('stdin is not tty');
 
-    const { adminFqdn, token: reqToken, rejectUnauthorized } = requestOptions(cmd.optsWithGlobals());
+    const execId = await createExecSession(app.id, args, { tty, lang }, options);
 
-    const searchParams = new URLSearchParams({
-        rows: stdout.rows || 24,
-        columns: stdout.columns || 80,
-        access_token: reqToken,
+    await runExecSession(app.id, execId, {
+        stdin: process.stdin,
+        stdout: process.stdout,
+        stderr: process.stderr,
         tty
-    });
-
-    const req = https.request({
-        hostname: adminFqdn,
-        path: `/api/v1/apps/${app.id}/exec/${execId}/start?${searchParams.toString()}`,
-        method: 'GET',
-        headers: {
-            'Connection': 'Upgrade',
-            'Upgrade': 'tcp'
-        },
-        rejectUnauthorized
-    }, function handler(res) {
-        if (res.statusCode === 403) exit('Unauthorized.'); // only admin or only owner (for docker addon)
-
-        exit('Could not upgrade connection to tcp. http status:', res.statusCode);
-    });
-
-    req.on('upgrade', function (resThatShouldNotBeUsed, socket /*, upgradeHead*/) {
-        // do not use res here! it's all socket from here on
-        socket.on('error', exit);
-
-        socket.setNoDelay(true);
-        socket.setKeepAlive(true);
-
-        if (tty) {
-            stdin.setRawMode(true);
-            stdin.pipe(socket, { end: false }); // the remote will close the connection
-            socket.pipe(stdout); // in tty mode, stdout/stderr is merged
-            socket.on('end', exitWithCode); // server closed the socket
-        } else { // create stdin process on demand
-            if (typeof stdin === 'function') stdin = stdin();
-
-            stdin.on('data', function (d) {
-                var buf = Buffer.alloc(4);
-                buf.writeUInt32BE(d.length, 0 /* offset */);
-                socket.write(buf);
-                socket.write(d);
-            });
-            stdin.on('end', function () {
-                var buf = Buffer.alloc(4);
-                buf.writeUInt32BE(0, 0 /* offset */);
-                socket.write(buf);
-            });
-
-            stdout.on('close', exitWithCode); // this is only emitted when stdout is a file and not the terminal
+    }, options);
 
-            demuxStream(socket, stdout, process.stderr); // can get separate streams in non-tty mode
-            socket.on('end', function () {  // server closed the socket
-                if (typeof stdin.end === 'function') stdin.end(); // required for this process to 'exit' cleanly. do not call exit() because writes may not have finished . the type check is required for when stdin: 'ignore' in execSync, not sure why
-                if (stdout !== process.stdout) stdout.end(); // for push stream
+    const execSession = await getExecSession(app.id, execId, options);
+    process.exit(execSession.exitCode);
+}
 
-                socket.end();
+async function pushApp(localDir, remote, localOptions, cmd) {
+    const options = cmd.optsWithGlobals();
 
-                // process._getActiveHandles(); process._getActiveRequests();
-                if (stdout === process.stdout) setImmediate(exitWithCode); // otherwise, we rely on the 'close' event above
-            });
-        }
-    });
+    const [error, app] = await safe(getApp(options));
+    if (error) return exit(error);
+    if (!app) return exit(NO_APP_FOUND_ERROR_STRING);
 
-    req.on('error', exit); // could not make a request
-    req.end(); // this makes the request
-}
+    if (app.installationState !== 'installed') exit('App is not yet running. Try again later.');
 
-function pushApp(localDir, remote, localOptions, cmd) {
-    // deal with paths prefixed with ~
     const local = localDir.replace(/^~(?=$|\/|\\)/, os.homedir());
     const stat = fs.existsSync(path.resolve(local)) ? fs.lstatSync(local) : null;
 
-    if (stat && stat.isDirectory())  {
+    let args, stdin;
+
+    if (stat && stat.isDirectory()) {
         // Create a functor for stdin. If no data event handlers are attached, and there are no stream.pipe() destinations, and the stream is
         // switched into flowing mode, then data will be lost. So, we have to start the tarzip only when exec is ready to attach event handlers.
-        localOptions._stdin = function () {
+        stdin = function () {
             var tarzip = spawn('tar', ['zcf', '-', '-C', path.dirname(local), path.basename(local)], { stdio: 'pipe' });
             return tarzip.stdout;
         };
-
-        execApp(['tar', 'zxvf', '-', '-C', remote], localOptions, cmd);
+        args = ['tar', 'zxvf', '-', '-C', remote];
     } else {
         if (local === '-') {
-            localOptions._stdin = process.stdin;
+            stdin = process.stdin;
         } else if (stat) {
-            localOptions._stdin = fs.createReadStream(local);
+            stdin = fs.createReadStream(local);
         } else {
             exit('local file ' + local + ' does not exist');
         }
 
-        localOptions._stdin.on('error', function (error) { exit('Error pushing', error); });
+        stdin.on('error', function (e) { exit('Error pushing', e); });
 
-        if (remote.endsWith('/')) { // dir
-            remote = remote + '/' + path.basename(local); // do not use path.join as we want this to be a UNIX path
+        if (remote.endsWith('/')) {
+            remote = remote + '/' + path.basename(local);
         }
 
-        execApp(['bash', '-c', `cat - > "${remote}"`], localOptions, cmd);
+        args = ['bash', '-c', `cat - > "${remote}"`];
     }
+
+    const execId = await createExecSession(app.id, args, { tty: false }, options);
+
+    await runExecSession(app.id, execId, {
+        stdin,
+        stdout: process.stdout,
+        stderr: process.stderr,
+        tty: false
+    }, options);
+
+    const execSession = await getExecSession(app.id, execId, options);
+    process.exit(execSession.exitCode);
 }
 
-function pull(remote, local, localOptions, cmd) {
-    if (remote.endsWith('/')) { // dir
-        var untar = tar.extract(local); // local directory is created if it doesn't exist!
+async function pull(remote, local, localOptions, cmd) {
+    const options = cmd.optsWithGlobals();
+
+    const [error, app] = await safe(getApp(options));
+    if (error) return exit(error);
+    if (!app) return exit(NO_APP_FOUND_ERROR_STRING);
+
+    if (app.installationState !== 'installed') exit('App is not yet running. Try again later.');
+
+    let args, stdout;
+
+    if (remote.endsWith('/')) {
+        var untar = tar.extract(local);
         var unzip = zlib.createGunzip();
 
         unzip.pipe(untar);
-        localOptions._stdout = unzip;
-
-        execApp(['tar', 'zcf', '-', '-C', remote, '.'], localOptions, cmd);
+        stdout = unzip;
+        args = ['tar', 'zcf', '-', '-C', remote, '.'];
     } else {
         if (fs.existsSync(local) && fs.lstatSync(local).isDirectory()) {
             local = path.join(local, path.basename(remote));
-            localOptions._stdout = fs.createWriteStream(local);
+            stdout = fs.createWriteStream(local);
         } else if (local === '-') {
-            localOptions._stdout = process.stdout;
+            stdout = process.stdout;
         } else {
-            localOptions._stdout = fs.createWriteStream(local);
+            stdout = fs.createWriteStream(local);
         }
 
-        localOptions._stdout.on('error', function (error) { exit('Error pulling', error); });
+        stdout.on('error', function (e) { exit('Error pulling', e); });
 
-        execApp(['cat', remote], localOptions, cmd);
+        args = ['cat', remote];
+    }
+
+    const execId = await createExecSession(app.id, args, { tty: false }, options);
+
+    await runExecSession(app.id, execId, {
+        stdin: process.stdin,
+        stdout,
+        stderr: process.stderr,
+        tty: false
+    }, options);
+
+    const execSession = await getExecSession(app.id, execId, options);
+    process.exit(execSession.exitCode);
+}
+
+async function syncPush(localDir, remoteDir, localOptions, cmd) {
+    try {
+        const options = cmd.optsWithGlobals();
+
+        const [error, app] = await safe(getApp(options));
+        if (error) return exit(error);
+        if (!app) return exit(NO_APP_FOUND_ERROR_STRING);
+
+        if (app.installationState !== 'installed') exit('App is not yet running. Try again later.');
+
+        const resolvedLocal = path.resolve(localDir);
+        if (!fs.existsSync(resolvedLocal)) exit('local path must be an existing file or directory');
+
+        const localStat = fs.lstatSync(resolvedLocal);
+        if (!localStat.isFile() && !localStat.isDirectory()) exit('local path must be an existing file or directory');
+
+        const { baseDir: tarBaseDir, files: localList } = getLocalFileList(resolvedLocal);
+
+        // rsync convention: no trailing slash on source dir appends the directory name to dest
+        if (localStat.isDirectory() && !localDir.endsWith('/') && !localDir.endsWith('/.')) {
+            remoteDir = remoteDir.replace(/\/+$/, '') + '/' + path.basename(resolvedLocal);
+        } else if (localStat.isFile()) {
+            remoteDir = remoteDir.replace(/\/+$/, '');
+        }
+
+        const remoteList = await getRemoteFileList(app.id, remoteDir, options);
+        const { changed: filesToSync, extraneous: filesToDelete } = diffFileLists(localList, remoteList);
+        const doDelete = options.delete && filesToDelete.length > 0;
+
+        if (filesToSync.length === 0 && !doDelete) {
+            console.log('Already up to date.');
+            return;
+        }
+
+        for (const f of filesToSync) console.log(f);
+        if (doDelete) for (const f of filesToDelete) console.log(`delete: ${f}`);
+        console.log(`Syncing ${filesToSync.length} file(s)${doDelete ? `, deleting ${filesToDelete.length} file(s)` : ''}...`);
+
+        if (filesToSync.length > 0) {
+            const localFileList = path.join(os.tmpdir(), `.cloudron-sync-${process.pid}`);
+            fs.writeFileSync(localFileList, filesToSync.join('\n') + '\n');
+
+            const stdin = function () {
+                const tarzip = spawn('tar', ['zcf', '-', '-C', tarBaseDir, '-T', localFileList], { stdio: 'pipe' });
+                tarzip.on('close', () => fs.unlinkSync(localFileList));
+                return tarzip.stdout;
+            };
+
+            const mkdirCmd = options.force
+                ? `test -f ${remoteDir} && rm -f ${remoteDir}; mkdir -p ${remoteDir}`
+                : `mkdir -p ${remoteDir}`;
+            const execId = await createExecSession(app.id, ['bash', '-c', `${mkdirCmd} && tar zxf - -C ${remoteDir}`], { tty: false }, options);
+
+            await runExecSession(app.id, execId, {
+                stdin,
+                stdout: process.stdout,
+                stderr: process.stderr,
+                tty: false
+            }, options);
+
+            const execSession = await getExecSession(app.id, execId, options);
+            if (execSession.exitCode !== 0) exit(`Sync failed with exit code ${execSession.exitCode}`);
+        }
+
+        if (doDelete) {
+            const CHUNK_SIZE = 500;
+            for (let i = 0; i < filesToDelete.length; i += CHUNK_SIZE) {
+                const chunk = filesToDelete.slice(i, i + CHUNK_SIZE).map(f => `${remoteDir}/${f}`);
+                const result = await runExecCommand(app.id, ['rm', '-f', ...chunk], options);
+                if (result.exitCode !== 0) exit(`Failed to delete remote files: ${result.stderr.toString().trim()}`);
+            }
+        }
+
+        console.log('Sync complete.');
+    } catch (e) {
+        exit(`Sync push failed: ${e.message}`);
+    }
+}
+
+async function syncPull(remoteDir, localDir, localOptions, cmd) {
+    try {
+        const options = cmd.optsWithGlobals();
+
+        const [error, app] = await safe(getApp(options));
+        if (error) return exit(error);
+        if (!app) return exit(NO_APP_FOUND_ERROR_STRING);
+
+        if (app.installationState !== 'installed') exit('App is not yet running. Try again later.');
+
+        // rsync convention: no trailing slash on source appends the directory name to dest
+        if (!remoteDir.endsWith('/')) {
+            localDir = path.join(localDir, path.basename(remoteDir));
+        }
+
+        const resolvedLocalDir = path.resolve(localDir);
+        if (fs.existsSync(resolvedLocalDir) && !fs.lstatSync(resolvedLocalDir).isDirectory()) {
+            if (options.force) fs.unlinkSync(resolvedLocalDir);
+            else exit(`local path '${localDir}' exists and is not a directory (use --force to remove)`);
+        }
+        if (!fs.existsSync(resolvedLocalDir)) fs.mkdirSync(resolvedLocalDir, { recursive: true });
+
+        const remoteList = await getRemoteFileList(app.id, remoteDir, options);
+        const { files: localList } = getLocalFileList(path.resolve(localDir));
+        const { changed: filesToSync, extraneous: filesToDelete } = diffFileLists(remoteList, localList);
+        const doDelete = options.delete && filesToDelete.length > 0;
+
+        if (filesToSync.length === 0 && !doDelete) {
+            console.log('Already up to date.');
+            return;
+        }
+
+        for (const f of filesToSync) console.log(f);
+        if (doDelete) for (const f of filesToDelete) console.log(`delete: ${f}`);
+        console.log(`Syncing ${filesToSync.length} file(s)${doDelete ? `, deleting ${filesToDelete.length} file(s)` : ''}...`);
+
+        if (filesToSync.length > 0) {
+            const remoteFileList = '/tmp/.cloudron-sync-filelist';
+            const fileListData = filesToSync.join('\n') + '\n';
+
+            const writeExecId = await createExecSession(app.id, ['bash', '-c', `cat > ${remoteFileList}`], { tty: false }, options);
+            await runExecSession(app.id, writeExecId, {
+                stdin: () => {
+                    const s = new PassThrough();
+                    s.end(fileListData);
+                    return s;
+                },
+                stdout: new PassThrough(),
+                stderr: process.stderr,
+                tty: false
+            }, options);
+            const writeSession = await getExecSession(app.id, writeExecId, options);
+            if (writeSession.exitCode !== 0) exit('Failed to write remote file list');
+
+            const untar = tar.extract(path.resolve(localDir));
+            const unzip = zlib.createGunzip();
+            unzip.pipe(untar);
+
+            const execId = await createExecSession(app.id, ['bash', '-c', `tar zcf - -C ${remoteDir} -T ${remoteFileList}; rm -f ${remoteFileList}`], { tty: false }, options);
+
+            await runExecSession(app.id, execId, {
+                stdin: () => {
+                    const s = new PassThrough();
+                    s.end();
+                    return s;
+                },
+                stdout: unzip,
+                stderr: process.stderr,
+                tty: false
+            }, options);
+
+            const execSession = await getExecSession(app.id, execId, options);
+            if (execSession.exitCode !== 0) exit(`Sync failed with exit code ${execSession.exitCode}`);
+        }
+
+        if (doDelete) {
+            for (const f of filesToDelete) {
+                fs.unlinkSync(path.join(path.resolve(localDir), f));
+            }
+        }
+
+        console.log('Sync complete.');
+    } catch (e) {
+        exit(`Sync pull failed: ${e.message}`);
     }
 }
 
@@ -1775,5 +2096,8 @@ export default {
     envGet,
     envSet,
     envList,
-    envUnset
+    envUnset,
+
+    syncPush,
+    syncPull
 };

package/src/build-actions.js

@@ -295,11 +295,6 @@ async function build(localOptions, cmd) {
 
     const appConfig = config.getCwdConfig(sourceDir);
     const buildServiceConfig = getEffectiveBuildServiceConfig(options);
-    if (buildServiceConfig.type === 'remote' && buildServiceConfig.url) {
-        console.log('Building using remote build service at %s', buildServiceConfig.url);
-    } else {
-        console.log('Building locally with Docker.');
-    }
 
     let repository = appConfig.repository;
     if (!repository || options.repository) {

package/bin/cloudron-appstore

@@ -1,65 +0,0 @@
-#!/usr/bin/env node
-
-import appstoreActions from '../src/appstore-actions.js';
-import { Command } from 'commander';
-import pkg from '../package.json' with { type: 'json' };
-
-const program = new Command();
-program.version(pkg.packageVersion);
-
-// global options. IMPORTANT: These cannot conflict with global options!
-program
-    .option('--appstore-token <token>', 'AppStore token');
-
-program.command('login')
-    .description('Login to the appstore')
-    .option('-e, --email <email>', 'Email address')
-    .option('-p, --password <password>', 'Password (unsafe)')
-    .action(appstoreActions.login);
-
-program.command('logout')
-    .description('Logout from the appstore')
-    .action(appstoreActions.logout);
-
-program.command('info')
-    .description('List info of published app')
-    .option('--appstore-id <appid@version>', 'Appstore id and version')
-    .action(appstoreActions.info);
-
-program.command('approve')
-    .description('Approve a submitted app version')
-    .option('--appstore-id <appid@version>', 'Appstore id and version')
-    .option('--no-git-push', 'Do not attempt to push to git repo')
-    .action(appstoreActions.approve);
-
-program.command('notify')
-    .description('Notify forum about successful app submission')
-    .action(appstoreActions.notify);
-
-program.command('revoke')
-    .description('Revoke a published app version')
-    .option('--appstore-id <appid@version>', 'Appstore id and version')
-    .action(appstoreActions.revoke);
-
-program.command('submit')
-    .description('Submit app to the store for review')
-    .action(appstoreActions.submit);
-
-program.command('upload')
-    .description('Upload app to the store for testing')
-    .option('-i, --image <image>', 'Docker image')
-    .option('-f, --force', 'Update existing version')
-    .action(appstoreActions.upload);
-
-program.command('verify-manifest')
-    .description('Verify if manifest is ready to be upload/published')
-    .action(appstoreActions.verifyManifest);
-
-program.command('versions')
-    .alias('list')
-    .description('List published versions')
-    .option('--appstore-id <id>', 'Appstore id')
-    .option('--raw', 'Dump versions as json')
-    .action(appstoreActions.listVersions);
-
-program.parse(process.argv);

package/bin/cloudron-build

@@ -1,71 +0,0 @@
-#!/usr/bin/env node
-
-import { program } from 'commander';
-import buildActions from '../src/build-actions.js';
-import pkg from '../package.json' with { type: 'json' };
-
-program.version(pkg.version);
-
-program.addHelpText('after', `
-
-Show help for default subcommand:
-  $ cloudron build build --help`);
-
-
-function collectArgs(value, collected) {
-    collected.push(value);
-    return collected;
-}
-
-// global options. IMPORTANT: These cannot conflict with global options!
-program.option('--server <server>', 'Cloudron domain')
-    .option('--build-service-url <url>', 'Build service URL')
-    .option('--build-service-token <token>', 'Build service token');
-
-program.command('build', { isDefault: true })
-    .description('Build an app. This is the default subcommand')
-    .option('--build-arg <namevalue>', 'Build arg passed to docker. Can be used multiple times', collectArgs, [])
-    .option('-f, --file <dockerfile>', 'Name of the Dockerfile')
-    .option('--repository [repository url]', 'Change the repository. This url is stored for future builds for this project. e.g registry/username/projectname')
-    .option('--no-cache', 'Do not use cache')
-    .option('--no-push', 'Do not push built image to registry')
-    .option('--raw', 'Raw output build log')
-    .option('--tag <docker image tag>', 'Docker image tag. Note that this does not include the repository name')
-    .action(buildActions.build);
-
-program.command('reset')
-    .description('Reset build configuration for this directory')
-    .action(buildActions.reset);
-
-program.command('info')
-    .description('Print build information')
-    .action(buildActions.info);
-
-program.command('login')
-    .description('Login to the build service')
-    .action(buildActions.login);
-
-program.command('logout')
-    .description('Logout from the build service')
-    .action(buildActions.logout);
-
-program.command('logs')
-    .description('Build logs. This works only when using the Build Service')
-    .option('--id <buildid>', 'Build ID')
-    .option('--raw', 'Raw output build log')
-    .action(buildActions.logs);
-
-program.command('push')
-    .description('Push the build image')
-    .option('--id <buildid>', 'Build ID')
-    .option('--repository [repository url]', 'Set repository to push to. e.g registry/username/projectname')
-    .option('--tag <docker image tag>', 'Docker image tag. Note that this does not include the repository name')
-    .option('--image <docker image>', 'Docker image of the form registry/repo:tag')
-    .action(buildActions.push);
-
-program.command('status')
-    .description('Build status. This works only when using the Build Service')
-    .option('--id <buildid>', 'Build ID')
-    .action(buildActions.status);
-
-program.parse(process.argv);

package/bin/cloudron-versions

@@ -1,33 +0,0 @@
-#!/usr/bin/env node
-
-import versionsActions from '../src/versions-actions.js';
-import { Command } from 'commander';
-import pkg from '../package.json' with { type: 'json' };
-
-const program = new Command();
-program.version(pkg.version);
-
-program.command('add')
-    .description('Add the current build to version file')
-    .option('--state <state>', 'Publish state (published or testing)')
-    .action(versionsActions.addOrUpdate);
-
-program.command('init')
-    .description('Create versions file')
-    .action(versionsActions.init);
-
-program.command('list')
-    .description('List existing versions')
-    .action(versionsActions.list);
-
-program.command('revoke')
-    .description('Revoke the latest version')
-    .action(versionsActions.revoke);
-
-program.command('update')
-    .description('Update existing version')
-    .option('--version <version>', 'Version to update')
-    .option('--state <state>', 'Publish state (published or testing)')
-    .action(versionsActions.addOrUpdate);
-
-program.parse(process.argv);