@getxflow/cli 0.9.1 → 0.10.0

package/dist/bin.js

@@ -24,6 +24,7 @@ const org_1 = require("./commands/org");
 const schedules_1 = require("./commands/schedules");
 const skills_1 = require("./commands/skills");
 const sources_1 = require("./commands/sources");
+const storage_1 = require("./commands/storage");
 const deploy_1 = require("./commands/deploy");
 const update_1 = require("./commands/update");
 async function run(args) {
@@ -164,6 +165,20 @@ async function run(args) {
                 return;
             }
             throw new errors_1.CliError(`Unknown command: db ${second}`, 'Available: status, schema, query and migrate');
+        case 'storage':
+            if (second === 'push') {
+                await (0, storage_1.storagePush)(rest);
+                return;
+            }
+            if (second === 'rm' || second === 'remove') {
+                await (0, storage_1.storageRemove)(rest);
+                return;
+            }
+            if (second === undefined || second === 'ls' || second === 'list') {
+                await (0, storage_1.storageList)(rest);
+                return;
+            }
+            throw new errors_1.CliError(`Unknown command: storage ${second}`, 'Available: ls, push and rm');
         case 'status':
             await (0, sources_1.status)();
             return;

package/dist/commands/storage.js

@@ -0,0 +1,370 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.contentTypeFor = contentTypeFor;
+exports.planBatches = planBatches;
+exports.caseCollisions = caseCollisions;
+exports.storageList = storageList;
+exports.storagePush = storagePush;
+exports.storageRemove = storageRemove;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const node_stream_1 = require("node:stream");
+const api_1 = require("../api");
+const args_1 = require("../args");
+const config_1 = require("../config");
+const errors_1 = require("../errors");
+const session_1 = require("../session");
+const ui_1 = require("../ui");
+/**
+ * File storage of the project: the place for heavy static assets of the
+ * application.
+ *
+ * Sources are capped at 10 MB and are rebuilt on every deploy, so photos, video
+ * and PDFs do not belong there. Files here live beside the versions instead:
+ * they survive a build, they are metered against the plan of the organization,
+ * and their addresses do not change when the application is rebuilt.
+ */
+/** Files per batch, and bytes per batch: whichever comes first closes it. */
+const BATCH_FILES = 10;
+const BATCH_BYTES = 20 * 1024 * 1024;
+/** Uploads running at the same time. */
+const PARALLEL_UPLOADS = 4;
+/** Below this a file is sent as one buffer; above it, streamed off disk. */
+const STREAM_OVER_BYTES = 16 * 1024 * 1024;
+const UPLOAD_TIMEOUT_MS = 10 * 60_000;
+const CONTENT_TYPES = {
+    '.png': 'image/png',
+    '.jpg': 'image/jpeg',
+    '.jpeg': 'image/jpeg',
+    '.gif': 'image/gif',
+    '.webp': 'image/webp',
+    '.avif': 'image/avif',
+    '.svg': 'image/svg+xml',
+    '.ico': 'image/x-icon',
+    '.mp4': 'video/mp4',
+    '.webm': 'video/webm',
+    '.mov': 'video/quicktime',
+    '.mp3': 'audio/mpeg',
+    '.wav': 'audio/wav',
+    '.ogg': 'audio/ogg',
+    '.pdf': 'application/pdf',
+    '.json': 'application/json',
+    '.csv': 'text/csv',
+    '.txt': 'text/plain',
+    '.md': 'text/markdown',
+    '.html': 'text/html',
+    '.css': 'text/css',
+    '.js': 'text/javascript',
+    '.woff': 'font/woff',
+    '.woff2': 'font/woff2',
+    '.ttf': 'font/ttf',
+    '.otf': 'font/otf',
+    '.zip': 'application/zip',
+};
+/** Unknown extensions go up as bytes: the browser then trusts the file name. */
+function contentTypeFor(name) {
+    return CONTENT_TYPES[(0, node_path_1.extname)(name).toLowerCase()] ?? 'application/octet-stream';
+}
+/**
+ * Files of a directory, recursively.
+ *
+ * A walker of its own instead of the one the sources use: that one reads every
+ * file into memory to hash it and rewrites CRLF into LF for anything without a
+ * NUL byte early on, which would quietly damage SVG, CSV and subtitles on the
+ * way to the bucket. It also obeys .xflowignore, and a media folder is exactly
+ * what one is asked to put there so it stays out of the 10 MB archive.
+ *
+ * Dot entries and symlinks are skipped: pointing push at a working copy should
+ * not upload .git, and a link is not ours to follow.
+ */
+function collectMedia(root, prefix = '') {
+    const found = [];
+    for (const entry of (0, node_fs_1.readdirSync)(root, { withFileTypes: true })) {
+        if (entry.name.startsWith('.') || entry.isSymbolicLink())
+            continue;
+        if (entry.name === 'Thumbs.db')
+            continue;
+        const absolute = (0, node_path_1.join)(root, entry.name);
+        const remote = prefix ? `${prefix}/${entry.name}` : entry.name;
+        if (entry.isDirectory()) {
+            found.push(...collectMedia(absolute, remote));
+            continue;
+        }
+        if (!entry.isFile())
+            continue;
+        found.push({ absolute, remote, size: (0, node_fs_1.statSync)(absolute).size });
+    }
+    return found;
+}
+/**
+ * Split the upload into batches.
+ *
+ * The window where an object is in the bucket but not yet recorded lasts from
+ * the PUT to the confirm, and everything in that window is invisible to the
+ * project while still taking up the quota. Ten files or twenty megabytes keep
+ * the window that size no matter whether these are icons or video: a file
+ * heavier than the limit goes up alone.
+ */
+function planBatches(files, maxFiles = BATCH_FILES, maxBytes = BATCH_BYTES) {
+    const batches = [];
+    let batch = [];
+    let bytes = 0;
+    for (const file of files) {
+        if (batch.length > 0 && (batch.length >= maxFiles || bytes + file.size > maxBytes)) {
+            batches.push(batch);
+            batch = [];
+            bytes = 0;
+        }
+        batch.push(file);
+        bytes += file.size;
+    }
+    if (batch.length > 0)
+        batches.push(batch);
+    return batches;
+}
+/**
+ * Paths that differ only in case.
+ *
+ * The bucket tells them apart and Windows does not, so a folder that looks like
+ * one file locally becomes two files on the platform, and the application asks
+ * for whichever the code spells.
+ */
+function caseCollisions(files) {
+    const seen = new Map();
+    for (const file of files) {
+        const key = file.remote.toLowerCase();
+        seen.set(key, [...(seen.get(key) ?? []), file.remote]);
+    }
+    return [...seen.values()].filter((group) => group.length > 1);
+}
+async function fetchPage(client, projectId, params) {
+    const query = new URLSearchParams(params).toString();
+    return (0, api_1.apiJson)(client, `/api/v1/projects/${projectId}/storage?${query}`);
+}
+/** Everything under a folder, following the cursor to the end. */
+async function fetchAll(client, projectId, folder) {
+    const files = [];
+    let cursor = null;
+    do {
+        const params = { limit: '500' };
+        if (folder)
+            params.folder = folder;
+        if (cursor)
+            params.cursor = cursor;
+        const page = await fetchPage(client, projectId, params);
+        files.push(...page.files);
+        cursor = page.next_cursor;
+    } while (cursor);
+    return files;
+}
+async function storageList(args) {
+    const { config } = (0, config_1.requireProject)();
+    const client = (0, session_1.connect)(config);
+    const folder = args.words[1] ?? '';
+    const files = await fetchAll(client, config.projectId, folder);
+    if ((0, args_1.flagBool)(args, 'json')) {
+        (0, ui_1.out)(JSON.stringify({ files }, null, 2));
+        return;
+    }
+    if (files.length === 0) {
+        (0, ui_1.note)(folder ? `No files under ${folder}` : 'The project has no files in storage');
+        (0, ui_1.note)((0, ui_1.dim)('  To upload a folder: xflow storage push ./media --to media'));
+        return;
+    }
+    (0, ui_1.table)(files.map((file) => [file.path, (0, ui_1.formatBytes)(file.size), file.url]));
+    (0, ui_1.note)((0, ui_1.dim)(`  ${files.length} file(s), ${(0, ui_1.formatBytes)(files.reduce((sum, f) => sum + f.size, 0))}`));
+}
+/** PUT the bytes straight into the bucket: they never pass through the platform. */
+async function putObject(file, ticket) {
+    const contentType = contentTypeFor(file.remote);
+    // Small files go as one buffer, big ones off the disk: 200 MB in memory is a
+    // price nothing here pays for.
+    const body = file.size > STREAM_OVER_BYTES
+        ? node_stream_1.Readable.toWeb((0, node_fs_1.createReadStream)(file.absolute))
+        : (0, node_fs_1.readFileSync)(file.absolute);
+    // `duplex` is required by Node to send a stream as a request body and is not
+    // in the type of fetch, hence the cast.
+    const init = {
+        method: 'PUT',
+        body,
+        headers: { 'Content-Type': contentType, 'Content-Length': String(file.size) },
+        signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS),
+        duplex: 'half',
+    };
+    const response = await fetch(ticket.upload_url, init);
+    if (!response.ok) {
+        throw new errors_1.CliError(`${file.remote}: the bucket rejected the upload (${response.status})`, 'Run the command again: what was uploaded before is kept');
+    }
+}
+/** Run tasks a few at a time, keeping the order of the results. */
+async function inParallel(items, limit, task) {
+    const results = new Array(items.length);
+    let next = 0;
+    async function worker() {
+        while (next < items.length) {
+            const index = next++;
+            results[index] = await task(items[index]);
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+    return results;
+}
+async function storagePush(args) {
+    const { config } = (0, config_1.requireProject)();
+    const client = (0, session_1.connect)(config);
+    const source = args.words[1];
+    if (!source) {
+        throw new errors_1.CliError('A folder or a file is required', 'For example: xflow storage push ./media --to media');
+    }
+    const absolute = (0, node_path_1.resolve)(source);
+    let stat;
+    try {
+        stat = (0, node_fs_1.statSync)(absolute);
+    }
+    catch {
+        throw new errors_1.CliError(`No such file or folder: ${source}`);
+    }
+    const target = ((0, args_1.flagString)(args, 'to') ?? '').replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');
+    const local = stat.isDirectory()
+        ? collectMedia(absolute)
+        : [{ absolute, remote: (0, node_path_1.basename)(absolute), size: stat.size }];
+    if (local.length === 0)
+        throw new errors_1.CliError(`Nothing to upload in ${source}`);
+    const files = local
+        .map((file) => ({ ...file, remote: target ? `${target}/${file.remote}` : file.remote }))
+        .sort((a, b) => a.remote.localeCompare(b.remote));
+    for (const group of caseCollisions(files)) {
+        (0, ui_1.warn)(`Names that differ only in case: ${group.join(', ')}`);
+        (0, ui_1.note)((0, ui_1.dim)('  The platform keeps them as separate files'));
+    }
+    const replace = (0, args_1.flagBool)(args, 'replace');
+    const asJson = (0, args_1.flagBool)(args, 'json');
+    // What is already there decides what to send: the same run repeated has to be
+    // cheap and quiet, not a pile of conflicts.
+    const known = new Map((await fetchAll(client, config.projectId, target)).map((f) => [f.path, f]));
+    const outcome = { uploaded: [], skipped: [], failed: [] };
+    const pending = [];
+    for (const file of files) {
+        const remote = known.get(file.remote);
+        if (!remote) {
+            pending.push(file);
+            continue;
+        }
+        if (remote.size === file.size) {
+            outcome.skipped.push({ path: file.remote, reason: 'already there' });
+            continue;
+        }
+        if (!replace) {
+            outcome.skipped.push({ path: file.remote, reason: 'differs, kept as is' });
+            continue;
+        }
+        pending.push(file);
+    }
+    if (pending.length === 0) {
+        report(outcome, asJson);
+        return;
+    }
+    const batches = planBatches(pending);
+    if (!asJson) {
+        const bytes = pending.reduce((sum, file) => sum + file.size, 0);
+        (0, ui_1.step)(`Uploading ${pending.length} file(s), ${(0, ui_1.formatBytes)(bytes)}`);
+    }
+    for (const batch of batches) {
+        const { files: tickets } = await (0, api_1.apiJson)(client, `/api/v1/projects/${config.projectId}/storage/upload-url`, {
+            method: 'POST',
+            body: {
+                files: batch.map((file) => ({
+                    path: file.remote,
+                    content_type: contentTypeFor(file.remote),
+                    size: file.size,
+                })),
+                overwrite: replace,
+            },
+        });
+        const byPath = new Map(tickets.map((ticket) => [ticket.path, ticket]));
+        const done = [];
+        const results = await inParallel(batch, PARALLEL_UPLOADS, async (file) => {
+            const ticket = byPath.get(file.remote);
+            if (!ticket)
+                return { file, error: 'the platform did not return an upload address' };
+            try {
+                await putObject(file, ticket);
+                return { file, ticket };
+            }
+            catch (e) {
+                return { file, error: e instanceof Error ? e.message : String(e) };
+            }
+        });
+        for (const result of results) {
+            if ('ticket' in result)
+                done.push({ file: result.file, ticket: result.ticket });
+            else
+                outcome.failed.push({ path: result.file.remote, reason: result.error });
+        }
+        if (done.length === 0)
+            continue;
+        // Confirm right after the batch, not at the end: an object that is in the
+        // bucket without a record is invisible to the project and still takes up
+        // the quota.
+        const confirmed = await (0, api_1.apiJson)(client, `/api/v1/projects/${config.projectId}/storage/confirm`, {
+            method: 'POST',
+            body: { files: done.map(({ ticket }) => ({ s3_key: ticket.s3_key })) },
+        });
+        outcome.uploaded.push(...confirmed.files.map((file) => ({ path: file.path, url: file.url, size: file.size })));
+        for (const failure of confirmed.failed) {
+            const owner = done.find(({ ticket }) => ticket.s3_key === failure.s3_key);
+            outcome.failed.push({ path: owner?.file.remote ?? failure.s3_key, reason: failure.error });
+        }
+    }
+    report(outcome, asJson);
+}
+function report(outcome, asJson) {
+    if (asJson) {
+        (0, ui_1.out)(JSON.stringify(outcome, null, 2));
+        return;
+    }
+    if (outcome.uploaded.length > 0) {
+        (0, ui_1.table)(outcome.uploaded.map((file) => [file.path, (0, ui_1.formatBytes)(file.size), file.url]));
+    }
+    const parts = [`uploaded ${outcome.uploaded.length}`];
+    if (outcome.skipped.length > 0)
+        parts.push(`skipped ${outcome.skipped.length}`);
+    if (outcome.failed.length > 0)
+        parts.push(`failed ${outcome.failed.length}`);
+    if (outcome.failed.length > 0) {
+        for (const file of outcome.failed)
+            (0, ui_1.warn)(`${file.path}: ${file.reason}`);
+        throw new errors_1.CliError(parts.join(', '), 'Run the command again: what went up is kept and skipped');
+    }
+    (0, ui_1.ok)(parts.join(', '));
+    if (outcome.uploaded.length > 0) {
+        (0, ui_1.note)((0, ui_1.dim)('  Addresses are permanent: replacing a file keeps its address'));
+        (0, ui_1.note)((0, ui_1.dim)('  The whole list in one piece: xflow storage ls --json'));
+    }
+}
+async function storageRemove(args) {
+    const { config } = (0, config_1.requireProject)();
+    const client = (0, session_1.connect)(config);
+    const folder = (0, args_1.flagString)(args, 'folder');
+    const url = args.words[1];
+    if ((!url && !folder) || (url && folder)) {
+        throw new errors_1.CliError('Either the address of a file or --folder is required', 'For example: xflow storage rm https://app.getxflow.com/api/storage/files/…/view, or xflow storage rm --folder photos --yes');
+    }
+    const query = new URLSearchParams(url ? { url } : { folder: folder });
+    if ((0, args_1.flagBool)(args, 'yes'))
+        query.set('confirm', 'true');
+    let result;
+    try {
+        result = await (0, api_1.apiJson)(client, `/api/v1/projects/${config.projectId}/storage?${query.toString()}`, { method: 'DELETE' });
+    }
+    catch (e) {
+        // The platform counts what a folder holds and refuses until the answer is
+        // seen. Said in the words of this command, not of the API.
+        if (e instanceof api_1.ApiError && e.code === 'confirm_required') {
+            throw new errors_1.CliError(e.message, `To delete them anyway: xflow storage rm --folder ${folder} --yes`);
+        }
+        throw e;
+    }
+    (0, ui_1.ok)(`${(0, ui_1.bold)(String(result.deleted))} file(s) deleted`);
+    (0, ui_1.note)((0, ui_1.dim)('  There is no undo and no copy on the platform side'));
+}