carto-cli 0.1.0-rc.1

package/dist/commands/imports.js

@@ -0,0 +1,234 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.importsCreate = importsCreate;
+const api_1 = require("../api");
+const colors_1 = require("../colors");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const crypto_1 = require("crypto");
+const http_1 = require("../http");
+const MAX_FILE_SIZE = 1 * 1024 * 1024 * 1024; // 1GB
+const POLL_INTERVAL = 3000; // 3 seconds
+async function importsCreate(options, token, baseUrl, jsonOutput, debug = false, profile) {
+    try {
+        const client = await api_1.ApiClient.create(token, baseUrl, debug, profile);
+        // Validate required parameters
+        if (!options.connection) {
+            throw new Error('--connection is required');
+        }
+        if (!options.destination) {
+            throw new Error('--destination is required');
+        }
+        // Validate that either --file or --url is provided (not both)
+        if (!options.file && !options.url) {
+            throw new Error('Either --file or --url must be provided');
+        }
+        if (options.file && options.url) {
+            throw new Error('Cannot specify both --file and --url');
+        }
+        let importUrl;
+        let previewSchema = null;
+        const startTime = Date.now();
+        // If --file is provided, upload it first
+        if (options.file) {
+            if (!jsonOutput) {
+                console.log((0, colors_1.dim)('Uploading file...'));
+            }
+            importUrl = await uploadFile(client, options.file, options.connection, jsonOutput, debug);
+            if (!jsonOutput) {
+                console.log((0, colors_1.success)('✓ File uploaded successfully'));
+            }
+            // Note: Preview/schema detection skipped - let import API handle it automatically
+        }
+        else {
+            importUrl = options.url;
+        }
+        // Create import job
+        if (!jsonOutput) {
+            console.log((0, colors_1.dim)('Creating import job...'));
+        }
+        const importBody = {
+            connection: options.connection,
+            url: importUrl,
+            destination: options.destination,
+        };
+        // For file uploads, add client workspace marker
+        if (options.file) {
+            importBody.client = 'workspace';
+        }
+        if (options.overwrite) {
+            importBody.overwrite = true;
+        }
+        if (options.noAutoguessing) {
+            importBody.autoguessing = false;
+        }
+        const jobResponse = await client.post('/v3/imports', importBody);
+        if (!jsonOutput) {
+            console.log((0, colors_1.dim)(`Job ID: ${jobResponse.jobId}`));
+        }
+        // If --async flag is set, return immediately
+        if (options.async) {
+            if (jsonOutput) {
+                console.log(JSON.stringify({
+                    success: true,
+                    jobId: jobResponse.jobId,
+                    status: jobResponse.status
+                }));
+            }
+            else {
+                console.log((0, colors_1.success)(`✓ Import job created: ${jobResponse.jobId}`));
+                console.log((0, colors_1.dim)(`Status: ${jobResponse.status}`));
+            }
+            return;
+        }
+        // Poll for completion
+        const finalStatus = await pollImportStatus(client, jobResponse.jobId, jsonOutput);
+        const elapsedTime = Math.round((Date.now() - startTime) / 1000);
+        if (finalStatus.status === 'success') {
+            if (jsonOutput) {
+                console.log(JSON.stringify({
+                    success: true,
+                    jobId: finalStatus.jobId,
+                    status: finalStatus.status,
+                    destination: finalStatus.destination,
+                    rowCount: finalStatus.rowCount,
+                    elapsedTime
+                }));
+            }
+            else {
+                console.log((0, colors_1.success)(`✓ Import completed successfully in ${elapsedTime}s`));
+                console.log((0, colors_1.bold)('Table: ') + finalStatus.destination);
+                if (finalStatus.rowCount) {
+                    console.log((0, colors_1.bold)('Rows: ') + finalStatus.rowCount.toLocaleString());
+                }
+            }
+        }
+        else {
+            if (jsonOutput) {
+                console.log(JSON.stringify({
+                    success: false,
+                    jobId: finalStatus.jobId,
+                    status: finalStatus.status,
+                    error: finalStatus.error,
+                    elapsedTime
+                }));
+            }
+            else {
+                console.log((0, colors_1.error)(`✗ Import failed after ${elapsedTime}s`));
+                if (finalStatus.error) {
+                    console.log((0, colors_1.error)('Error: ') + finalStatus.error);
+                }
+            }
+            process.exit(1);
+        }
+    }
+    catch (err) {
+        if (jsonOutput) {
+            console.log(JSON.stringify({ success: false, error: err.message }));
+        }
+        else {
+            if (err.message.includes('401') || err.message.includes('Token not defined')) {
+                console.log((0, colors_1.error)('✗ Authentication required'));
+                console.log('Please run: carto auth login');
+            }
+            else {
+                console.log((0, colors_1.error)('✗ Failed to create import: ' + err.message));
+            }
+        }
+        process.exit(1);
+    }
+}
+async function uploadFile(client, filePath, connectionName, jsonOutput, debug) {
+    // Validate file exists
+    if (!fs_1.default.existsSync(filePath)) {
+        throw new Error(`File not found: ${filePath}`);
+    }
+    // Validate file size
+    const stats = fs_1.default.statSync(filePath);
+    if (stats.size > MAX_FILE_SIZE) {
+        throw new Error(`File size exceeds 1GB limit (${Math.round(stats.size / 1024 / 1024)}MB)`);
+    }
+    if (!jsonOutput && debug) {
+        console.log((0, colors_1.dim)(`File size: ${Math.round(stats.size / 1024 / 1024 * 100) / 100} MB`));
+    }
+    // Get user ID
+    const userId = await getUserId(client);
+    // Generate unique filename
+    const ext = path_1.default.extname(filePath);
+    const filename = `${userId}/${(0, crypto_1.randomUUID)()}${ext}`;
+    // Get signed upload URL
+    const signedUrlResponse = await client.postWorkspace('/storage/sign', {
+        type: 'import',
+        filename,
+        connectionName
+    });
+    // Upload file to GCS
+    const fileContent = fs_1.default.readFileSync(filePath);
+    const uploadHeaders = {
+        'Content-Length': stats.size.toString(),
+        ...signedUrlResponse.headers
+    };
+    if (debug) {
+        console.error((0, colors_1.dim)('\n--- Upload Debug ---'));
+        console.error((0, colors_1.bold)('Method: ') + 'PUT');
+        console.error((0, colors_1.bold)('URL: ') + signedUrlResponse.url);
+        console.error((0, colors_1.bold)('Headers:'));
+        Object.entries(uploadHeaders).forEach(([key, value]) => {
+            console.error(`  ${key}: ${value}`);
+        });
+        console.error((0, colors_1.dim)('-------------------\n'));
+    }
+    const uploadResponse = await (0, http_1.request)(signedUrlResponse.url, {
+        method: 'PUT',
+        headers: uploadHeaders,
+        body: fileContent
+    });
+    if (uploadResponse.statusCode >= 400) {
+        throw new Error(`Failed to upload file: ${uploadResponse.statusCode} ${uploadResponse.body}`);
+    }
+    // Get a NEW signed URL for reading (the upload URL is write-only)
+    const readUrlResponse = await client.postWorkspace('/storage/sign', {
+        type: 'import',
+        filename,
+        connectionName
+    });
+    // Return the read URL for the import API
+    return readUrlResponse.url;
+}
+async function getUserId(client) {
+    try {
+        const userInfo = await client.getAccounts('/users/me', true);
+        return userInfo.user_id;
+    }
+    catch (err) {
+        throw new Error(`Failed to get user ID: ${err.message}`);
+    }
+}
+async function getPreviewSchema(client, url, connection) {
+    try {
+        const preview = await client.post('/v3/imports/preview', {
+            connection,
+            url,
+            client: 'workspace'
+        });
+        return preview;
+    }
+    catch (err) {
+        throw new Error(`Failed to get file schema: ${err.message}`);
+    }
+}
+async function pollImportStatus(client, jobId, jsonOutput) {
+    let status;
+    do {
+        // Wait before polling
+        await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL));
+        status = await client.get(`/v3/imports/${jobId}`);
+        if (!jsonOutput) {
+            console.log((0, colors_1.dim)(`Status: ${status.status}...`));
+        }
+    } while (status.status === 'pending' || status.status === 'running');
+    return status;
+}