@pioneer-platform/pioneer-discovery 8.11.11 → 8.11.15

package/scripts/upload-to-s3.js

@@ -0,0 +1,363 @@
+#!/usr/bin/env node
+
+/*
+ * Upload Asset Icons to S3 (DigitalOcean Spaces)
+ *
+ * This script:
+ * 1. Reads generatedAssetData.json
+ * 2. For each asset, checks if local icon exists
+ * 3. Uploads to S3 with PUBLIC-READ ACL
+ * 4. Verifies upload is accessible (not 403)
+ * 5. Tracks progress and failures
+ *
+ * CRITICAL: Sets ACL to 'public-read' on each file to avoid 403 errors
+ */
+
+const fs = require('fs');
+const path = require('path');
+const { execSync } = require('child_process');
+const https = require('https');
+
+// Configuration
+const S3_ENDPOINT = 'https://sfo3.digitaloceanspaces.com';
+const S3_BUCKET = 'keepkey';
+const S3_REGION = 'sfo3';
+const CDN_URL = 'https://keepkey.sfo3.cdn.digitaloceanspaces.com';
+
+// Paths
+const DATA_FILE = path.join(__dirname, '..', 'src', 'generatedAssetData.json');
+const LOCAL_COINS_DIR = path.join(__dirname, '..', '..', '..', '..', '..', 'services', 'pioneer-server', 'public', 'coins');
+const PROGRESS_FILE = path.join(__dirname, '.upload-progress.json');
+const FAILED_FILE = path.join(__dirname, '.upload-failed.json');
+
+// AWS credentials from environment
+const AWS_ACCESS_KEY_ID = process.env.AWS_ACCESS_KEY_ID || 'DO00FXP8KK64LCXYAEZP';
+const AWS_SECRET_ACCESS_KEY = process.env.AWS_SECRET_ACCESS_KEY || 'Uyw/cq63rrQmFV9yy1HbovTSMNhLkEwImqPa88N/E/s';
+
+if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY) {
+  console.error('❌ Missing AWS credentials');
+  process.exit(1);
+}
+
+/**
+ * Convert assetId to base64-encoded format
+ */
+function encodeAssetId(assetId) {
+  return Buffer.from(assetId).toString('base64');
+}
+
+/**
+ * Check if URL is accessible (returns 200)
+ */
+async function urlExists(url, timeout = 5000) {
+  return new Promise((resolve) => {
+    const req = https.request(url, { method: 'HEAD', timeout }, (res) => {
+      resolve(res.statusCode === 200);
+    });
+
+    req.on('error', () => resolve(false));
+    req.on('timeout', () => {
+      req.destroy();
+      resolve(false);
+    });
+
+    req.end();
+  });
+}
+
+/**
+ * Upload file to S3 with public-read ACL
+ */
+function uploadToS3(localPath, s3Key) {
+  try {
+    const cmd = `AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} aws s3 cp "${localPath}" "s3://${S3_BUCKET}/${s3Key}" --endpoint-url=${S3_ENDPOINT} --acl public-read --region ${S3_REGION}`;
+
+    execSync(cmd, { stdio: 'pipe' });
+    return true;
+  } catch (error) {
+    console.error(`   ❌ Upload error: ${error.message}`);
+    return false;
+  }
+}
+
+/**
+ * Set ACL to public-read for existing file
+ */
+function setPublicAcl(s3Key) {
+  try {
+    const cmd = `AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} aws s3api put-object-acl --bucket ${S3_BUCKET} --key "${s3Key}" --acl public-read --endpoint-url=${S3_ENDPOINT}`;
+
+    execSync(cmd, { stdio: 'pipe' });
+    return true;
+  } catch (error) {
+    console.error(`   ❌ ACL set error: ${error.message}`);
+    return false;
+  }
+}
+
+/**
+ * Wait for specified milliseconds
+ */
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+/**
+ * Load progress file
+ */
+function loadProgress() {
+  if (fs.existsSync(PROGRESS_FILE)) {
+    return JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf8'));
+  }
+  return { processed: [], uploaded: 0, verified: 0, failed: 0 };
+}
+
+/**
+ * Save progress file
+ */
+function saveProgress(progress) {
+  fs.writeFileSync(PROGRESS_FILE, JSON.stringify(progress, null, 2));
+}
+
+/**
+ * Load or initialize failed tracking
+ */
+function loadFailed() {
+  if (fs.existsSync(FAILED_FILE)) {
+    return JSON.parse(fs.readFileSync(FAILED_FILE, 'utf8'));
+  }
+  return { assets: [] };
+}
+
+/**
+ * Save failed assets
+ */
+function saveFailed(failed) {
+  fs.writeFileSync(FAILED_FILE, JSON.stringify(failed, null, 2));
+}
+
+/**
+ * Process a single asset
+ */
+async function processAsset(assetId, assetData, progress, failed, stats) {
+  const encoded = encodeAssetId(assetId);
+  const localPath = path.join(LOCAL_COINS_DIR, `${encoded}.png`);
+  const s3Key = `coins/${encoded}.png`;
+  const cdnUrl = `${CDN_URL}/coins/${encoded}.png`;
+
+  stats.total++;
+
+  // Skip if already processed
+  if (progress.processed.includes(assetId)) {
+    console.log(`⏭️  [${stats.total}] Skipping (already processed): ${assetData.symbol}`);
+    stats.skipped++;
+    return;
+  }
+
+  console.log(`\n🔄 [${stats.total}] Processing: ${assetData.symbol} (${assetId.substring(0, 50)}...)`);
+
+  // Check if local file exists
+  if (!fs.existsSync(localPath)) {
+    console.log(`   ⚠️  No local file - skipping upload`);
+    progress.processed.push(assetId);
+    stats.noLocal++;
+    return;
+  }
+
+  const fileSize = fs.statSync(localPath).size;
+  console.log(`   📦 Local file size: ${(fileSize / 1024).toFixed(2)} KB`);
+
+  // Upload to S3 with public-read ACL
+  console.log(`   📤 Uploading to S3...`);
+  const uploaded = uploadToS3(localPath, s3Key);
+
+  if (!uploaded) {
+    console.log(`   ❌ Upload failed`);
+    failed.assets.push({
+      assetId,
+      symbol: assetData.symbol,
+      reason: 'upload_failed',
+      localPath,
+      s3Key,
+    });
+    progress.processed.push(assetId);
+    progress.failed++;
+    stats.uploadFailed++;
+    return;
+  }
+
+  console.log(`   ✅ Uploaded successfully`);
+  progress.uploaded++;
+  stats.uploaded++;
+
+  // Wait a bit for S3 to propagate
+  await sleep(500);
+
+  // Verify accessibility via CDN
+  console.log(`   🔍 Verifying CDN access: ${cdnUrl.substring(0, 80)}...`);
+  const accessible = await urlExists(cdnUrl);
+
+  if (!accessible) {
+    console.log(`   ⚠️  Not accessible (403?) - trying to set ACL...`);
+
+    const aclSet = setPublicAcl(s3Key);
+    if (aclSet) {
+      await sleep(1000); // Wait for ACL to propagate
+      const retryAccessible = await urlExists(cdnUrl);
+
+      if (retryAccessible) {
+        console.log(`   ✅ Now accessible after ACL fix!`);
+        progress.verified++;
+        stats.verified++;
+      } else {
+        console.log(`   ❌ Still not accessible after ACL fix`);
+        failed.assets.push({
+          assetId,
+          symbol: assetData.symbol,
+          reason: 'not_accessible_after_acl',
+          cdnUrl,
+          s3Key,
+        });
+        progress.failed++;
+        stats.verifyFailed++;
+      }
+    } else {
+      console.log(`   ❌ Failed to set ACL`);
+      failed.assets.push({
+        assetId,
+        symbol: assetData.symbol,
+        reason: 'acl_set_failed',
+        cdnUrl,
+        s3Key,
+      });
+      progress.failed++;
+      stats.verifyFailed++;
+    }
+  } else {
+    console.log(`   ✅ Verified accessible!`);
+    progress.verified++;
+    stats.verified++;
+  }
+
+  progress.processed.push(assetId);
+
+  // Save progress every 10 assets
+  if (stats.total % 10 === 0) {
+    saveProgress(progress);
+    saveFailed(failed);
+    console.log(`\n📊 Progress saved - Total: ${stats.total}, Uploaded: ${stats.uploaded}, Verified: ${stats.verified}, Failed: ${stats.uploadFailed + stats.verifyFailed}`);
+  }
+}
+
+/**
+ * Main execution
+ */
+async function main() {
+  console.log('🚀 Starting S3 upload process...\n');
+  console.log(`📂 Local coins directory: ${LOCAL_COINS_DIR}`);
+  console.log(`☁️  S3 bucket: ${S3_BUCKET}`);
+  console.log(`🌐 CDN URL: ${CDN_URL}\n`);
+
+  // Check local directory exists
+  if (!fs.existsSync(LOCAL_COINS_DIR)) {
+    console.error(`❌ Local coins directory not found: ${LOCAL_COINS_DIR}`);
+    console.log('💡 Run the download script first to get icons locally');
+    process.exit(1);
+  }
+
+  // Load data and progress
+  const data = JSON.parse(fs.readFileSync(DATA_FILE, 'utf8'));
+  const entries = Object.entries(data);
+  const progress = loadProgress();
+  const failed = loadFailed();
+
+  const stats = {
+    total: 0,
+    skipped: 0,
+    noLocal: 0,
+    uploaded: 0,
+    verified: 0,
+    uploadFailed: 0,
+    verifyFailed: 0,
+  };
+
+  console.log(`📊 Total assets in database: ${entries.length}`);
+  console.log(`📊 Already processed: ${progress.processed.length}`);
+  console.log(`📊 Remaining: ${entries.length - progress.processed.length}\n`);
+
+  if (progress.processed.length > 0) {
+    const answer = require('readline').createInterface({
+      input: process.stdin,
+      output: process.stdout
+    });
+
+    const response = await new Promise(resolve => {
+      answer.question('Resume from previous progress? (y/n): ', resolve);
+    });
+
+    if (response.toLowerCase() !== 'y') {
+      console.log('Starting fresh...');
+      progress.processed = [];
+      progress.uploaded = 0;
+      progress.verified = 0;
+      progress.failed = 0;
+      failed.assets = [];
+    }
+
+    answer.close();
+  }
+
+  // Process assets sequentially (to respect rate limits)
+  for (const [assetId, assetData] of entries) {
+    await processAsset(assetId, assetData, progress, failed, stats);
+
+    // Small delay between uploads to be nice to S3
+    await sleep(100);
+  }
+
+  // Final save
+  saveProgress(progress);
+  saveFailed(failed);
+
+  // Cleanup progress file if complete
+  if (progress.processed.length === entries.length && failed.assets.length === 0) {
+    fs.unlinkSync(PROGRESS_FILE);
+    console.log('\n✅ All assets processed successfully - progress file cleaned up');
+  }
+
+  // Print summary
+  console.log('\n' + '='.repeat(80));
+  console.log('📊 FINAL SUMMARY');
+  console.log('='.repeat(80));
+  console.log(`Total assets:                ${entries.length}`);
+  console.log(`Already processed (skipped): ${stats.skipped}`);
+  console.log(`No local file:               ${stats.noLocal}`);
+  console.log(`Uploaded successfully:       ${stats.uploaded}`);
+  console.log(`Verified accessible:         ${stats.verified}`);
+  console.log(`Upload failed:               ${stats.uploadFailed}`);
+  console.log(`Verification failed:         ${stats.verifyFailed}`);
+  console.log(`Total failed:                ${stats.uploadFailed + stats.verifyFailed}`);
+  console.log('='.repeat(80));
+
+  if (failed.assets.length > 0) {
+    console.log(`\n⚠️  ${failed.assets.length} assets failed - see ${FAILED_FILE}`);
+
+    // Group by failure reason
+    const byReason = {};
+    failed.assets.forEach(asset => {
+      byReason[asset.reason] = (byReason[asset.reason] || 0) + 1;
+    });
+
+    console.log('\nFailure reasons:');
+    Object.entries(byReason).forEach(([reason, count]) => {
+      console.log(`   ${reason}: ${count}`);
+    });
+  }
+
+  console.log('\n✅ Upload process complete!\n');
+}
+
+main().catch(err => {
+  console.error('❌ Fatal error:', err);
+  process.exit(1);
+});