@pioneer-platform/pioneer-discovery 0.8.0 → 0.8.2

package/scripts/update-asset-data.ts

@@ -0,0 +1,187 @@
+/*
+ * Asset Data Update Script
+ * Updates generatedAssetData.json with verified api.keepkey.info URLs
+ * ONLY run after deployment is verified!
+ */
+
+import * as fs from 'fs';
+import * as path from 'path';
+
+const log = require('@pioneer-platform/loggerdog')();
+const TAG = ' | update-asset-data | ';
+
+interface AssetData {
+  [key: string]: {
+    assetId: string;
+    icon?: string;
+    name?: string;
+    symbol?: string;
+    [key: string]: any;
+  };
+}
+
+interface DownloadManifest {
+  [caipId: string]: {
+    filename: string;
+    originalUrl: string;
+    status: 'downloaded' | 'cached' | 'failed';
+    filesize?: number;
+    symbol?: string;
+    name?: string;
+  };
+}
+
+interface VerificationResult {
+  timestamp: string;
+  baseUrl: string;
+  stats: {
+    total: number;
+    ok: number;
+    broken: number;
+    timeout: number;
+    error: number;
+  };
+  results: Array<{
+    caipId: string;
+    url: string;
+    status: 'ok' | 'broken' | 'timeout' | 'error';
+  }>;
+}
+
+// Configuration
+const ROOT_DIR = path.join(__dirname, '..', '..');
+const DATA_FILE = path.join(ROOT_DIR, 'src/generatedAssetData.json');
+const BACKUP_FILE = path.join(ROOT_DIR, 'src/generatedAssetData.backup.json');
+const MANIFEST_FILE = path.join(ROOT_DIR, 'audit-results/download-manifest.json');
+const VERIFICATION_FILE = path.join(ROOT_DIR, 'audit-results/verification-results.json');
+const BASE_URL = 'https://api.keepkey.info/coins';
+
+async function updateAssetData() {
+  log.info(TAG, 'Starting asset data update...');
+
+  // Check verification results first
+  if (!fs.existsSync(VERIFICATION_FILE)) {
+    log.error(TAG, 'Verification results not found. Run verify-deployment.ts first!');
+    process.exit(1);
+  }
+
+  const verification: VerificationResult = JSON.parse(fs.readFileSync(VERIFICATION_FILE, 'utf-8'));
+  const successRate = verification.stats.total > 0
+    ? Math.round((verification.stats.ok / verification.stats.total) * 100)
+    : 0;
+
+  log.info(TAG, `Verification success rate: ${successRate}% (${verification.stats.ok}/${verification.stats.total})`);
+
+  if (successRate < 95) {
+    log.error(TAG, `Success rate too low (${successRate}%). Fix broken URLs first!`);
+    log.error(TAG, 'Aborting update to prevent data corruption.');
+    process.exit(1);
+  }
+
+  if (successRate < 100) {
+    log.warn(TAG, `Warning: ${verification.stats.broken + verification.stats.timeout + verification.stats.error} URLs still broken.`);
+    log.warn(TAG, 'Continue anyway? (only verified URLs will be updated)');
+  }
+
+  // Read manifest
+  if (!fs.existsSync(MANIFEST_FILE)) {
+    log.error(TAG, 'Download manifest not found. Run download-images-v2.ts first.');
+    process.exit(1);
+  }
+
+  const manifest: DownloadManifest = JSON.parse(fs.readFileSync(MANIFEST_FILE, 'utf-8'));
+
+  // Create verified URL map
+  const verifiedUrls = new Map<string, string>();
+  for (const result of verification.results) {
+    if (result.status === 'ok') {
+      verifiedUrls.set(result.caipId, result.url);
+    }
+  }
+
+  log.info(TAG, `${verifiedUrls.size} verified URLs ready for update`);
+
+  // Read asset data
+  const assetData: AssetData = JSON.parse(fs.readFileSync(DATA_FILE, 'utf-8'));
+
+  // Create backup
+  log.info(TAG, 'Creating backup...');
+  fs.copyFileSync(DATA_FILE, BACKUP_FILE);
+  log.info(TAG, `Backup saved to: ${BACKUP_FILE}`);
+
+  // Track statistics
+  const stats = {
+    total: 0,
+    updated: 0,
+    skipped: 0,
+    failed: 0
+  };
+
+  // Update asset data
+  log.info(TAG, 'Updating asset data...');
+
+  for (const [assetId, asset] of Object.entries(assetData)) {
+    stats.total++;
+
+    // Check if we have a verified URL for this asset
+    const verifiedUrl = verifiedUrls.get(assetId);
+
+    if (verifiedUrl) {
+      asset.icon = verifiedUrl;
+      stats.updated++;
+    } else if (manifest[assetId]) {
+      // Asset was processed but not verified (failed or broken)
+      log.debug(TAG, `Skipping ${asset.symbol} (${assetId}) - not verified`);
+      stats.skipped++;
+    } else {
+      // Asset was never processed
+      stats.failed++;
+    }
+  }
+
+  // Save updated asset data
+  log.info(TAG, 'Saving updated asset data...');
+  fs.writeFileSync(DATA_FILE, JSON.stringify(assetData, null, 2));
+
+  // Generate summary
+  log.info(TAG, '\nUpdate Summary:');
+  log.info(TAG, `  Total assets: ${stats.total}`);
+  log.info(TAG, `  ✅ Updated: ${stats.updated}`);
+  log.info(TAG, `  ⏭️  Skipped (not verified): ${stats.skipped}`);
+  log.info(TAG, `  ❌ Failed (not processed): ${stats.failed}`);
+
+  const updateRate = Math.round((stats.updated / stats.total) * 100);
+  log.info(TAG, `\nUpdate rate: ${updateRate}%`);
+
+  // Save update stats
+  const updateStats = {
+    timestamp: new Date().toISOString(),
+    baseUrl: BASE_URL,
+    stats,
+    backupFile: BACKUP_FILE
+  };
+
+  fs.writeFileSync(
+    path.join(ROOT_DIR, 'audit-results/update-stats.json'),
+    JSON.stringify(updateStats, null, 2)
+  );
+
+  log.info(TAG, '\n✅ Asset data updated successfully!');
+  log.info(TAG, `Backup available at: ${BACKUP_FILE}`);
+  log.info(TAG, '\nTo rollback: cp src/generatedAssetData.backup.json src/generatedAssetData.json');
+
+  // Verify update
+  log.info(TAG, '\nVerifying update...');
+  const verifyCount = Object.values(assetData).filter(a =>
+    a.icon?.startsWith(BASE_URL)
+  ).length;
+  log.info(TAG, `${verifyCount} assets now using ${BASE_URL}`);
+
+  return updateStats;
+}
+
+// Run update
+updateAssetData().catch(error => {
+  log.error(TAG, 'Update failed:', error);
+  process.exit(1);
+});

package/scripts/verify-deployment.d.ts

@@ -0,0 +1 @@
+export {};

package/scripts/verify-deployment.js

@@ -0,0 +1,283 @@
+"use strict";
+/*
+ * Deployment Verification Script
+ * Verifies images are accessible at api.keepkey.info/coins/
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __generator = (this && this.__generator) || function (thisArg, body) {
+    var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
+    return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
+    function verb(n) { return function (v) { return step([n, v]); }; }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while (g && (g = 0, op[0] && (_ = 0)), _) try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [op[0] & 2, t.value];
+            switch (op[0]) {
+                case 0: case 1: t = op; break;
+                case 4: _.label++; return { value: op[1], done: false };
+                case 5: _.label++; y = op[1]; op = [0]; continue;
+                case 7: op = _.ops.pop(); _.trys.pop(); continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
+                    if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
+                    if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
+                    if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop(); continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
+        if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
+    }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+var fs = __importStar(require("fs"));
+var path = __importStar(require("path"));
+var log = require('@pioneer-platform/loggerdog')();
+var TAG = ' | verify-deployment | ';
+// Configuration
+var CONCURRENT_REQUESTS = 10;
+var REQUEST_TIMEOUT = 5000;
+var ROOT_DIR = path.join(__dirname, '..', '..');
+var MANIFEST_FILE = path.join(ROOT_DIR, 'audit-results/download-manifest.json');
+var OUTPUT_DIR = path.join(ROOT_DIR, 'audit-results');
+var BASE_URL = 'https://api.keepkey.info/coins';
+function testUrl(url) {
+    return __awaiter(this, void 0, void 0, function () {
+        var controller_1, timeoutId, response, error_1;
+        return __generator(this, function (_a) {
+            switch (_a.label) {
+                case 0:
+                    _a.trys.push([0, 2, , 3]);
+                    controller_1 = new AbortController();
+                    timeoutId = setTimeout(function () { return controller_1.abort(); }, REQUEST_TIMEOUT);
+                    return [4 /*yield*/, fetch(url, {
+                            method: 'HEAD',
+                            signal: controller_1.signal,
+                            headers: {
+                                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36'
+                            }
+                        })];
+                case 1:
+                    response = _a.sent();
+                    clearTimeout(timeoutId);
+                    return [2 /*return*/, { status: response.status }];
+                case 2:
+                    error_1 = _a.sent();
+                    if (error_1.name === 'AbortError') {
+                        return [2 /*return*/, { status: null, error: 'timeout' }];
+                    }
+                    return [2 /*return*/, { status: null, error: error_1.message }];
+                case 3: return [2 /*return*/];
+            }
+        });
+    });
+}
+function verifyDeployment() {
+    return __awaiter(this, void 0, void 0, function () {
+        var manifest, entries, stats, results, i, batch, batchResults, progress, successRate, verificationData, brokenResults, report, _i, _a, result;
+        var _this = this;
+        return __generator(this, function (_b) {
+            switch (_b.label) {
+                case 0:
+                    log.info(TAG, "Verifying deployment at ".concat(BASE_URL, "..."));
+                    // Read manifest
+                    if (!fs.existsSync(MANIFEST_FILE)) {
+                        log.error(TAG, 'Download manifest not found. Run download-images-v2.ts first.');
+                        process.exit(1);
+                    }
+                    manifest = JSON.parse(fs.readFileSync(MANIFEST_FILE, 'utf-8'));
+                    entries = Object.entries(manifest);
+                    log.info(TAG, "Testing ".concat(entries.length, " images..."));
+                    stats = {
+                        total: 0,
+                        ok: 0,
+                        broken: 0,
+                        timeout: 0,
+                        error: 0,
+                        skipped: 0
+                    };
+                    results = [];
+                    i = 0;
+                    _b.label = 1;
+                case 1:
+                    if (!(i < entries.length)) return [3 /*break*/, 4];
+                    batch = entries.slice(i, i + CONCURRENT_REQUESTS);
+                    return [4 /*yield*/, Promise.all(batch.map(function (_a) { return __awaiter(_this, [_a], void 0, function (_b) {
+                            var url, result, status;
+                            var caipId = _b[0], info = _b[1];
+                            return __generator(this, function (_c) {
+                                switch (_c.label) {
+                                    case 0:
+                                        // Skip failed downloads
+                                        if (info.status === 'failed') {
+                                            stats.skipped++;
+                                            return [2 /*return*/, null];
+                                        }
+                                        stats.total++;
+                                        url = "".concat(BASE_URL, "/").concat(info.filename);
+                                        return [4 /*yield*/, testUrl(url)];
+                                    case 1:
+                                        result = _c.sent();
+                                        if (result.status === 200) {
+                                            status = 'ok';
+                                            stats.ok++;
+                                        }
+                                        else if (result.error === 'timeout') {
+                                            status = 'timeout';
+                                            stats.timeout++;
+                                        }
+                                        else if (result.status) {
+                                            status = 'broken';
+                                            stats.broken++;
+                                        }
+                                        else {
+                                            status = 'error';
+                                            stats.error++;
+                                        }
+                                        return [2 /*return*/, {
+                                                caipId: caipId,
+                                                url: url,
+                                                status: status,
+                                                statusCode: result.status || undefined,
+                                                symbol: info.symbol,
+                                                error: result.error
+                                            }];
+                                }
+                            });
+                        }); }))];
+                case 2:
+                    batchResults = _b.sent();
+                    results.push.apply(results, batchResults.filter(function (r) { return r !== null; }));
+                    progress = Math.min(i + CONCURRENT_REQUESTS, entries.length);
+                    if (progress % 100 === 0 || progress === entries.length) {
+                        log.info(TAG, "Progress: ".concat(progress, "/").concat(entries.length, " (").concat(Math.round(progress / entries.length * 100), "%)"));
+                    }
+                    _b.label = 3;
+                case 3:
+                    i += CONCURRENT_REQUESTS;
+                    return [3 /*break*/, 1];
+                case 4:
+                    successRate = stats.total > 0 ? Math.round((stats.ok / stats.total) * 100) : 0;
+                    log.info(TAG, '\nVerification Summary:');
+                    log.info(TAG, "  Total tested: ".concat(stats.total));
+                    log.info(TAG, "  \u2705 OK: ".concat(stats.ok, " (").concat(successRate, "%)"));
+                    log.info(TAG, "  \u274C Broken: ".concat(stats.broken));
+                    log.info(TAG, "  \u23F1\uFE0F  Timeout: ".concat(stats.timeout));
+                    log.info(TAG, "  \u26A0\uFE0F  Error: ".concat(stats.error));
+                    log.info(TAG, "  \u23ED\uFE0F  Skipped: ".concat(stats.skipped));
+                    verificationData = {
+                        timestamp: new Date().toISOString(),
+                        baseUrl: BASE_URL,
+                        stats: stats,
+                        results: results
+                    };
+                    fs.writeFileSync(path.join(OUTPUT_DIR, 'verification-results.json'), JSON.stringify(verificationData, null, 2));
+                    brokenResults = results.filter(function (r) { return r.status !== 'ok'; });
+                    fs.writeFileSync(path.join(OUTPUT_DIR, 'verification-broken.json'), JSON.stringify(brokenResults, null, 2));
+                    report = [
+                        '# Deployment Verification Report',
+                        "Generated: ".concat(new Date().toISOString()),
+                        "Base URL: ".concat(BASE_URL),
+                        '',
+                        '## Summary',
+                        "- Total tested: ".concat(stats.total),
+                        "- \u2705 Working: ".concat(stats.ok, " (").concat(successRate, "%)"),
+                        "- \u274C Broken: ".concat(stats.broken),
+                        "- \u23F1\uFE0F Timeout: ".concat(stats.timeout),
+                        "- \u26A0\uFE0F Error: ".concat(stats.error),
+                        "- \u23ED\uFE0F Skipped: ".concat(stats.skipped),
+                        ''
+                    ];
+                    if (successRate === 100) {
+                        report.push('## ✅ SUCCESS - All images verified!');
+                        report.push('');
+                        report.push('Next step: Run update-asset-data.ts to update generatedAssetData.json');
+                    }
+                    else {
+                        report.push('## ⚠️ WARNING - Some images failed verification');
+                        report.push('');
+                        report.push('Please fix broken URLs before updating generatedAssetData.json');
+                    }
+                    if (brokenResults.length > 0) {
+                        report.push('');
+                        report.push('## Broken URLs');
+                        report.push('');
+                        for (_i = 0, _a = brokenResults.slice(0, 50); _i < _a.length; _i++) {
+                            result = _a[_i];
+                            report.push("- ".concat(result.url));
+                            report.push("  - Symbol: ".concat(result.symbol));
+                            report.push("  - CAIP: ".concat(result.caipId));
+                            report.push("  - Status: ".concat(result.status, " (").concat(result.statusCode || 'N/A', ")"));
+                            if (result.error)
+                                report.push("  - Error: ".concat(result.error));
+                        }
+                        if (brokenResults.length > 50) {
+                            report.push("... and ".concat(brokenResults.length - 50, " more (see verification-broken.json)"));
+                        }
+                    }
+                    fs.writeFileSync(path.join(OUTPUT_DIR, 'verification-report.md'), report.join('\n'));
+                    log.info(TAG, "\nResults saved to ".concat(OUTPUT_DIR, "/"));
+                    log.info(TAG, '  - verification-results.json: Complete results');
+                    log.info(TAG, '  - verification-broken.json: Only broken URLs');
+                    log.info(TAG, '  - verification-report.md: Human-readable report');
+                    if (successRate === 100) {
+                        log.info(TAG, '\n✅ Deployment verified! Ready to update asset data.');
+                        return [2 /*return*/, true];
+                    }
+                    else {
+                        log.warn(TAG, '\n⚠️  Some images failed verification. Fix broken URLs first.');
+                        return [2 /*return*/, false];
+                    }
+                    return [2 /*return*/];
+            }
+        });
+    });
+}
+// Run verification
+verifyDeployment().catch(function (error) {
+    log.error(TAG, 'Verification failed:', error);
+    process.exit(1);
+});