openrxiv 0.0.0

package/dist/cli/src/aws/config.js

@@ -0,0 +1,191 @@
+import inquirer from 'inquirer';
+import { S3Client, ListObjectsV2Command } from '@aws-sdk/client-s3';
+import chalk from 'chalk';
+import { homedir } from 'os';
+import { join } from 'path';
+import { readFile, writeFile, mkdir } from 'fs/promises';
+import { existsSync } from 'fs';
+const CONFIG_DIR = join(homedir(), '.biorxiv');
+const CONFIG_FILE = join(CONFIG_DIR, 'config.json');
+// Global configuration for requester pays
+let globalRequesterPays = false; // Default to false (no requester pays)
+export function setGlobalRequesterPays(enabled) {
+    globalRequesterPays = enabled;
+}
+export function getGlobalRequesterPays() {
+    return globalRequesterPays;
+}
+async function ensureConfigDir() {
+    if (!existsSync(CONFIG_DIR)) {
+        await mkdir(CONFIG_DIR, { recursive: true });
+    }
+}
+async function loadConfig() {
+    try {
+        if (existsSync(CONFIG_FILE)) {
+            const data = await readFile(CONFIG_FILE, 'utf-8');
+            return JSON.parse(data);
+        }
+    }
+    catch (error) {
+        // If config file is corrupted, start fresh
+    }
+    return {
+        aws: {
+            region: 'us-east-1',
+        },
+    };
+}
+async function saveConfig(config) {
+    await ensureConfigDir();
+    await writeFile(CONFIG_FILE, JSON.stringify(config, null, 2));
+}
+export async function setCredentials(options) {
+    let { accessKey, secretKey, region } = options;
+    // Prompt for missing credentials
+    if (!accessKey) {
+        const answer = await inquirer.prompt([
+            {
+                type: 'input',
+                name: 'accessKey',
+                message: 'Enter your AWS Access Key ID:',
+                validate: (input) => (input.length > 0 ? true : 'Access Key is required'),
+            },
+        ]);
+        accessKey = answer.accessKey;
+    }
+    if (!secretKey) {
+        const answer = await inquirer.prompt([
+            {
+                type: 'password',
+                name: 'secretKey',
+                message: 'Enter your AWS Secret Access Key:',
+                validate: (input) => (input.length > 0 ? true : 'Secret Key is required'),
+            },
+        ]);
+        secretKey = answer.secretKey;
+    }
+    if (!region) {
+        region = 'us-east-1';
+    }
+    // Save credentials
+    const config = await loadConfig();
+    config.aws.accessKeyId = accessKey;
+    config.aws.secretAccessKey = secretKey;
+    config.aws.region = region;
+    await saveConfig(config);
+    console.log(chalk.green('✓ AWS credentials saved successfully'));
+    console.log(chalk.blue(`Region: ${region}`));
+}
+export async function testConnection() {
+    const credentials = await getCredentials();
+    // If no credentials are provided, try to use instance metadata (EC2 IAM role)
+    if (!credentials.accessKeyId || !credentials.secretAccessKey) {
+        console.log(chalk.yellow('No AWS credentials found in config, attempting to use EC2 instance metadata...'));
+    }
+    const client = new S3Client({
+        region: credentials.region,
+        ...(credentials.accessKeyId && credentials.secretAccessKey
+            ? {
+                credentials: {
+                    accessKeyId: credentials.accessKeyId,
+                    secretAccessKey: credentials.secretAccessKey,
+                },
+            }
+            : {}),
+        requestHandler: {
+            httpOptions: {
+                timeout: 300000, // 5 minutes timeout for large operations
+            },
+        },
+    });
+    try {
+        console.log(chalk.blue('Testing connection to bioRxiv bucket...'));
+        // Test connection by listing a small amount of content
+        const listCommandOptions = {
+            Bucket: 'biorxiv-src-monthly',
+            MaxKeys: 1,
+        };
+        // Only add RequestPayer if requester pays is enabled
+        if (getGlobalRequesterPays()) {
+            listCommandOptions.RequestPayer = 'requester';
+        }
+        const listCommand = new ListObjectsV2Command(listCommandOptions);
+        await client.send(listCommand);
+        console.log(chalk.green('✓ Successfully connected to bioRxiv bucket'));
+        console.log(chalk.blue('Bucket: biorxiv-src-monthly'));
+        console.log(chalk.blue('Region: us-east-1'));
+        if (getGlobalRequesterPays()) {
+            console.log(chalk.blue('Requester-pays: Enabled'));
+            console.log(chalk.yellow('⚠️  You will be charged for S3 requests'));
+        }
+        else {
+            console.log(chalk.blue('Requester-pays: Disabled'));
+            console.log(chalk.green('✓ No charges for S3 requests (using EC2 IAM role or bucket owner pays)'));
+        }
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            if (error.message.includes('Access Denied') && !getGlobalRequesterPays()) {
+                console.log(chalk.red('✗ Access denied. This bucket requires requester pays.'));
+                console.log(chalk.yellow('Try running with --requester-pays flag for local development.'));
+                console.log(chalk.blue('On EC2 with proper IAM role, this should work without the flag.'));
+            }
+            else {
+                throw new Error(`Connection failed: ${error.message}`);
+            }
+        }
+        throw error;
+    }
+}
+export async function showConfig() {
+    const credentials = await getCredentials();
+    console.log(chalk.blue('Current bioRxiv Configuration:'));
+    console.log(chalk.blue('=============================='));
+    if (credentials.accessKeyId) {
+        console.log(chalk.green('✓ AWS Access Key: Configured'));
+        console.log(chalk.blue(`Region: ${credentials.region}`));
+    }
+    else {
+        console.log(chalk.red('✗ AWS credentials not configured'));
+        console.log(chalk.yellow('Run "biorxiv config set-credentials" to configure'));
+    }
+}
+export async function getCredentials() {
+    const config = await loadConfig();
+    return {
+        accessKeyId: config.aws.accessKeyId,
+        secretAccessKey: config.aws.secretAccessKey,
+        region: config.aws.region,
+    };
+}
+export async function getS3Client() {
+    const credentials = await getCredentials();
+    // If no credentials are provided, try to use instance metadata (EC2 IAM role)
+    if (!credentials.accessKeyId || !credentials.secretAccessKey) {
+        console.log(chalk.yellow('No AWS credentials found in config, attempting to use EC2 instance metadata...'));
+        return new S3Client({
+            region: credentials.region,
+            // AWS SDK will automatically use instance metadata service for credentials
+            requestHandler: {
+                httpOptions: {
+                    timeout: 300000, // 5 minutes timeout for large operations
+                },
+            },
+        });
+    }
+    return new S3Client({
+        region: credentials.region,
+        credentials: {
+            accessKeyId: credentials.accessKeyId,
+            secretAccessKey: credentials.secretAccessKey,
+        },
+        // bioRxiv bucket is requester-pays, so we need to configure the client accordingly
+        requestHandler: {
+            // This ensures the client can handle requester-pays operations
+            httpOptions: {
+                timeout: 300000, // 5 minutes timeout for large operations
+            },
+        },
+    });
+}

package/dist/cli/src/aws/downloader.d.ts

@@ -0,0 +1,13 @@
+export interface DownloadOptions {
+    output?: string;
+    filename?: string;
+    server?: 'biorxiv' | 'medrxiv';
+}
+export interface DownloadProgress {
+    downloaded: number;
+    total: number;
+    speed: number;
+    eta: number;
+}
+export declare function downloadFile(path: string, options: DownloadOptions): Promise<void>;
+//# sourceMappingURL=downloader.d.ts.map

package/dist/cli/src/aws/downloader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"downloader.d.ts","sourceRoot":"","sources":["../../../../src/aws/downloader.ts"],"names":[],"mappings":"AAaA,MAAM,WAAW,eAAe;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CAChC;AAED,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;CACb;AAED,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAwHxF"}

package/dist/cli/src/aws/downloader.js

@@ -0,0 +1,115 @@
+import { GetObjectCommand, HeadObjectCommand } from '@aws-sdk/client-s3';
+import { createWriteStream } from 'fs';
+import { mkdir } from 'fs/promises';
+import { dirname, join } from 'path';
+import { pipeline } from 'stream/promises';
+import { Transform } from 'stream';
+import chalk from 'chalk';
+import ora from 'ora';
+import cliProgress from 'cli-progress';
+import { getS3Client, getGlobalRequesterPays } from './config.js';
+import { getDefaultServer } from '../utils/default-server.js';
+import { getBucketName } from './bucket-explorer.js';
+export async function downloadFile(path, options) {
+    const { output = './downloads', server = getDefaultServer() } = options;
+    const bucket = getBucketName(server);
+    const client = await getS3Client();
+    console.log(chalk.blue(`Downloading: ${path}`));
+    console.log(chalk.blue('=============================='));
+    try {
+        // Get file metadata
+        const headCommandOptions = {
+            Bucket: bucket,
+            Key: path,
+        };
+        // Only add RequestPayer if requester pays is enabled
+        if (getGlobalRequesterPays()) {
+            headCommandOptions.RequestPayer = 'requester';
+        }
+        const headCommand = new HeadObjectCommand(headCommandOptions);
+        const metadata = await client.send(headCommand);
+        const fileSize = metadata.ContentLength || 0;
+        const fileName = options.filename || path.split('/').pop() || 'unknown';
+        const outputPath = join(output, fileName);
+        // Create output directory
+        await mkdir(dirname(outputPath), { recursive: true });
+        // Start download
+        const spinner = ora('Preparing download...').start();
+        const getCommandOptions = {
+            Bucket: bucket,
+            Key: path,
+        };
+        // Only add RequestPayer if requester pays is enabled
+        if (getGlobalRequesterPays()) {
+            getCommandOptions.RequestPayer = 'requester';
+        }
+        const getCommand = new GetObjectCommand(getCommandOptions);
+        const response = await client.send(getCommand);
+        if (!response.Body) {
+            throw new Error('No file content received');
+        }
+        spinner.succeed('Download started');
+        // Create progress bar
+        const progressBar = new cliProgress.SingleBar({
+            format: 'Downloading |{bar}| {percentage}% | {value}/{total} bytes | Speed: {speed} | ETA: {eta}',
+            barCompleteChar: '\u2588',
+            barIncompleteChar: '\u2591',
+            hideCursor: true,
+        });
+        progressBar.start(fileSize, 0);
+        let downloadedBytes = 0;
+        const startTime = Date.now();
+        // Create transform stream to track progress
+        const progressStream = new (class extends Transform {
+            constructor() {
+                super();
+            }
+            _transform(chunk, encoding, callback) {
+                downloadedBytes += chunk.length;
+                const elapsed = (Date.now() - startTime) / 1000;
+                const speed = downloadedBytes / elapsed;
+                const eta = (fileSize - downloadedBytes) / speed;
+                progressBar.update(downloadedBytes);
+                callback(null, chunk);
+            }
+        })();
+        // Download file
+        const writeStream = createWriteStream(outputPath);
+        await pipeline(response.Body, progressStream, writeStream);
+        progressBar.stop();
+        console.log(chalk.green(`✓ Download completed: ${outputPath}`));
+        console.log(chalk.blue(`File size: ${formatFileSize(fileSize)}`));
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            // Check for specific AWS errors that indicate requester pays is needed
+            if (error.message.includes('Access Denied') || error.message.includes('403')) {
+                if (!getGlobalRequesterPays()) {
+                    throw new Error(`Download failed: Access denied. This bucket requires requester pays for downloads. ` +
+                        `Try running with --requester-pays flag or ensure your IAM role has requester pays permissions.`);
+                }
+                else {
+                    throw new Error(`Download failed: Access denied. Check your AWS credentials and permissions.`);
+                }
+            }
+            else if (error.message.includes('NoSuchKey')) {
+                throw new Error(`Download failed: File not found in S3 bucket.`);
+            }
+            else if (error.message.includes('NoSuchBucket')) {
+                throw new Error(`Download failed: S3 bucket not found.`);
+            }
+            else {
+                throw new Error(`Download failed: ${error.message}`);
+            }
+        }
+        throw error;
+    }
+}
+function formatFileSize(bytes) {
+    if (bytes === 0)
+        return '0 B';
+    const k = 1024;
+    const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+}

package/dist/cli/src/aws/month-lister.d.ts

@@ -0,0 +1,18 @@
+export interface S3FileInfo {
+    s3Bucket: string;
+    s3Key: string;
+    fileSize: number;
+    lastModified: Date;
+    batch: string;
+}
+export interface ListMonthOptions {
+    month?: string;
+    batch?: string;
+    server?: 'biorxiv' | 'medrxiv';
+    limit?: number;
+}
+/**
+ * Lists MECA files in S3 for a specific month with pagination support
+ */
+export declare function listMonthFiles(options: ListMonthOptions): Promise<S3FileInfo[]>;
+//# sourceMappingURL=month-lister.d.ts.map

package/dist/cli/src/aws/month-lister.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"month-lister.d.ts","sourceRoot":"","sources":["../../../../src/aws/month-lister.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,IAAI,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CAoGrF"}

package/dist/cli/src/aws/month-lister.js

@@ -0,0 +1,90 @@
+import { ListObjectsV2Command } from '@aws-sdk/client-s3';
+import { getS3Client } from './config.js';
+import { getFolderStructure } from 'biorxiv-utils';
+import { getDefaultServer } from '../utils/default-server.js';
+import { getBucketName } from './bucket-explorer.js';
+/**
+ * Lists MECA files in S3 for a specific month with pagination support
+ */
+export async function listMonthFiles(options) {
+    const { month, batch, limit = 1000, server = getDefaultServer() } = options;
+    const awsBucket = getBucketName(server);
+    if (!month && !batch) {
+        throw new Error('Either month or batch must be specified');
+    }
+    const description = month ? `month: ${month}` : `batch: ${batch}`;
+    console.log(`🔍 Listing files for ${description} from AWS S3 bucket: ${awsBucket}`);
+    try {
+        const s3Client = await getS3Client();
+        // Determine folder structure based on options
+        const folder = getFolderStructure({ month, batch, server: options.server || 'biorxiv' });
+        const s3Prefix = folder.prefix;
+        console.log(`🔍 Content Type: ${folder.type === 'current' ? 'Current Content' : 'Back Content'}`);
+        if (folder.batch) {
+            console.log(`🔍 Batch: ${folder.batch}`);
+        }
+        console.log(`🔍 Searching S3 prefix: ${s3Prefix}`);
+        const allFiles = [];
+        let continuationToken;
+        let batchCount = 0;
+        // Use pagination to get all files
+        do {
+            batchCount++;
+            console.log(`📦 Fetching batch ${batchCount}...`);
+            const listCommand = new ListObjectsV2Command({
+                Bucket: awsBucket,
+                Prefix: s3Prefix,
+                MaxKeys: Math.min(1000, limit - allFiles.length), // Don't fetch more than we need
+                ContinuationToken: continuationToken,
+                RequestPayer: 'requester',
+            });
+            const response = await s3Client.send(listCommand);
+            if (response.Contents) {
+                for (const item of response.Contents) {
+                    if (!item.Key || !item.Size)
+                        continue;
+                    // Only process .meca files
+                    if (!item.Key.endsWith('.meca'))
+                        continue;
+                    // Extract S3 file information
+                    const s3Key = item.Key;
+                    const fileSize = item.Size;
+                    const lastModified = item.LastModified || new Date();
+                    const fileInfo = {
+                        s3Bucket: awsBucket,
+                        s3Key: s3Key, // This is already the full path from S3
+                        fileSize: fileSize,
+                        lastModified: lastModified,
+                        batch: folder.batch,
+                    };
+                    allFiles.push(fileInfo);
+                    // Check if we've reached the limit
+                    if (allFiles.length >= limit) {
+                        console.log(`📋 Reached limit of ${limit} files`);
+                        break;
+                    }
+                }
+                console.log(`   Found ${response.Contents.length} files in this batch`);
+            }
+            continuationToken = response.NextContinuationToken;
+            // Break if we've reached the limit
+            if (allFiles.length >= limit) {
+                break;
+            }
+        } while (continuationToken);
+        console.log(`📋 Found ${allFiles.length} MECA files in S3 bucket`);
+        return allFiles;
+    }
+    catch (error) {
+        if (error instanceof Error) {
+            console.error(`❌ Error listing S3 files: ${error.message}`);
+            if (error.message.includes('AWS credentials not configured')) {
+                console.error('💡 Run "biorxiv config set-credentials" to configure AWS access');
+            }
+        }
+        else {
+            console.error('❌ Unknown error listing S3 files:', error);
+        }
+        return [];
+    }
+}

package/dist/cli/src/commands/batch-process.d.ts

@@ -0,0 +1,3 @@
+import { Command } from 'commander';
+export declare const batchProcessCommand: Command;
+//# sourceMappingURL=batch-process.d.ts.map

package/dist/cli/src/commands/batch-process.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"batch-process.d.ts","sourceRoot":"","sources":["../../../../src/commands/batch-process.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAU,MAAM,WAAW,CAAC;AAwC5C,eAAO,MAAM,mBAAmB,SA0Q5B,CAAC"}