openrxiv-cli 0.0.2 → 0.0.3

package/dist/aws/bucket-explorer.d.ts

@@ -7,6 +7,7 @@ export interface ListOptions {
     batch?: string;
     limit?: number;
     server?: 'biorxiv' | 'medrxiv';
+    output?: string;
 }
 export interface SearchOptions {
     type?: 'pdf' | 'xml' | 'all';

package/dist/aws/bucket-explorer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"bucket-explorer.d.ts","sourceRoot":"","sources":["../../src/aws/bucket-explorer.ts"],"names":[],"mappings":"AAOA;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,GAAE,SAAS,GAAG,SAA8B,GAAG,MAAM,CAUxF;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,IAAI,CAAC;IACnB,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,CAAC;CACxC;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAuE3E;AA+GD,wBAAsB,cAAc,CAClC,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;CAAO,GACnE,OAAO,CAAC,IAAI,CAAC,CAyCf"}
+{"version":3,"file":"bucket-explorer.d.ts","sourceRoot":"","sources":["../../src/aws/bucket-explorer.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,GAAE,SAAS,GAAG,SAA8B,GAAG,MAAM,CAUxF;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,IAAI,CAAC;IACnB,IAAI,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,OAAO,CAAC;CACxC;AAED,wBAAsB,iBAAiB,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CA0J3E;AAuJD,wBAAsB,cAAc,CAClC,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS,CAAA;CAAO,GACnE,OAAO,CAAC,IAAI,CAAC,CAyCf"}

package/dist/aws/bucket-explorer.js

@@ -1,5 +1,6 @@
 import { ListObjectsV2Command, HeadObjectCommand } from '@aws-sdk/client-s3';
 import chalk from 'chalk';
+import { writeFile } from 'fs/promises';
 import { getS3Client } from './config.js';
 import { getFolderStructure } from 'openrxiv-utils';
 import { getDefaultServer } from '../utils/default-server.js';
@@ -19,14 +20,14 @@ export function getBucketName(server = getDefaultServer()) {
 }
 export async function listBucketContent(options) {
     const client = await getS3Client();
-    const { month, batch, limit = 50, server = getDefaultServer() } = options;
+    const { month, batch, limit = 50, server = getDefaultServer(), output } = options;
     const bucketName = getBucketName(server);
     console.log(chalk.blue(`Listing ${server} bucket content...`));
     console.log(chalk.blue('===================================='));
     try {
         // If no month or batch specified, show the available content structure
         if (!month && !batch) {
-            await listFolder(client, server);
+            await listFolder(client, server, output);
             return;
         }
         let prefix = '';
@@ -40,29 +41,93 @@ export async function listBucketContent(options) {
                 console.log(chalk.gray(`🔍 Batch: ${folder.batch}`));
             }
         }
-        const commandOptions = {
-            Bucket: bucketName,
-            Prefix: prefix,
-            MaxKeys: parseInt(limit.toString()),
-            RequestPayer: 'requester',
-        };
-        const command = new ListObjectsV2Command(commandOptions);
-        const response = await client.send(command);
-        if (!response.Contents || response.Contents.length === 0) {
+        // Collect all items across pages
+        const allItems = [];
+        let continuationToken = undefined;
+        const maxKeysPerRequest = 1000; // S3 maximum
+        const requestedLimit = parseInt(limit.toString());
+        let pageNumber = 1;
+        // Prepare data for CSV export if output is specified
+        const csvRows = [];
+        if (output) {
+            csvRows.push('Key,Size (bytes),Date Modified');
+        }
+        // Paginate through all results, respecting the limit
+        do {
+            const remainingItems = requestedLimit - allItems.length;
+            const keysToFetch = Math.min(maxKeysPerRequest, remainingItems);
+            if (keysToFetch <= 0) {
+                break;
+            }
+            // Show progress when exporting to CSV
+            if (output) {
+                console.log(chalk.gray(`📄 Fetching page ${pageNumber}...`));
+            }
+            const commandOptions = {
+                Bucket: bucketName,
+                Prefix: prefix,
+                MaxKeys: keysToFetch,
+                RequestPayer: 'requester',
+            };
+            if (continuationToken) {
+                commandOptions.ContinuationToken = continuationToken;
+            }
+            const command = new ListObjectsV2Command(commandOptions);
+            const response = await client.send(command);
+            if (response.Contents && response.Contents.length > 0) {
+                const validItems = response.Contents.filter((item) => item.Key !== undefined).map((item) => ({
+                    Key: item.Key,
+                    Size: item.Size,
+                    LastModified: item.LastModified,
+                }));
+                const itemsToAdd = validItems.slice(0, remainingItems);
+                allItems.push(...itemsToAdd);
+                if (output) {
+                    console.log(chalk.gray(`   ✓ Page ${pageNumber}: ${itemsToAdd.length} items (Total: ${allItems.length})`));
+                }
+            }
+            if (allItems.length >= requestedLimit) {
+                break;
+            }
+            continuationToken = response.NextContinuationToken;
+            pageNumber++;
+        } while (continuationToken && allItems.length < requestedLimit);
+        if (allItems.length === 0) {
             console.log(chalk.yellow('No content found'));
             return;
         }
-        console.log(chalk.green(`Found ${response.Contents.length} items:`));
-        console.log('');
-        for (const item of response.Contents) {
+        const escapeCsvValue = (value) => {
+            if (value.includes(',') || value.includes('"') || value.includes('\n')) {
+                return `"${value.replace(/"/g, '""')}"`;
+            }
+            return value;
+        };
+        if (!output) {
+            console.log(chalk.green(`Found ${allItems.length} items:`));
+            console.log('');
+        }
+        for (const item of allItems) {
             if (!item.Key)
                 continue;
             const type = getContentType(item.Key);
             const size = formatFileSize(item.Size || 0);
             const date = item.LastModified ? item.LastModified.toLocaleDateString() : 'Unknown';
-            console.log(`${chalk.cyan(item.Key)}`);
-            console.log(`  Type: ${chalk.yellow(type)} | Size: ${chalk.blue(size)} | Modified: ${chalk.gray(date)}`);
-            console.log('');
+            const sizeBytes = item.Size || 0;
+            const dateModified = item.LastModified ? item.LastModified.toISOString() : 'Unknown';
+            if (!output) {
+                console.log(`${chalk.cyan(item.Key)}`);
+                console.log(`  Type: ${chalk.yellow(type)} | Size: ${chalk.blue(size)} | Modified: ${chalk.gray(date)}`);
+                console.log('');
+            }
+            if (output) {
+                csvRows.push(`${escapeCsvValue(item.Key)},${sizeBytes},${escapeCsvValue(dateModified)}`);
+            }
+        }
+        if (output) {
+            const csvContent = csvRows.join('\n');
+            const csvFileName = output.endsWith('.csv') ? output : `${output}.csv`;
+            await writeFile(csvFileName, csvContent, 'utf-8');
+            console.log(chalk.green(`✅ Exported ${allItems.length} items to ${csvFileName}`));
         }
     }
     catch (error) {
@@ -76,15 +141,19 @@ export async function listBucketContent(options) {
  * Lists the available content structure in the specified server bucket
  * Shows available months and batches
  */
-async function listFolder(client, server = getDefaultServer()) {
-    console.log(chalk.cyan('📁 Available Content Structure'));
-    console.log(chalk.cyan('=============================='));
-    console.log('');
-    try {
-        // List Current_Content folders (monthly content)
-        console.log(chalk.blue('📅 Current Content (Monthly):'));
-        console.log(chalk.gray('   Recent content organized by month'));
+async function listFolder(client, server = getDefaultServer(), output) {
+    const folderNames = [];
+    if (!output) {
+        console.log(chalk.cyan('📁 Available Content Structure'));
+        console.log(chalk.cyan('=============================='));
         console.log('');
+    }
+    try {
+        if (!output) {
+            console.log(chalk.blue('📅 Current Content (Monthly):'));
+            console.log(chalk.gray('   Recent content organized by month'));
+            console.log('');
+        }
         const bucketName = getBucketName(server);
         const currentContentCommand = new ListObjectsV2Command({
             Bucket: bucketName,
@@ -120,17 +189,26 @@ async function listFolder(client, server = getDefaultServer()) {
                 return monthOrder.indexOf(monthB) - monthOrder.indexOf(monthA);
             });
             for (const month of months) {
-                console.log(`   ${chalk.green('📁')} ${chalk.cyan(month)}`);
+                if (output) {
+                    folderNames.push(month);
+                }
+                else {
+                    console.log(`   ${chalk.green('📁')} ${chalk.cyan(month)}`);
+                }
             }
         }
         else {
-            console.log(chalk.gray('   No monthly content found'));
+            if (!output) {
+                console.log(chalk.gray('   No monthly content found'));
+            }
+        }
+        if (!output) {
+            console.log('');
+            // List Back_Content batches
+            console.log(chalk.blue('📦 Back Content (Historical Batches):'));
+            console.log(chalk.gray('   Legacy content organized in batches'));
+            console.log('');
         }
-        console.log('');
-        // List Back_Content batches
-        console.log(chalk.blue('📦 Back Content (Historical Batches):'));
-        console.log(chalk.gray('   Legacy content organized in batches'));
-        console.log('');
         const backContentCommand = new ListObjectsV2Command({
             Bucket: bucketName,
             Prefix: 'Back_Content/',
@@ -144,24 +222,48 @@ async function listFolder(client, server = getDefaultServer()) {
                 .filter(Boolean)
                 .sort();
             for (const batch of batches) {
-                console.log(`   ${chalk.green('📁')} ${chalk.cyan(batch)}`);
+                if (output) {
+                    folderNames.push(batch);
+                }
+                else {
+                    console.log(`   ${chalk.green('📁')} ${chalk.cyan(batch)}`);
+                }
             }
         }
         else {
-            console.log(chalk.gray('   No historical batches found'));
+            if (!output) {
+                console.log(chalk.gray('   No historical batches found'));
+            }
+        }
+        if (!output) {
+            console.log('');
+            console.log(chalk.blue('💡 Usage Examples:'));
+            console.log(chalk.gray(`   List specific month:\t${server} list --month 2024-01`));
+            console.log(chalk.gray(`   List specific batch:\t${server} list --batch Batch_01`));
+            console.log(chalk.gray(`   List with limit:\t${server} list --month 2024-01 --limit 100`));
+            console.log(chalk.gray(`   File listing (CSV):\t${server} list -m 2025-01 --limit 10000 -o 2025-01.csv`));
+            console.log(chalk.gray(`   Folder overview:\t${server} list -o folders.txt`));
+            console.log('');
+        }
+        else {
+            const textContent = folderNames.join('\n');
+            await writeFile(output, textContent, 'utf-8');
+            console.log(chalk.green(`✅ Exported ${folderNames.length} folders to ${output}`));
         }
-        console.log('');
-        console.log(chalk.blue('💡 Usage Examples:'));
-        console.log(chalk.gray(`   List specific month: ${server} list --month 2024-01`));
-        console.log(chalk.gray(`   List specific batch: ${server} list --batch Batch_01`));
-        console.log(chalk.gray(`   List with limit: ${server} list --month 2024-01 --limit 100`));
-        console.log('');
     }
     catch (error) {
         if (error instanceof Error) {
-            console.log(chalk.yellow(`⚠️  Warning: Could not fetch content structure: ${error.message}`));
-            console.log(chalk.gray('   This may be due to AWS permissions or network issues'));
-            console.log('');
+            if (output) {
+                throw new Error(`Failed to list folder structure: ${error.message}`);
+            }
+            else {
+                console.log(chalk.yellow(`⚠️  Warning: Could not fetch content structure: ${error.message}`));
+                console.log(chalk.gray('   This may be due to AWS permissions or network issues'));
+                console.log('');
+            }
+        }
+        else {
+            throw error;
         }
     }
 }

package/dist/commands/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/commands/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,eAAO,MAAM,eAAe,SA0ExB,CAAC"}
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/commands/download.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAMpC,eAAO,MAAM,eAAe,SA+ExB,CAAC"}

package/dist/commands/download.js

@@ -5,7 +5,7 @@ import { setGlobalRequesterPays } from '../aws/config.js';
 import { displayRequesterPaysError } from '../utils/requester-pays-error.js';
 export const downloadCommand = new Command('download')
     .description('Download MECA files from the bioRxiv/medRxiv S3 bucket by DOI')
-    .argument('<doi>', 'DOI of the paper (e.g., "10.1101/2024.01.15.123456")')
+    .argument('<doi>', 'DOI of the paper (e.g., "10.1101/2024.01.15.123456" or "10.64898/2025.12.15.123456")')
     .option('-o, --output <dir>', 'Output directory for downloaded files', './downloads')
     .option('-a, --api-url <url>', 'API base URL', 'https://openrxiv.csf.now')
     .option('--requester-pays', 'Enable requester-pays for S3 bucket access')
@@ -14,7 +14,7 @@ export const downloadCommand = new Command('download')
     try {
         // Validate DOI format
         if (!doi.includes('/')) {
-            console.error('❌ Invalid DOI format. Expected format: 10.1101/2024.01.15.123456');
+            console.error('❌ Invalid DOI format. Expected format: 10.1101 or 10.64898/2024.01.15.123456 (or legacy /XXXXXX)');
             process.exit(1);
         }
         // Split DOI into prefix and suffix

package/dist/commands/list.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../src/commands/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,eAAO,MAAM,WAAW,SAapB,CAAC"}
+{"version":3,"file":"list.d.ts","sourceRoot":"","sources":["../../src/commands/list.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,eAAO,MAAM,WAAW,SAiBpB,CAAC"}

package/dist/commands/list.js

@@ -7,6 +7,7 @@ export const listCommand = new Command('list')
     .option('-b, --batch <batch>', 'Filter by specific batch (e.g., "Batch_01")')
     .option('-l, --limit <number>', 'Limit the number of results', '50')
     .option('-s, --server <server>', 'Server to use: "biorxiv" or "medrxiv"', getDefaultServer())
+    .option('-o, --output <file>', 'Export results to file (CSV for file listings with .csv extension; text for folder overview)')
     .action(async (options) => {
     try {
         await listBucketContent(options);

package/dist/version.d.ts

@@ -1,3 +1,3 @@
-declare const version = "0.0.2";
+declare const version = "0.0.3";
 export default version;
 //# sourceMappingURL=version.d.ts.map

package/dist/version.js

@@ -1,2 +1,2 @@
-const version = '0.0.2';
+const version = '0.0.3';
 export default version;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "openrxiv-cli",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "CLI tool to download openRxiv MECA files from AWS S3 for text and data mining",
   "main": "dist/index.js",
   "files": [
@@ -34,27 +34,27 @@
     "node": ">=18.0.0"
   },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.0.0",
-    "@aws-sdk/s3-request-presigner": "^3.0.0",
-    "axios": "^1.6.0",
-    "openrxiv-utils": "^0.0.2",
+    "@aws-sdk/client-s3": "^3.995.0",
+    "@aws-sdk/s3-request-presigner": "^3.995.0",
+    "axios": "^1.13.5",
+    "openrxiv-utils": "^0.0.3",
     "boxen": "^8.0.1",
     "character-entities": "^2.0.2",
-    "chalk": "^5.0.0",
+    "chalk": "^5.6.2",
     "cli-progress": "^3.12.0",
-    "commander": "^14.0.0",
-    "conf": "^10.0.0",
-    "inquirer": "^9.0.0",
+    "commander": "^14.0.3",
+    "conf": "^15.1.0",
+    "inquirer": "^13.2.5",
     "jszip": "^3.10.1",
-    "ora": "^7.0.0",
-    "adm-zip": "^0.5.10",
-    "unified": "^11.0.0",
+    "ora": "^9.3.0",
+    "adm-zip": "^0.5.16",
+    "unified": "^11.0.5",
     "xast-util-from-xml": "^4.0.0",
-    "p-limit": "^7.0.0"
+    "p-limit": "^7.3.0"
   },
   "devDependencies": {
-    "@types/cli-progress": "^3.11.0",
-    "@types/inquirer": "^9.0.0"
+    "@types/cli-progress": "^3.11.6",
+    "@types/inquirer": "^9.0.9"
   },
   "repository": {
     "type": "git",