openrxiv 0.0.0

package/dist/utils/src/biorxiv-parser.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Utility functions for parsing bioRxiv URLs and DOIs
+ */
+export interface ParsedBiorxivURL {
+    doi: string;
+    baseDOI: string;
+    version: string | null;
+    fullURL: string;
+    isValid: boolean;
+}
+export interface DOIParts {
+    doi: string;
+    prefix: string;
+    suffix: string;
+    date: string | null;
+    identifier: string;
+    version: string | null;
+}
+/**
+ * Extract DOI from a bioRxiv URL
+ */
+export declare function extractDOIFromURL(url: string): string | null;
+/**
+ * Parse a bioRxiv DOI into its components
+ * Supports both legacy numeric format (2019 and earlier) and current date-based format (2019+)
+ */
+export declare function parseDOI(doi: string): DOIParts | null;
+/**
+ * Extract base DOI (without version)
+ * Works with both legacy numeric and current date-based formats
+ */
+export declare function extractBaseDOI(doi: string): string;
+/**
+ * Extract version from DOI
+ * Works with both legacy numeric and current date-based formats
+ */
+export declare function extractVersion(doi: string): string | null;
+/**
+ * Check if a DOI is a valid bioRxiv DOI
+ * Supports both legacy numeric and current date-based formats
+ */
+export declare function isValidBiorxivDOI(doi: string): boolean;
+/**
+ * Check if a URL is a valid bioRxiv URL
+ */
+export declare function isValidBiorxivURL(url: string): boolean;
+/**
+ * Parse a bioRxiv URL and extract all relevant information
+ */
+export declare function parseBiorxivURL(url: string): ParsedBiorxivURL | null;
+//# sourceMappingURL=biorxiv-parser.d.ts.map

package/dist/utils/src/biorxiv-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"biorxiv-parser.d.ts","sourceRoot":"","sources":["../../../../utils/src/biorxiv-parser.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,QAAQ;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAoC5D;AAED;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAsCrD;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAGlD;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAGzD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAEtD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAGtD;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAiBpE"}

package/dist/utils/src/biorxiv-parser.js

@@ -0,0 +1,126 @@
+/**
+ * Utility functions for parsing bioRxiv URLs and DOIs
+ */
+/**
+ * Extract DOI from a bioRxiv URL
+ */
+export function extractDOIFromURL(url) {
+    // Handle various bioRxiv URL formats
+    let doi = null;
+    // Check for bioRxiv content URLs
+    if (url.includes('biorxiv.org/content/')) {
+        const match = url.match(/biorxiv\.org\/content\/([^?#]+)/);
+        if (match && match[1]) {
+            doi = match[1];
+        }
+    }
+    // Check for medRxiv content URLs
+    else if (url.includes('medrxiv.org/content/')) {
+        const match = url.match(/medrxiv\.org\/content\/([^?#]+)/);
+        if (match && match[1]) {
+            doi = match[1];
+        }
+    }
+    // Check for doi.org redirects
+    else if (url.includes('doi.org/')) {
+        const match = url.match(/doi\.org\/([^?#]+)/);
+        if (match && match[1]) {
+            doi = match[1];
+        }
+    }
+    // Check for direct DOI input
+    else if (url.startsWith('10.1101/')) {
+        doi = url;
+    }
+    if (doi) {
+        // Clean up the extracted DOI (remove any trailing extensions)
+        return doi.replace(/\.(article-info|full|abstract|pdf|suppl)$/, '');
+    }
+    return null;
+}
+/**
+ * Parse a bioRxiv DOI into its components
+ * Supports both legacy numeric format (2019 and earlier) and current date-based format (2019+)
+ */
+export function parseDOI(doi) {
+    // Handle current date-based format (2019+): 10.1101/YYYY.MM.DD.XXXXXXvN
+    const currentPattern = /^10\.1101\/(\d{4})\.(\d{2})\.(\d{2})\.(\d{6,8})(v\d+)?$/;
+    const currentMatch = doi.match(currentPattern);
+    if (currentMatch) {
+        const [prefix, suffix] = doi.split('/');
+        const [, year, month, day, identifier, version] = currentMatch;
+        const date = `${year}-${month}-${day}`;
+        return {
+            doi,
+            prefix,
+            suffix: suffix.replace(/(v\d+)$/, ''),
+            date,
+            identifier,
+            version: version || null,
+        };
+    }
+    // Handle legacy numeric format (2019 and earlier): 10.1101/XXXXXX
+    const legacyPattern = /^10\.1101\/(\d{6,8})(v\d+)?$/;
+    const legacyMatch = doi.match(legacyPattern);
+    if (legacyMatch) {
+        const [prefix, suffix] = doi.split('/');
+        const [, identifier, version] = legacyMatch;
+        return {
+            doi,
+            prefix,
+            suffix: suffix.replace(/(v\d+)$/, ''),
+            date: null,
+            identifier,
+            version: version || null,
+        };
+    }
+    return null;
+}
+/**
+ * Extract base DOI (without version)
+ * Works with both legacy numeric and current date-based formats
+ */
+export function extractBaseDOI(doi) {
+    // Remove version suffix if present
+    return doi.replace(/v\d+$/, '');
+}
+/**
+ * Extract version from DOI
+ * Works with both legacy numeric and current date-based formats
+ */
+export function extractVersion(doi) {
+    const match = doi.match(/v(\d+)$/);
+    return match ? match[1] : null;
+}
+/**
+ * Check if a DOI is a valid bioRxiv DOI
+ * Supports both legacy numeric and current date-based formats
+ */
+export function isValidBiorxivDOI(doi) {
+    return parseDOI(doi) !== null;
+}
+/**
+ * Check if a URL is a valid bioRxiv URL
+ */
+export function isValidBiorxivURL(url) {
+    const doi = extractDOIFromURL(url);
+    return doi !== null && isValidBiorxivDOI(doi);
+}
+/**
+ * Parse a bioRxiv URL and extract all relevant information
+ */
+export function parseBiorxivURL(url) {
+    const doi = extractDOIFromURL(url);
+    if (!doi || !isValidBiorxivDOI(doi)) {
+        return null;
+    }
+    const baseDOI = extractBaseDOI(doi);
+    const version = extractVersion(doi);
+    return {
+        doi,
+        baseDOI,
+        version,
+        fullURL: url,
+        isValid: true,
+    };
+}

package/dist/utils/src/folder-structure.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Utility functions for determining bioRxiv folder structure
+ * based on the date requested.
+ *
+ * The bioRxiv structure is:
+ * - Before late 2018: Files are in Back_Content/Batch_[nn]/ folders
+ * - After late 2018: Files are in Current_Content/[Month]_[Year]/ folders
+ */
+export interface FolderStructure {
+    server: 'biorxiv' | 'medrxiv';
+    type: 'current' | 'back';
+    prefix: string;
+    batch: string;
+}
+export interface FolderStructureOptions {
+    server?: 'biorxiv' | 'medrxiv';
+    month?: string;
+    batch?: string;
+}
+/**
+ * Normalizes batch input to the standard "Batch_XX" format
+ * @param batch - Batch input in various formats (e.g., "1", "batch-1", "Batch_01", "batch_01")
+ * @param server - Server type to determine batch format (e.g., "biorxiv", "medrxiv")
+ * @returns Normalized batch string in appropriate format
+ */
+export declare function normalizeBatch(batch: string | number, server?: string): string;
+/**
+ * Determines the folder structure for a given month or batch
+ * @param options - Options containing month or batch
+ * @returns FolderStructure with the appropriate prefix and type
+ */
+export declare function getFolderStructure(options: FolderStructureOptions): FolderStructure;
+export declare function removeDuplicateFolders(folders: FolderStructure[]): FolderStructure[];
+/**
+ * Sort folders chronologically, putting batches before months
+ */
+export declare function sortFoldersChronologically(folders: FolderStructure[]): FolderStructure[];
+/**
+ * Normalizes various month formats to YYYY-MM
+ * @param month - Month in various formats
+ * @returns Normalized YYYY-MM format or null if invalid
+ */
+export declare function normalizeMonthToYYYYMM(month: string): string | null;
+//# sourceMappingURL=folder-structure.d.ts.map

package/dist/utils/src/folder-structure.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"folder-structure.d.ts","sourceRoot":"","sources":["../../../../utils/src/folder-structure.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,SAAS,GAAG,SAAS,CAAC;IAC9B,IAAI,EAAE,SAAS,GAAG,MAAM,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,CAAC,EAAE,SAAS,GAAG,SAAS,CAAC;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,GAAG,MAAM,EACtB,MAAM,GAAE,MAA2B,GAClC,MAAM,CAsCR;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,sBAAsB,GAAG,eAAe,CAuDnF;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CAWpF;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,eAAe,EAAE,CAsBxF;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAuBnE"}

package/dist/utils/src/folder-structure.js

@@ -0,0 +1,207 @@
+/**
+ * Utility functions for determining bioRxiv folder structure
+ * based on the date requested.
+ *
+ * The bioRxiv structure is:
+ * - Before late 2018: Files are in Back_Content/Batch_[nn]/ folders
+ * - After late 2018: Files are in Current_Content/[Month]_[Year]/ folders
+ */
+/**
+ * Normalizes batch input to the standard "Batch_XX" format
+ * @param batch - Batch input in various formats (e.g., "1", "batch-1", "Batch_01", "batch_01")
+ * @param server - Server type to determine batch format (e.g., "biorxiv", "medrxiv")
+ * @returns Normalized batch string in appropriate format
+ */
+export function normalizeBatch(batch, server = getDefaultServer()) {
+    if (typeof batch === 'number') {
+        if (batch < 1) {
+            throw new Error(`Invalid batch format: ${batch}. Expected a positive number or batch identifier.`);
+        }
+        const batchNum = batch.toString().padStart(2, '0');
+        return server.toLowerCase() === 'medrxiv' ? `medRxiv_Batch_${batchNum}` : `Batch_${batchNum}`;
+    }
+    // Remove common prefixes and normalize
+    const normalized = batch
+        .toLowerCase()
+        .replace(/^batch[-_]?/i, '') // Remove "batch", "batch-", "batch_"
+        .replace(/^medrxiv[-_]?batch[-_]?/i, '') // Remove "medrxiv_batch", "medrxiv-batch", etc.
+        .replace(/^0+/, '') // Remove leading zeros
+        .trim();
+    const matchInt = normalized.match(/^\d+$/);
+    if (!matchInt) {
+        throw new Error(`Invalid batch format: ${batch}. Expected a positive number or batch identifier.`);
+    }
+    // Parse the number and format it
+    const batchNum = parseInt(normalized, 10);
+    if (isNaN(batchNum) || batchNum < 1) {
+        throw new Error(`Invalid batch format: ${batch}. Expected a positive number or batch identifier.`);
+    }
+    const formattedBatchNum = batchNum.toString().padStart(2, '0');
+    return server.toLowerCase() === 'medrxiv'
+        ? `medRxiv_Batch_${formattedBatchNum}`
+        : `Batch_${formattedBatchNum}`;
+}
+/**
+ * Determines the folder structure for a given month or batch
+ * @param options - Options containing month or batch
+ * @returns FolderStructure with the appropriate prefix and type
+ */
+export function getFolderStructure(options) {
+    if (options.month && options.batch) {
+        throw new Error('Either month or batch must be specified, not both');
+    }
+    if (!options.month && !options.batch) {
+        throw new Error('Either month or batch must be specified');
+    }
+    if (options.batch) {
+        // If batch is specified, use Back_Content structure
+        const normalizedBatch = normalizeBatch(options.batch, options.server);
+        return {
+            server: options.server || 'biorxiv',
+            type: 'back',
+            prefix: `Back_Content/${normalizedBatch}/`,
+            batch: normalizedBatch,
+        };
+    }
+    if (options.month) {
+        // Normalize month format to YYYY-MM
+        const normalizedMonth = normalizeMonthToYYYYMM(options.month);
+        if (!normalizedMonth) {
+            throw new Error(`Invalid month format: ${options.month}. Expected YYYY-MM or Month_YYYY format.`);
+        }
+        const [year, monthNum] = normalizedMonth.split('-').map(Number);
+        // bioRxiv switched from Back_Content to Current_Content in late 2018
+        // We'll use December 2018 as the cutoff point to be safe
+        const cutoffDate = new Date(2018, 11, 1); // December 1, 2018 (0-indexed month)
+        const requestedDate = new Date(year, monthNum - 1, 1);
+        if (requestedDate < cutoffDate) {
+            // Use Back_Content structure - but we don't know which batch
+            // User should specify batch explicitly for pre-2019 content
+            throw new Error(`Date ${options.month} is in the Back_Content period. Please specify a batch using --batch option. ` +
+                `Available batches can be listed with 'biorxiv list' command.`);
+        }
+        else {
+            // Use Current_Content structure
+            const monthName = getMonthName(monthNum);
+            return {
+                server: options.server || 'biorxiv',
+                type: 'current',
+                prefix: `Current_Content/${monthName}_${year}/`,
+                batch: `${monthName}_${year}`,
+            };
+        }
+    }
+    throw new Error('Invalid folder structure options');
+}
+export function removeDuplicateFolders(folders) {
+    return folders.filter((folder, index, arr) => arr.findIndex((f) => f.batch === folder.batch &&
+        f.server === folder.server &&
+        f.type === folder.type &&
+        f.prefix === folder.prefix) === index);
+}
+/**
+ * Sort folders chronologically, putting batches before months
+ */
+export function sortFoldersChronologically(folders) {
+    return folders.sort((a, b) => {
+        // Put batches before months
+        if (a.type === 'back' && b.type === 'current')
+            return -1;
+        if (a.type === 'current' && b.type === 'back')
+            return 1;
+        // For batches, sort by batch number
+        if (a.type === 'back' && b.type === 'back') {
+            const aNum = parseInt(a.batch.replace(/\D/g, ''));
+            const bNum = parseInt(b.batch.replace(/\D/g, ''));
+            return aNum - bNum;
+        }
+        // For months, sort chronologically (newest first)
+        if (a.type === 'current' && b.type === 'current') {
+            const aDate = new Date(a.batch);
+            const bDate = new Date(b.batch);
+            return aDate.getTime() - bDate.getTime();
+        }
+        return 0;
+    });
+}
+/**
+ * Normalizes various month formats to YYYY-MM
+ * @param month - Month in various formats
+ * @returns Normalized YYYY-MM format or null if invalid
+ */
+export function normalizeMonthToYYYYMM(month) {
+    // Already in YYYY-MM format
+    if (month.match(/^\d{4}-\d{2}$/)) {
+        const [, monthNum] = month.split('-').map(Number);
+        if (monthNum < 1 || monthNum > 12) {
+            return null; // Invalid month number
+        }
+        return month;
+    }
+    // Month_YYYY format (e.g., "November_2018")
+    const monthYearMatch = month.match(/^([A-Za-z]+)(?:[-_])(\d{4})$/);
+    if (monthYearMatch) {
+        const monthName = monthYearMatch[1];
+        const year = monthYearMatch[2];
+        const monthNum = getMonthNumber(monthName);
+        if (monthNum !== null) {
+            return `${year}-${monthNum.toString().padStart(2, '0')}`;
+        }
+    }
+    return null;
+}
+/**
+ * Gets month number from month name
+ * @param monthName - Month name (case insensitive)
+ * @returns Month number (1-12) or null if invalid
+ */
+function getMonthNumber(monthName) {
+    const monthNames = [
+        'january',
+        'february',
+        'march',
+        'april',
+        'may',
+        'june',
+        'july',
+        'august',
+        'september',
+        'october',
+        'november',
+        'december',
+    ];
+    const normalizedName = monthName.toLowerCase();
+    let monthIndex = monthNames.indexOf(normalizedName);
+    if (monthIndex === -1) {
+        monthIndex = monthNames.map((m) => m.slice(0, 3).toLowerCase()).indexOf(normalizedName);
+    }
+    return monthIndex !== -1 ? monthIndex + 1 : null;
+}
+/**
+ * Gets month name from month number
+ * @param monthNum - Month number (1-12)
+ * @returns Month name (e.g., "January")
+ */
+function getMonthName(monthNum) {
+    const monthNames = [
+        'January',
+        'February',
+        'March',
+        'April',
+        'May',
+        'June',
+        'July',
+        'August',
+        'September',
+        'October',
+        'November',
+        'December',
+    ];
+    if (monthNum < 1 || monthNum > 12) {
+        throw new Error(`Invalid month number: ${monthNum}. Must be 1-12.`);
+    }
+    return monthNames[monthNum - 1];
+}
+function getDefaultServer() {
+    throw new Error('Function not implemented.');
+}

package/dist/utils/src/index.d.ts

@@ -0,0 +1,3 @@
+export * from './biorxiv-parser.js';
+export * from './folder-structure.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/utils/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../utils/src/index.ts"],"names":[],"mappings":"AACA,cAAc,qBAAqB,CAAC;AACpC,cAAc,uBAAuB,CAAC"}

package/dist/utils/src/index.js

@@ -0,0 +1,3 @@
+// Export all utility functions
+export * from './biorxiv-parser.js';
+export * from './folder-structure.js';

package/package.json

@@ -0,0 +1,76 @@
+{
+  "name": "openrxiv",
+  "version": "0.0.0",
+  "description": "CLI tool to download openRxiv MECA files from AWS S3 for text and data mining",
+  "main": "dist/src/index.js",
+  "bin": {
+    "biorxiv": "./dist/cli.cjs",
+    "openrxiv": "./dist/cli.cjs",
+    "medrxiv": "./dist/cli.cjs"
+  },
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "scripts": {
+    "clean": "rimraf dist",
+    "unlink": "npm uninstall -g biorxiv",
+    "link": "npm run unlink; npm link;",
+    "dev": "npm run link && esbuild src/index.ts --bundle --outfile=dist/cli.cjs --platform=node --external:fsevents --target=node14 --watch",
+    "lint": "eslint \"src/**/!(*.spec).ts\" -c ../../.eslintrc.cjs",
+    "lint:format": "npx prettier --check \"src/**/*.ts\"",
+    "test": "vitest run",
+    "copy:version": "echo \"const version = '\"$npm_package_version\"';\nexport default version;\" > src/version.ts",
+    "test:watch": "vitest watch",
+    "build:esm": "tsc",
+    "build:cli": "esbuild src/index.ts --bundle --outfile=dist/cli.cjs --platform=node --external:fsevents --target=node14",
+    "build": "npm-run-all -l clean copy:version -p build:cli build:esm"
+  },
+  "keywords": [
+    "biorxiv",
+    "cli",
+    "aws",
+    "s3",
+    "download",
+    "meca",
+    "research",
+    "text-mining",
+    "data-mining"
+  ],
+  "author": "Curvenote",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.0.0",
+    "@aws-sdk/s3-request-presigner": "^3.0.0",
+    "axios": "^1.6.0",
+    "biorxiv-utils": "^0.0.0",
+    "boxen": "^8.0.1",
+    "character-entities": "^2.0.2",
+    "chalk": "^5.0.0",
+    "cli-progress": "^3.12.0",
+    "commander": "^11.0.0",
+    "conf": "^10.0.0",
+    "inquirer": "^9.0.0",
+    "jszip": "^3.10.1",
+    "ora": "^7.0.0",
+    "adm-zip": "^0.5.10",
+    "unified": "^11.0.0",
+    "xast-util-from-xml": "^4.0.0",
+    "p-limit": "^7.0.0"
+  },
+  "devDependencies": {
+    "@types/cli-progress": "^3.11.0",
+    "@types/inquirer": "^9.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/continuous-foundation/openrxiv.git"
+  },
+  "bugs": {
+    "url": "https://github.com/continuous-foundation/openrxiv/issues"
+  },
+  "homepage": "https://github.com/continuous-foundation/openrxiv#readme"
+}