openrxiv-cli 0.0.2

package/dist/utils/months.js

@@ -0,0 +1,135 @@
+/**
+ * Month utility functions for batch processing
+ */
+/**
+ * Generate a range of months to process backwards from current month to 2018-12
+ */
+export function generateMonthRange() {
+    const months = [];
+    const now = new Date();
+    const currentDate = new Date(now.getFullYear(), now.getMonth(), 1);
+    // Go back from current month to December 2018
+    while (currentDate.getFullYear() >= 2018) {
+        const year = currentDate.getFullYear();
+        const month = String(currentDate.getMonth() + 1).padStart(2, '0');
+        // Stop at 2018-12 (inclusive)
+        if (year === 2018 && month === '12') {
+            months.push('2018-12');
+            break;
+        }
+        months.push(`${year}-${month}`);
+        // Move to previous month
+        currentDate.setMonth(currentDate.getMonth() - 1);
+    }
+    return months;
+}
+/**
+ * Parse month input and return array of months to process
+ */
+export function parseMonthInput(monthInput) {
+    // Handle comma-separated list
+    if (monthInput.includes(',')) {
+        const parts = monthInput
+            .split(',')
+            .map((m) => m.trim())
+            .filter((m) => m.length > 0);
+        // Process each part (which may contain wildcards)
+        const result = [];
+        for (const part of parts) {
+            if (part.includes('*')) {
+                // Expand wildcard pattern
+                result.push(...parseWildcardPattern(part));
+            }
+            else {
+                // Single month
+                result.push(part);
+            }
+        }
+        return result;
+    }
+    // Handle wildcard pattern (e.g., "2025-*")
+    if (monthInput.includes('*')) {
+        return parseWildcardPattern(monthInput);
+    }
+    // Single month
+    return [monthInput];
+}
+/**
+ * Parse wildcard pattern like "2025-*" to get all months in that year
+ */
+export function parseWildcardPattern(pattern) {
+    const months = [];
+    // Extract year from pattern
+    const yearMatch = pattern.match(/^(\d{4})-\*$/);
+    if (!yearMatch) {
+        throw new Error(`Invalid wildcard pattern: ${pattern}. Use format like "2025-*"`);
+    }
+    const year = parseInt(yearMatch[1], 10);
+    const now = new Date();
+    const currentYear = now.getFullYear();
+    const currentMonth = now.getMonth() + 1;
+    // Generate all months in the year, but only up to current month if it's current year
+    for (let month = 1; month <= 12; month++) {
+        if (year === currentYear && month > currentMonth) {
+            break; // Don't process future months
+        }
+        const monthStr = String(month).padStart(2, '0');
+        months.push(`${year}-${monthStr}`);
+    }
+    return months;
+}
+/**
+ * Validate month format (YYYY-MM)
+ */
+export function validateMonthFormat(month) {
+    const monthRegex = /^\d{4}-\d{2}$/;
+    if (!monthRegex.test(month)) {
+        return false;
+    }
+    const [year, monthNum] = month.split('-');
+    const yearNum = parseInt(year, 10);
+    const monthInt = parseInt(monthNum, 10);
+    return yearNum <= 2100 && monthInt >= 1 && monthInt <= 12;
+}
+/**
+ * Sort months chronologically (oldest first)
+ */
+export function sortMonthsChronologically(months) {
+    return months.sort((a, b) => {
+        const [yearA, monthA] = a.split('-').map(Number);
+        const [yearB, monthB] = b.split('-').map(Number);
+        if (yearA !== yearB) {
+            return yearA - yearB;
+        }
+        return monthA - monthB;
+    });
+}
+/**
+ * Get current month in YYYY-MM format
+ */
+export function getCurrentMonth() {
+    const now = new Date();
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, '0');
+    return `${year}-${month}`;
+}
+/**
+ * Get previous month in YYYY-MM format
+ */
+export function getPreviousMonth() {
+    const now = new Date();
+    now.setMonth(now.getMonth() - 1);
+    const year = now.getFullYear();
+    const month = String(now.getMonth() + 1).padStart(2, '0');
+    return `${year}-${month}`;
+}
+/**
+ * Check if a month is in the future
+ */
+export function isFutureMonth(month) {
+    const [year, monthNum] = month.split('-').map(Number);
+    const now = new Date();
+    const currentYear = now.getFullYear();
+    const currentMonth = now.getMonth() + 1;
+    return year > currentYear || (year === currentYear && monthNum > currentMonth);
+}

package/dist/utils/months.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=months.test.d.ts.map

package/dist/utils/months.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"months.test.d.ts","sourceRoot":"","sources":["../../src/utils/months.test.ts"],"names":[],"mappings":""}

package/dist/utils/months.test.js

@@ -0,0 +1,209 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { generateMonthRange, parseMonthInput, parseWildcardPattern, validateMonthFormat, sortMonthsChronologically, getCurrentMonth, getPreviousMonth, isFutureMonth, } from './months.js';
+import { getFolderStructure, removeDuplicateFolders, sortFoldersChronologically, } from 'openrxiv-utils';
+describe('Month Utilities', () => {
+    beforeEach(() => {
+        // Mock the current date to 2025-01-15 for consistent testing
+        const mockDate = new Date('2025-01-15T12:00:00Z');
+        vi.useFakeTimers();
+        vi.setSystemTime(mockDate);
+    });
+    afterEach(() => {
+        vi.useRealTimers();
+    });
+    describe('generateMonthRange', () => {
+        it('should generate months from current month back to 2018-12', () => {
+            const months = generateMonthRange();
+            // Should start from current month (2025-01)
+            expect(months[0]).toBe('2025-01');
+            // Should end at 2018-12
+            expect(months[months.length - 1]).toBe('2018-12');
+            // Should include 2018-12
+            expect(months).toContain('2018-12');
+            // Should not include 2018-11 or earlier
+            expect(months).not.toContain('2018-11');
+            expect(months).not.toContain('2017-12');
+            // Should have correct number of months
+            // From 2025-01 to 2018-12: (2025-2018+1) * 12 = 8 * 12 = 96 months
+            // But we stop at 2018-12, so it's actually 74 months
+            expect(months).toHaveLength(74);
+        });
+        it('should generate months in descending order (newest first)', () => {
+            const months = generateMonthRange();
+            // First few months should be newest
+            expect(months[0]).toBe('2025-01');
+            expect(months[1]).toBe('2024-12');
+            expect(months[2]).toBe('2024-11');
+            // Last few months should be oldest
+            expect(months[months.length - 3]).toBe('2019-02');
+            expect(months[months.length - 2]).toBe('2019-01');
+            expect(months[months.length - 1]).toBe('2018-12');
+        });
+    });
+    describe('parseMonthInput', () => {
+        it('should handle single month', () => {
+            const result = parseMonthInput('2025-01');
+            expect(result).toEqual(['2025-01']);
+        });
+        it('should handle comma-separated months', () => {
+            const result = parseMonthInput('2025-01,2024-12,2024-11');
+            expect(result).toEqual(['2025-01', '2024-12', '2024-11']);
+        });
+        it('should handle comma-separated months with spaces', () => {
+            const result = parseMonthInput(' 2025-01 , 2024-12 , 2024-11 ');
+            expect(result).toEqual(['2025-01', '2024-12', '2024-11']);
+        });
+        it('should handle wildcard pattern', () => {
+            const result = parseMonthInput('2024-*');
+            expect(result).toHaveLength(12);
+            expect(result).toContain('2024-01');
+            expect(result).toContain('2024-12');
+        });
+        it('should filter out empty strings', () => {
+            const result = parseMonthInput('2025-01,,2024-12,');
+            expect(result).toEqual(['2025-01', '2024-12']);
+        });
+    });
+    describe('parseWildcardPattern', () => {
+        it('should parse valid wildcard pattern', () => {
+            const result = parseWildcardPattern('2024-*');
+            expect(result).toHaveLength(12);
+            expect(result).toContain('2024-01');
+            expect(result).toContain('2024-06');
+            expect(result).toContain('2024-12');
+        });
+        it('should handle current year and stop at current month', () => {
+            const result = parseWildcardPattern('2025-*');
+            // Since we're mocking 2025-01, should only include January
+            expect(result).toHaveLength(1);
+            expect(result).toEqual(['2025-01']);
+        });
+        it('should throw error for invalid wildcard pattern', () => {
+            expect(() => parseWildcardPattern('2024-')).toThrow('Invalid wildcard pattern');
+            expect(() => parseWildcardPattern('2024-*x')).toThrow('Invalid wildcard pattern');
+            expect(() => parseWildcardPattern('*-2024')).toThrow('Invalid wildcard pattern');
+        });
+        it('should handle past years correctly', () => {
+            const result = parseWildcardPattern('2020-*');
+            expect(result).toHaveLength(12);
+            expect(result).toContain('2020-01');
+            expect(result).toContain('2020-12');
+        });
+    });
+    describe('validateMonthFormat', () => {
+        it('should validate correct month formats', () => {
+            expect(validateMonthFormat('2025-01')).toBe(true);
+            expect(validateMonthFormat('2024-12')).toBe(true);
+            expect(validateMonthFormat('2018-12')).toBe(true);
+            expect(validateMonthFormat('2020-06')).toBe(true);
+        });
+        it('should reject invalid month formats', () => {
+            expect(validateMonthFormat('2025-1')).toBe(false); // Missing leading zero
+            expect(validateMonthFormat('2025-13')).toBe(false); // Invalid month
+            expect(validateMonthFormat('2025-00')).toBe(false); // Invalid month
+            expect(validateMonthFormat('2025-1a')).toBe(false); // Non-numeric
+            expect(validateMonthFormat('2025')).toBe(false); // Missing month
+            expect(validateMonthFormat('2025-01-01')).toBe(false); // Too many parts
+        });
+        it('should reject months outside valid year range', () => {
+            expect(validateMonthFormat('2101-01')).toBe(false); // After 2100
+        });
+        it('should accept boundary years', () => {
+            expect(validateMonthFormat('2018-01')).toBe(true);
+            expect(validateMonthFormat('2018-12')).toBe(true);
+            expect(validateMonthFormat('2100-01')).toBe(true);
+            expect(validateMonthFormat('2100-12')).toBe(true);
+        });
+    });
+    describe('sortMonthsChronologically', () => {
+        it('should sort months in chronological order (oldest first)', () => {
+            const months = ['2025-01', '2024-12', '2024-01', '2025-02'];
+            const sorted = sortMonthsChronologically(months);
+            expect(sorted).toEqual(['2024-01', '2024-12', '2025-01', '2025-02']);
+        });
+        it('should handle months within same year', () => {
+            const months = ['2024-12', '2024-01', '2024-06'];
+            const sorted = sortMonthsChronologically(months);
+            expect(sorted).toEqual(['2024-01', '2024-06', '2024-12']);
+        });
+        it('should handle single month', () => {
+            const months = ['2024-06'];
+            const sorted = sortMonthsChronologically(months);
+            expect(sorted).toEqual(['2024-06']);
+        });
+        it('should handle empty array', () => {
+            const months = [];
+            const sorted = sortMonthsChronologically(months);
+            expect(sorted).toEqual([]);
+        });
+    });
+    describe('getCurrentMonth', () => {
+        it('should return current month in YYYY-MM format', () => {
+            const currentMonth = getCurrentMonth();
+            expect(currentMonth).toBe('2025-01');
+        });
+    });
+    describe('getPreviousMonth', () => {
+        it('should return previous month in YYYY-MM format', () => {
+            const previousMonth = getPreviousMonth();
+            expect(previousMonth).toBe('2024-12');
+        });
+        it('should handle year boundary correctly', () => {
+            // Mock to January 2024 to test year boundary
+            vi.setSystemTime(new Date('2024-01-15T12:00:00Z'));
+            const previousMonth = getPreviousMonth();
+            expect(previousMonth).toBe('2023-12');
+        });
+    });
+    describe('isFutureMonth', () => {
+        it('should identify future months correctly', () => {
+            expect(isFutureMonth('2025-02')).toBe(true);
+            expect(isFutureMonth('2025-12')).toBe(true);
+            expect(isFutureMonth('2026-01')).toBe(true);
+        });
+        it('should identify current and past months correctly', () => {
+            expect(isFutureMonth('2025-01')).toBe(false); // Current month
+            expect(isFutureMonth('2024-12')).toBe(false); // Past month
+            expect(isFutureMonth('2024-01')).toBe(false); // Past month
+            expect(isFutureMonth('2018-12')).toBe(false); // Past month
+        });
+        it('should handle year boundaries correctly', () => {
+            expect(isFutureMonth('2024-12')).toBe(false); // Past year
+            expect(isFutureMonth('2026-01')).toBe(true); // Future year
+        });
+    });
+    describe('Integration tests', () => {
+        it('should handle complete workflow: parse, validate, sort, deduplicate', () => {
+            const input = '2025-01,2024-12,2025-01,2024-11,2024-12';
+            // Parse input
+            const parsed = parseMonthInput(input);
+            expect(parsed).toEqual(['2025-01', '2024-12', '2025-01', '2024-11', '2024-12']);
+            // Validate all months
+            const valid = parsed.filter(validateMonthFormat);
+            expect(valid).toEqual(['2025-01', '2024-12', '2025-01', '2024-11', '2024-12']);
+            // Remove duplicates
+            const folders = valid.map((month) => getFolderStructure({ month }));
+            const unique = removeDuplicateFolders(folders);
+            expect(unique.length).toEqual(3);
+            // Sort chronologically
+            const sorted = sortFoldersChronologically(unique);
+            expect(sorted.length).toEqual(3);
+            expect(sorted[0].batch).toEqual('November_2024');
+            expect(sorted[1].batch).toEqual('December_2024');
+            expect(sorted[2].batch).toEqual('January_2025');
+        });
+        it('should handle wildcard with validation and sorting', () => {
+            const input = '2024-*';
+            // Parse wildcard
+            const parsed = parseMonthInput(input);
+            expect(parsed).toHaveLength(12);
+            // Validate all months
+            const valid = parsed.filter(validateMonthFormat);
+            expect(valid).toHaveLength(12);
+            // Sort chronologically
+            const sorted = sortMonthsChronologically(valid);
+            expect(sorted[0]).toBe('2024-01');
+            expect(sorted[11]).toBe('2024-12');
+        });
+    });
+});

package/dist/utils/requester-pays-error.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Utility function to display the requester-pays error message
+ * Used when S3 operations fail due to requester-pays bucket requirements
+ */
+export declare function displayRequesterPaysError(): void;
+//# sourceMappingURL=requester-pays-error.d.ts.map

package/dist/utils/requester-pays-error.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"requester-pays-error.d.ts","sourceRoot":"","sources":["../../src/utils/requester-pays-error.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,yBAAyB,IAAI,IAAI,CAehD"}

package/dist/utils/requester-pays-error.js

@@ -0,0 +1,20 @@
+/**
+ * Utility function to display the requester-pays error message
+ * Used when S3 operations fail due to requester-pays bucket requirements
+ */
+export function displayRequesterPaysError() {
+    console.error(`
+❌ Operation failed: S3 bucket requires requester-pays
+💡 This bucket has requester-pays enabled, which means:
+   • You need to pay for data transfer costs
+   • Your AWS credentials must be configured
+   • The bucket policy must allow your account
+
+🔧 To fix this:
+   1. Ensure your AWS credentials are configured
+   2. Verify you have permission to access the bucket
+   3. Add the --requester-pays flag to your command
+
+📚 For more help, see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/RequesterPaysBuckets.html
+`);
+}

package/dist/version.d.ts

@@ -0,0 +1,3 @@
+declare const version = "0.0.2";
+export default version;
+//# sourceMappingURL=version.d.ts.map

package/dist/version.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["../src/version.ts"],"names":[],"mappings":"AAAA,QAAA,MAAM,OAAO,UAAU,CAAC;AACxB,eAAe,OAAO,CAAC"}

package/dist/version.js

@@ -0,0 +1,2 @@
+const version = '0.0.2';
+export default version;

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "openrxiv-cli",
+  "version": "0.0.2",
+  "description": "CLI tool to download openRxiv MECA files from AWS S3 for text and data mining",
+  "main": "dist/index.js",
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "scripts": {
+    "clean": "rimraf dist",
+    "lint": "eslint \"src/**/!(*.spec).ts\" -c ../../.eslintrc.cjs",
+    "lint:format": "npx prettier --check \"src/**/*.ts\"",
+    "test": "vitest run",
+    "copy:version": "echo \"const version = '\"$npm_package_version\"';\nexport default version;\" > src/version.ts",
+    "test:watch": "vitest watch",
+    "build:esm": "tsc",
+    "build": "npm-run-all -l clean copy:version -p build:esm"
+  },
+  "keywords": [
+    "biorxiv",
+    "cli",
+    "aws",
+    "s3",
+    "download",
+    "meca",
+    "research",
+    "text-mining",
+    "data-mining"
+  ],
+  "author": "Curvenote",
+  "license": "MIT",
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@aws-sdk/client-s3": "^3.0.0",
+    "@aws-sdk/s3-request-presigner": "^3.0.0",
+    "axios": "^1.6.0",
+    "openrxiv-utils": "^0.0.2",
+    "boxen": "^8.0.1",
+    "character-entities": "^2.0.2",
+    "chalk": "^5.0.0",
+    "cli-progress": "^3.12.0",
+    "commander": "^14.0.0",
+    "conf": "^10.0.0",
+    "inquirer": "^9.0.0",
+    "jszip": "^3.10.1",
+    "ora": "^7.0.0",
+    "adm-zip": "^0.5.10",
+    "unified": "^11.0.0",
+    "xast-util-from-xml": "^4.0.0",
+    "p-limit": "^7.0.0"
+  },
+  "devDependencies": {
+    "@types/cli-progress": "^3.11.0",
+    "@types/inquirer": "^9.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/continuous-foundation/openrxiv.git"
+  },
+  "bugs": {
+    "url": "https://github.com/continuous-foundation/openrxiv/issues"
+  },
+  "homepage": "https://github.com/continuous-foundation/openrxiv#readme"
+}