confluence-exporter 1.0.0

package/src/commands/download.command.ts

@@ -0,0 +1,324 @@
+/**
+ * Download command handler - Downloads HTML pages from _queue.yaml
+ */
+
+import type { CommandHandler, CommandContext } from './types.js';
+import { ConfluenceApi } from '../api.js';
+import type { PageTreeNode, PageIndexEntry, ConfluenceConfig } from '../types.js';
+import path, { join } from 'path';
+import { mkdirSync, writeFileSync, readFileSync, existsSync } from 'fs';
+import { parse, stringify } from 'yaml';
+import { format } from 'prettier';
+import { updateIndexEntry, readIndexEntry } from '../utils.js';
+
+export class DownloadCommand implements CommandHandler {
+  name = 'download';
+  description = 'Download HTML pages from Confluence';
+
+  constructor(private config: ConfluenceConfig) { }
+
+  async execute(context: CommandContext): Promise<void> {
+    const api = new ConfluenceApi(this.config);
+
+    // Single page mode
+    if (this.config.pageId) {
+      console.log(`\n📄 Downloading single page: ${this.config.pageId}\n`);
+      
+      // Check if page needs downloading
+      const indexPath = join(this.config.outputDir, '_index.yaml');
+      const indexEntry = readIndexEntry(indexPath, this.config.pageId);
+      
+      if (indexEntry && indexEntry.downloadedVersion !== undefined && indexEntry.downloadedAt) {
+        // Check if we have current version info
+        const currentVersion = indexEntry.version ?? 0;
+        const downloadedVersion = indexEntry.downloadedVersion;
+        
+        if (currentVersion === downloadedVersion) {
+          console.log(`⏭️  Page ${this.config.pageId} is up-to-date (v${downloadedVersion}), skipping download`);
+          console.log('\n✅ Download complete!\n');
+          return;
+        } else {
+          console.log(`📥 Updating page ${this.config.pageId} from v${downloadedVersion} to v${currentVersion}`);
+        }
+      } else {
+        console.log(`📥 Downloading new page ${this.config.pageId}`);
+      }
+      
+      await this.downloadPage(api, this.config.pageId);
+      console.log('\n✅ Download complete!\n');
+      return;
+    }
+
+    // Queue mode - check for tree first, then fallback to queue
+    const treeFile = join(this.config.outputDir, '_tree.yaml');
+    const queueFile = join(this.config.outputDir, '_queue.yaml');
+
+    console.log(`\n🔍 Checking for tree file: ${treeFile}`);
+    console.log(`🔍 Checking for queue file: ${queueFile}\n`);
+
+    const hasTree = existsSync(treeFile);
+    const hasQueue = existsSync(queueFile);
+
+    if (!hasTree || !hasQueue) {
+      throw new Error(
+        `❌ Tree or queue file not found. Run 'plan' command first to create the tree and queue.`
+      );
+    }
+
+    // Read queue
+    const queueContent = readFileSync(queueFile, 'utf-8');
+    const queue = parse(queueContent) as PageIndexEntry[];
+
+    console.log(`📊 Queue contains ${queue.length} pages\n`);
+
+    // Apply limit if specified
+    const pagesToProcess = this.config.limit ? queue.slice(0, this.config.limit) : queue;
+
+    await this.downloadFromQueueWithHierarchy(api, this.config, pagesToProcess);
+  }
+
+  private async downloadFromQueueWithHierarchy(
+    api: ConfluenceApi,
+    config: ConfluenceConfig,
+    queue: PageIndexEntry[]
+  ): Promise<void> {
+    const treeFile = join(config.outputDir, '_tree.yaml');
+    const treeContent = readFileSync(treeFile, 'utf-8');
+    const tree = parse(treeContent) as PageTreeNode[];
+
+    // Build a map of pageId -> path
+    const pagePathMap = new Map<string, string>();
+
+    const buildPathMap = (node: PageTreeNode, currentPath: string) => {
+      // Store the path for this page
+      pagePathMap.set(node.id, currentPath);
+
+      // If node has children, build paths for them too
+      if (node.children && node.children.length > 0) {
+        const slug = this.slugify(node.title);
+        const childDir = join(currentPath, `${node.id}-${slug}`);
+
+        for (const child of node.children) {
+          buildPathMap(child, childDir);
+        }
+      }
+    };
+
+    // Create root folder for space
+    const rootDir = join(config.outputDir, config.spaceKey);
+    mkdirSync(rootDir, { recursive: true });
+
+    // Build the path map from tree
+    for (const node of tree) {
+      buildPathMap(node, rootDir);
+    }
+   
+    // Display first page path as tree
+    const firstPagePath = pagePathMap.get(queue[0].id);
+    if (firstPagePath) {
+      this.displayPathAsTree(path.dirname(firstPagePath), rootDir);
+    }
+
+    // Download pages from queue using the path map in batches
+    const batchSize = config.parallel || 5;
+    for (let batchStart = 0; batchStart < queue.length; batchStart += batchSize) {
+      const batchEnd = Math.min(batchStart + batchSize, queue.length);
+      const batch = queue.slice(batchStart, batchEnd);
+      
+      console.log(`\n📦 Processing batch ${Math.floor(batchStart / batchSize) + 1}/${Math.ceil(queue.length / batchSize)} (${batch.length} pages)`);
+      
+      await this.downloadBatch(api, config, batch, pagePathMap, rootDir);
+    }
+
+    console.log('\n✅ Download complete!\n');
+  }
+
+  private async downloadBatch(
+    api: ConfluenceApi,
+    config: ConfluenceConfig,
+    batch: PageIndexEntry[],
+    pagePathMap: Map<string, string>,
+    rootDir: string
+  ): Promise<void> {
+    const indexPath = join(config.outputDir, '_index.yaml');
+    
+    // Read index once for the entire batch
+    const indexContent = readFileSync(indexPath, 'utf-8');
+    const allIndexEntries = parse(indexContent) as PageIndexEntry[];
+    const indexMap = new Map(allIndexEntries.map(entry => [entry.id, entry]));
+
+    // Determine which pages need downloading
+    const pagesToDownload: Array<{ entry: PageIndexEntry; pagePath: string; skipReason: string }> = [];
+    const skippedPages: Array<{ entry: PageIndexEntry; pagePath: string; skipReason: string }> = [];
+
+    for (const entry of batch) {
+      const pagePath = pagePathMap.get(entry.id) || rootDir;
+      const indexEntry = indexMap.get(entry.id);
+      
+      let skip = false;
+      let skipReason = '';
+      
+      if (indexEntry && indexEntry.downloadedVersion !== undefined && indexEntry.downloadedAt) {
+        // Page has been downloaded before
+        const currentVersion = entry.version ?? 0;
+        const downloadedVersion = indexEntry.downloadedVersion;
+        
+        if (currentVersion === downloadedVersion) {
+          skip = true;
+          skipReason = `(⏭️ skipped - up-to-date v${downloadedVersion})`;
+        } else {
+          skipReason = `(📥 updating v${downloadedVersion} → v${currentVersion})`;
+        }
+      } else {
+        skipReason = `(📥 new download)`;
+      }
+
+      const filename = `${entry.id}-${this.slugify(entry.title)}.html`;
+      const deep = Math.max(0, pagePath.split(path.sep).length - rootDir.split(path.sep).length - 1);
+      console.log(`${'  '.repeat(deep)}/${filename} ${skipReason}`);
+
+      if (skip) {
+        skippedPages.push({ entry, pagePath, skipReason });
+      } else {
+        pagesToDownload.push({ entry, pagePath, skipReason });
+      }
+    }
+
+    // Download pages in parallel
+    if (pagesToDownload.length > 0) {
+      console.log(`\n📥 Downloading ${pagesToDownload.length} pages in parallel...`);
+      
+      const downloadPromises = pagesToDownload.map(async ({ entry, pagePath }) => {
+        try {
+          // Create directory if it doesn't exist
+          mkdirSync(pagePath, { recursive: true });
+          const page = await api.getPage(entry.id);
+          
+          // Update the index entry in memory
+          const indexEntry = indexMap.get(entry.id);
+          if (indexEntry) {
+            indexEntry.downloadedVersion = page.version ?? 0;
+            indexEntry.downloadedAt = new Date().toISOString();
+          }
+          
+          return { entry, page, pagePath, success: true };
+        } catch (error) {
+          console.error(`❌ Failed to download page ${entry.id}:`, error);
+          return { entry, page: null, pagePath, success: false };
+        }
+      });
+
+      const results = await Promise.all(downloadPromises);
+
+      // Process successful downloads
+      for (const result of results) {
+        if (result.success && result.page) {
+          try {
+            await this.savePageToFile(result.page, result.pagePath);
+          } catch (error) {
+            console.error(`❌ Failed to save page ${result.entry.id}:`, error);
+          }
+        }
+      }
+    }
+
+    // Write updated index back to file once for the entire batch
+    if (pagesToDownload.length > 0) {
+      const yamlContent = stringify(allIndexEntries, {
+        indent: 2,
+        lineWidth: 0
+      });
+      writeFileSync(indexPath, yamlContent, 'utf-8');
+      console.log(`💾 Updated index with ${pagesToDownload.length} downloaded pages`);
+    }
+
+    console.log(`✅ Batch complete: ${pagesToDownload.length} downloaded, ${skippedPages.length} skipped`);
+  }
+
+  private async savePageToFile(
+    page: { id: string; title: string; body: string; version?: number },
+    filePath: string
+  ): Promise<void> {
+    const slug = this.slugify(page.title);
+    const filename = `${page.id}-${slug}.html`;
+    const fullPath = join(filePath, filename);
+
+    // Format HTML with Prettier
+    let formattedHtml = page.body;
+    try {
+      formattedHtml = await format(page.body, {
+        parser: 'html',
+        printWidth: 120,
+        htmlWhitespaceSensitivity: 'ignore',
+        tabWidth: 2,
+      });
+    } catch {
+      console.warn(`⚠️  Failed to format HTML for ${page.title}, saving unformatted`);
+    }
+
+    writeFileSync(fullPath, formattedHtml, 'utf-8');
+  }
+
+  private displayPathAsTree(fullPath: string, rootDir: string): void {
+    // Get relative path from root
+    const relativePath = path.relative(rootDir, fullPath);
+    
+    // Split path into segments
+    const segments = relativePath.split(path.sep).filter(s => s);
+    
+    // Display each segment with proper indentation
+    for (let i = 0; i < segments.length; i++) {
+      const indent = '  '.repeat(i);
+      console.log(`${indent}/${segments[i]}`);
+    }
+  }
+
+  private async downloadPage(
+    api: ConfluenceApi,
+    pageId: string,
+    outputDir?: string
+  ): Promise<void> {
+    const page = await api.getPage(pageId);
+    const slug = this.slugify(page.title);
+    const filename = `${pageId}-${slug}.html`;
+
+    const dir = outputDir || process.cwd();
+    const filepath = join(dir, filename);
+
+    // Format HTML with Prettier
+    let formattedHtml = page.body;
+    try {
+      formattedHtml = await format(page.body, {
+        parser: 'html',
+        printWidth: 120,
+        htmlWhitespaceSensitivity: 'ignore',
+        tabWidth: 2,
+      });
+    } catch {
+      console.warn(`⚠️  Failed to format HTML for ${page.title}, saving unformatted`);
+    }
+
+    writeFileSync(filepath, formattedHtml, 'utf-8');
+
+    // Update _index.yaml with download metadata
+    const indexPath = join(this.config.outputDir, '_index.yaml');
+    const success = updateIndexEntry(indexPath, page.id, {
+      downloadedVersion: page.version ?? 0,
+      downloadedAt: new Date().toISOString()
+    });
+
+    if (!success) {
+      console.warn(`⚠️  Failed to update index for page ${page.title}`);
+    }
+  }
+
+  private slugify(text: string): string {
+    return text
+      .toLowerCase()
+      .replace(/[^\w\s-]/g, '')
+      .replace(/\s+/g, '-')
+      .replace(/-+/g, '-')
+      .trim()
+      .substring(0, 50); // Limit length for filesystem
+  }
+}

package/src/commands/executor.ts

@@ -0,0 +1,62 @@
+/**
+ * Command executor - orchestrates command execution
+ */
+
+import { ConfluenceConfig } from 'src/types.js';
+import { CommandRegistry } from './registry.js';
+import type { Command, CommandContext } from './types.js';
+
+export class CommandExecutor {
+  private registry: CommandRegistry;
+
+  constructor(private config: ConfluenceConfig) {
+    this.registry = new CommandRegistry(config);
+  }
+
+  /**
+   * Validate commands and return parsed list
+   */
+  validateCommands(commands: string[]): Command[] {
+    const validated: Command[] = [];
+
+    for (const cmd of commands) {
+      const command = cmd.toLowerCase();
+      if (this.registry.isValidCommand(command)) {
+        validated.push(command);
+      } else {
+        throw new Error(`Unknown command: "${cmd}"`);
+      }
+    }
+
+    return validated;
+  }
+
+  /**
+   * Execute a sequence of commands
+   */
+  async executeCommands(commands: Command[], context: CommandContext): Promise<void> {
+    for (let i = 0; i < commands.length; i++) {
+      const command = commands[i];
+      
+      if (i > 0) {
+        console.log('\n' + '─'.repeat(60) + '\n');
+      }
+
+      const handler = this.registry.getHandler(command);
+      if (!handler) {
+        throw new Error(`No handler found for command: ${command}`);
+      }
+
+      await handler.execute(context);
+    }
+    
+    console.log('\n✓ All commands completed successfully!');
+  }
+
+  /**
+   * Get list of valid commands
+   */
+  getValidCommands(): Command[] {
+    return this.registry.getValidCommands();
+  }
+}

package/src/commands/help.command.ts

@@ -0,0 +1,72 @@
+/**
+ * Help command handler
+ */
+
+import { ConfluenceConfig } from 'src/types.js';
+import type { CommandContext, CommandHandler } from './types.js';
+
+export class HelpCommand implements CommandHandler {
+  async execute(_context: CommandContext): Promise<void> {
+    this.showHelp();
+  }
+
+  private showHelp(): void {
+    console.log('Minimal Confluence to Markdown Exporter\n');
+    console.log('Usage: node index.js [command] [options]\n');
+    console.log('If no command is provided, runs full sync: update index (or create if not exists), plan, download, and transform.\n');
+    console.log('Commands:');
+    console.log('  help                          Show this help message');
+    console.log('  index                         Create _index.yaml with page metadata');
+    console.log('  update                        Check for new/updated pages and update _index.yaml');
+    console.log('  plan                          Create _queue.yaml for download (from index or specific page tree)');
+    console.log('  download                      Download HTML pages from _queue.yaml');
+    console.log('  transform                     Transform HTML files to Markdown (skips existing MD files, creates links structure)');
+    console.log('  index plan download transform Run all commands in sequence\n');
+    console.log('Options:');
+    console.log('  -u, --url <url>          Confluence base URL');
+    console.log('  -n, --username <email>   Confluence username/email');
+    console.log('  -p, --password <token>   Confluence API token');
+    console.log('  -s, --space <key>        Confluence space key');
+    console.log('  -i, --pageId <id>        Download specific page ID only (optional)');
+    console.log('  -o, --output <dir>       Output directory (default: ./output)');
+    console.log('  -l, --limit <number>     Limit number of pages to process (optional)');
+    console.log('  --parallel <number>      Number of concurrent operations (default: 5)');
+    console.log('  -f, --force              Force re-download of all pages (skip version check)');
+    console.log('  --clear                  Clear existing MD files and images before transforming');
+    console.log('  --pageSize <number>      Items per API page (default: 25)');
+    console.log('  -h, --help               Show this help message\n');
+    console.log('Environment Variables:');
+    console.log('  CONFLUENCE_BASE_URL');
+    console.log('  CONFLUENCE_USERNAME');
+    console.log('  CONFLUENCE_PASSWORD');
+    console.log('  CONFLUENCE_SPACE_KEY');
+    console.log('  OUTPUT_DIR\n');
+    console.log('Examples:');
+    console.log('  # Full workflow: index, plan, download, and transform');
+    console.log('  node index.js index plan download transform -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Full workflow with limit (process first 10 pages only)');
+    console.log('  node index.js index plan download transform -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE -l 10\n');
+    console.log('  # Create index only (Phase 1)');
+    console.log('  node index.js index -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Check for new/updated pages and update existing index');
+    console.log('  node index.js update -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Create download queue from existing index (Phase 2)');
+    console.log('  node index.js plan -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Create download queue for specific page and all children');
+    console.log('  node index.js plan -i 123456789 -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Force re-download all pages (ignore version check)');
+    console.log('  node index.js plan --force -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Download HTML pages from existing queue (Phase 3)');
+    console.log('  node index.js download -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Transform HTML to Markdown (Phase 4)');
+    console.log('  node index.js transform -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Transform HTML to Markdown with clear (remove existing MD files first)');
+    console.log('  node index.js transform --clear -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Download and transform together');
+    console.log('  node index.js download transform -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Download and transform with higher concurrency');
+    console.log('  node index.js download transform --parallel 10 -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE\n');
+    console.log('  # Download single page HTML only (no index/plan needed)');
+    console.log('  node index.js download -i 123456789 -u https://mysite.atlassian.net -n user@example.com -p token -s MYSPACE');
+  }
+}

package/src/commands/index.command.ts

@@ -0,0 +1,111 @@
+/**
+ * Index command handler - Creates _index.yaml with all page metadata
+ */
+
+import { promises as fs } from 'fs';
+import path from 'path';
+import yaml from 'yaml';
+import { ConfluenceApi } from '../api.js';
+import type { ConfluenceConfig, PageIndexEntry } from '../types.js';
+import type { CommandContext, CommandHandler } from './types.js';
+
+export class IndexCommand implements CommandHandler {
+  constructor(private config: ConfluenceConfig) {}
+
+  async execute(context: CommandContext): Promise<void> {
+    const api = new ConfluenceApi(this.config);
+
+    // Create output directory if it doesn't exist
+    await fs.mkdir(this.config.outputDir, { recursive: true });
+
+    console.log(`Starting indexing of space: ${this.config.spaceKey}`);
+    console.log(`Output directory: ${this.config.outputDir}\n`);
+
+    // Phase 1: Create _index.yaml
+    console.log('Phase 1: Creating _index.yaml...');
+    await this.createIndex(api, this.config);
+
+    console.log(`\nIndexing complete!`);
+    console.log(`Index saved to: ${this.config.outputDir}/_index.yaml`);
+  }
+
+  /**
+   * Create _index.yaml file with all pages to download
+   */
+  private async createIndex(api: ConfluenceApi, config: CommandContext['config']): Promise<void> {
+    const indexPath = path.join(config.outputDir, '_index.yaml');
+    
+    let pageCount = 0;
+    let pageSize = config.pageSize || 100;
+    let startFrom = 0;
+    
+    // Check if _index.yaml already exists (resume functionality)
+    try {
+      const existingContent = await fs.readFile(indexPath, 'utf-8');
+      
+      // Extract page size from comment if it exists
+      const pageSizeMatch = existingContent.match(/# Page Size: (\d+)/);
+      if (pageSizeMatch) {
+        pageSize = parseInt(pageSizeMatch[1], 10);
+        console.log(`Using existing page size from index: ${pageSize}`);
+      }
+      
+      const existingPages = yaml.parse(existingContent) as PageIndexEntry[];
+      
+      if (existingPages && Array.isArray(existingPages)) {
+        pageCount = existingPages.length;
+        // Calculate the start position for the API
+        startFrom = pageCount;
+        console.log(`Found existing index with ${pageCount} pages. Resuming from position ${startFrom + 1}...\n`);
+      }
+    } catch (_error) {
+      // File doesn't exist or is invalid, start fresh
+      const header = `# Confluence Export Index
+# Space: ${config.spaceKey}
+# Export Date: ${new Date().toISOString()}
+# Page Size: ${pageSize}
+
+`;
+      await fs.writeFile(indexPath, header, 'utf-8');
+      console.log(`Creating new index with page size: ${pageSize}...\n`);
+    }
+    
+    // Fetch pages starting from where we left off
+    for await (const page of api.getAllPages(config.spaceKey, pageSize, startFrom)) {
+      pageCount++;
+      console.log(`[${pageCount}] Indexed: ${page.title} (${page.id}) [API Page ${page.apiPageNumber}]`);
+      
+      // Create page entry
+      const pageEntry: PageIndexEntry = {
+        id: page.id,
+        title: page.title,
+        version: page.version,
+        parentId: page.parentId,
+        modifiedDate: page.modifiedDate,
+        indexedDate: new Date().toISOString(),
+        pageNumber: page.apiPageNumber
+      };
+      
+      // Convert to YAML and format as array item (with leading -)
+      const yamlDoc = yaml.stringify(pageEntry).trim();
+      const lines = yamlDoc.split('\n');
+      const arrayItem = lines.map((line, index) => {
+        if (index === 0) {
+          return `- ${line}`;
+        }
+        return `  ${line}`;
+      }).join('\n');
+      
+      await fs.appendFile(indexPath, arrayItem + '\n', 'utf-8');
+      
+      // Check if limit is reached
+      if (config.limit && pageCount >= config.limit) {
+        console.log(`\n⚠ Limit reached: ${config.limit} pages indexed`);
+        break;
+      }
+    }
+    
+    console.log(`\n✓ Index created: ${indexPath}`);
+    console.log(`  Total pages indexed: ${pageCount}`);
+  }
+}

package/src/commands/index.ts

@@ -0,0 +1,14 @@
+/**
+ * Commands module exports
+ * Re-exports all command-related classes and types for easier importing
+ */
+
+export { HelpCommand } from './help.command.js';
+export { IndexCommand } from './index.command.js';
+export { UpdateCommand } from './update.command.js';
+export { PlanCommand } from './plan.command.js';
+export { DownloadCommand } from './download.command.js';
+export { TransformCommand } from './transform.command.js';
+export { CommandRegistry } from './registry.js';
+export { CommandExecutor } from './executor.js';
+export type { Command, CommandContext, CommandHandler } from './types.js';