confluence-exporter 1.0.0

package/src/commands/types.ts

@@ -0,0 +1,16 @@
+/**
+ * Command-related type definitions
+ */
+
+import type { ConfluenceConfig } from '../types.js';
+
+export type Command = 'help' | 'index' | 'update' | 'plan' | 'download' | 'transform';
+
+export interface CommandContext {
+  config: ConfluenceConfig;
+  args: Record<string, unknown>;
+}
+
+export interface CommandHandler {
+  execute(context: CommandContext): Promise<void>;
+}

package/src/commands/update.command.ts

@@ -0,0 +1,229 @@
+/**
+ * Update command handler - Checks for new/updated pages and updates _index.yaml
+ */
+
+import { promises as fs } from 'fs';
+import path from 'path';
+import yaml from 'yaml';
+import { ConfluenceApi } from '../api.js';
+import type { ConfluenceConfig, PageIndexEntry, PageMetadata } from '../types.js';
+import type { CommandContext, CommandHandler } from './types.js';
+
+export class UpdateCommand implements CommandHandler {
+  constructor(private config: ConfluenceConfig) {}
+
+  async execute(context: CommandContext): Promise<void> {
+    const api = new ConfluenceApi(this.config);
+    const indexPath = path.join(this.config.outputDir, '_index.yaml');
+
+    console.log(`Checking for updates in space: ${this.config.spaceKey}`);
+    console.log(`Index file: ${indexPath}\n`);
+
+    // Load existing index
+    let existingIndex: Map<string, PageIndexEntry>;
+    let oldestIndexedDate: string | undefined;
+    
+    try {
+      const existingContent = await fs.readFile(indexPath, 'utf-8');
+      const existingPages = yaml.parse(existingContent) as PageIndexEntry[];
+      existingIndex = new Map(existingPages.map(p => [p.id, p]));
+      
+      // Find the oldest indexedDate to use as the starting point for CQL search
+      for (const page of existingPages) {
+        if (page.indexedDate) {
+          if (!oldestIndexedDate || page.indexedDate < oldestIndexedDate) {
+            oldestIndexedDate = page.indexedDate;
+          }
+        }
+      }
+      
+      console.log(`Loaded existing index with ${existingIndex.size} pages`);
+      if (oldestIndexedDate) {
+        console.log(`Oldest indexed date: ${oldestIndexedDate}\n`);
+      }
+    } catch (error) {
+      console.error(`Error: _index.yaml not found. Run 'index' command first.`);
+      process.exit(1);
+    }
+
+    // Use CQL to fetch only pages modified since oldest indexed date
+    // This is much more efficient than fetching all pages
+    let modifiedPages: PageMetadata[] = [];
+    
+    if (oldestIndexedDate) {
+      // Format date for CQL: "yyyy-MM-dd" or "yyyy-MM-dd HH:mm"
+      const searchDate = oldestIndexedDate.split('T')[0]; // Get just the date part
+      const cql = `space = "${this.config.spaceKey}" AND type = page AND lastmodified >= "${searchDate}" ORDER BY lastmodified DESC`;
+      
+      console.log(`Searching for pages modified since ${searchDate}...`);
+      console.log(`CQL: ${cql}\n`);
+      
+      try {
+        modifiedPages = await api.searchPages(cql, this.config.pageSize || 100);
+        console.log(`Found ${modifiedPages.length} pages modified since ${searchDate}\n`);
+      } catch (error) {
+        console.warn(`CQL search failed, falling back to full space scan...`);
+        console.warn(`Error: ${error}\n`);
+        modifiedPages = [];
+      }
+    }
+
+    // If CQL search returned results, use them; otherwise fall back to full scan
+    let currentPages: Map<string, PageMetadata>;
+    let needsFullScan = modifiedPages.length === 0;
+    
+    if (!needsFullScan) {
+      // Build a map from modified pages and merge with existing index
+      // This approach won't detect deleted pages without a full scan
+      currentPages = new Map(Array.from(existingIndex.entries()).map(([id, entry]) => [id, {
+        id: entry.id,
+        title: entry.title,
+        version: entry.version,
+        parentId: entry.parentId,
+        modifiedDate: entry.modifiedDate
+      }] as [string, PageMetadata]));
+      
+      // Update with modified pages
+      for (const page of modifiedPages) {
+        currentPages.set(page.id, page);
+      }
+      
+      console.log(`Note: Using incremental update. To detect deleted pages, run with --full flag.\n`);
+    } else {
+      // Full scan - fetch all pages metadata
+      console.log('Fetching all pages metadata (metadata only, no content)...\n');
+      
+      currentPages = new Map();
+      let pageCount = 0;
+      
+      for await (const page of api.getAllPagesMetadata(this.config.spaceKey, this.config.pageSize || 100)) {
+        currentPages.set(page.id, page);
+        pageCount++;
+        
+        // Progress indicator every 100 pages
+        if (pageCount % 100 === 0) {
+          console.log(`  Fetched ${pageCount} pages...`);
+        }
+      }
+      
+      console.log(`\nFetched ${currentPages.size} pages from space\n`);
+    }
+
+    // Compare and find changes
+    const newPages: PageMetadata[] = [];
+    const updatedPages: Array<{ current: PageMetadata; indexed: PageIndexEntry }> = [];
+    const deletedPages: PageIndexEntry[] = [];
+
+    // Find new and updated pages
+    for (const [id, current] of currentPages) {
+      const indexed = existingIndex.get(id);
+      
+      if (!indexed) {
+        newPages.push(current);
+      } else if (current.version && indexed.version && current.version > indexed.version) {
+        updatedPages.push({ current, indexed });
+      }
+    }
+
+    // Find deleted pages (only if we did a full scan)
+    if (!needsFullScan) {
+      // Skip deleted page detection for incremental updates
+    } else {
+      for (const [id, indexed] of existingIndex) {
+        if (!currentPages.has(id)) {
+          deletedPages.push(indexed);
+        }
+      }
+    }
+
+    // Report findings
+    console.log('=== Update Summary ===');
+    console.log(`New pages:     ${newPages.length}`);
+    console.log(`Updated pages: ${updatedPages.length}`);
+    if (needsFullScan) {
+      console.log(`Deleted pages: ${deletedPages.length}`);
+    } else {
+      console.log(`Deleted pages: (skipped - use --full for deletion detection)`);
+    }
+    console.log('');
+
+    if (newPages.length === 0 && updatedPages.length === 0 && deletedPages.length === 0) {
+      console.log('✓ Index is up to date. No changes needed.');
+      return;
+    }
+
+    // Show details
+    if (newPages.length > 0) {
+      console.log('\n--- New Pages ---');
+      for (const page of newPages) {
+        console.log(`  + ${page.title} (${page.id})`);
+      }
+    }
+
+    if (updatedPages.length > 0) {
+      console.log('\n--- Updated Pages ---');
+      for (const { current, indexed } of updatedPages) {
+        console.log(`  ~ ${current.title} (${current.id}) v${indexed.version} → v${current.version}`);
+      }
+    }
+
+    if (deletedPages.length > 0) {
+      console.log('\n--- Deleted Pages ---');
+      for (const page of deletedPages) {
+        console.log(`  - ${page.title} (${page.id})`);
+      }
+    }
+
+    // Update the index
+    console.log('\n\nUpdating _index.yaml...');
+    
+    // Build updated index
+    const updatedIndex: PageIndexEntry[] = [];
+    const now = new Date().toISOString();
+
+    for (const [id, current] of currentPages) {
+      const existing = existingIndex.get(id);
+      
+      if (existing && (!current.version || !existing.version || current.version === existing.version)) {
+        // Keep existing entry unchanged
+        updatedIndex.push(existing);
+      } else {
+        // New or updated page
+        updatedIndex.push({
+          id: current.id,
+          title: current.title,
+          version: current.version,
+          parentId: current.parentId,
+          modifiedDate: current.modifiedDate,
+          indexedDate: now,
+          pageNumber: existing?.pageNumber || 0
+        });
+      }
+    }
+
+    // Write updated index
+    const header = `# Confluence Export Index
+# Space: ${this.config.spaceKey}
+# Export Date: ${now}
+# Page Size: ${this.config.pageSize || 100}
+
+`;
+
+    const yamlContent = updatedIndex.map(entry => {
+      const yamlDoc = yaml.stringify(entry).trim();
+      const lines = yamlDoc.split('\n');
+      return lines.map((line, index) => {
+        if (index === 0) {
+          return `- ${line}`;
+        }
+        return `  ${line}`;
+      }).join('\n');
+    }).join('\n');
+
+    await fs.writeFile(indexPath, header + yamlContent + '\n', 'utf-8');
+
+    console.log(`\n✓ Index updated successfully!`);
+    console.log(`  Total pages in index: ${updatedIndex.length}`);
+    console.log(`  Added: ${newPages.length}, Updated: ${updatedPages.length}, Removed: ${deletedPages.length}`);
+  }
+}

package/src/index.ts

@@ -0,0 +1,120 @@
+#!/usr/bin/env node
+/**
+ * Minimal Confluence to Markdown Exporter - CLI Entry Point
+ */
+
+import minimist from 'minimist';
+import { config as loadEnv } from 'dotenv';
+import { promises as fs } from 'fs';
+import path from 'path';
+import { CommandExecutor } from './commands/executor.js';
+import { HelpCommand } from './commands/help.command.js';
+import type { ConfluenceConfig } from './types.js';
+import type { CommandContext } from './commands/types.js';
+
+async function main() {
+  // Load .env file if it exists
+  loadEnv();
+
+  // Parse command line arguments
+  const args = minimist(process.argv.slice(2), {
+    string: ['url', 'username', 'password', 'space', 'output', 'pageId', 'pageSize', 'limit', 'parallel'],
+    boolean: ['clear', 'force', 'debug'],
+    alias: {
+      u: 'url',
+      n: 'username',
+      p: 'password',
+      s: 'space',
+      o: 'output',
+      i: 'pageId',
+      l: 'limit',
+      f: 'force',
+      d: 'debug',
+      h: 'help'
+    }
+  });
+
+  // Show help if requested
+  if (args.help) {
+    const helpCommand = new HelpCommand();
+    await helpCommand.execute({ config: {} as ConfluenceConfig, args });
+    process.exit(0);
+  }
+
+  // Build config from args or environment variables
+  const config: ConfluenceConfig = {
+    baseUrl: args.url || process.env.CONFLUENCE_BASE_URL || '',
+    username: args.username || process.env.CONFLUENCE_USERNAME || '',
+    password: args.password || process.env.CONFLUENCE_PASSWORD || '',
+    spaceKey: args.space || process.env.CONFLUENCE_SPACE_KEY || '',
+    outputDir: args.output || process.env.OUTPUT_DIR || './output',
+    pageId: args.pageId || undefined,
+    pageSize: args.pageSize ? parseInt(args.pageSize, 10) : 100,
+    limit: args.limit ? parseInt(args.limit, 10) : undefined,
+    clear: args.clear || false,
+    force: args.force || false,
+    debug: args.debug || false,
+    parallel: args.parallel ? parseInt(args.parallel, 10) : 5
+  };
+
+  // Extract commands from positional arguments
+  let commands: string[];
+  if (args._.length === 0) {
+    // No commands provided - run full sync workflow
+    const indexPath = path.join(config.outputDir, '_index.yaml');
+    try {
+      await fs.access(indexPath);
+      commands = ['update', 'plan', 'download', 'transform'];
+    } catch {
+      commands = ['index', 'plan', 'download', 'transform'];
+    }
+  } else {
+    commands = args._ as string[];
+  }
+
+  // Configure logger debug mode if requested
+  if ((config as any).debug) {
+    // Lazy import to avoid top-level cycles
+    const { logger } = await import('./logger.js');
+    logger.setDebug(true);
+  }
+
+  // Validate config (except for help command which doesn't need it)
+  if (!config.baseUrl || !config.username || !config.password || !config.spaceKey) {
+    console.error('Error: Missing required configuration.\n');
+    console.error('Please provide all required options or set environment variables.');
+    console.error('Run with --help for usage information.\n');
+    process.exit(1);
+  }
+
+  const executor = new CommandExecutor(config);
+
+  // Validate commands
+  let requestedCommands: Awaited<ReturnType<typeof executor.validateCommands>>;
+  try {
+    requestedCommands = executor.validateCommands(commands);
+  } catch (error) {
+    console.error(`Error: ${error instanceof Error ? error.message : error}\n`);
+    const helpCommand = new HelpCommand();
+    await helpCommand.execute({ config: {} as ConfluenceConfig, args });
+    process.exit(1);
+  }
+
+  // Handle help command
+  if (requestedCommands.includes('help')) {
+    const helpCommand = new HelpCommand();
+    await helpCommand.execute({ config: {} as ConfluenceConfig, args });
+    process.exit(0);
+  }
+
+  try {
+    const context: CommandContext = { config, args };
+    await executor.executeCommands(requestedCommands, context);
+    process.exit(0);
+  } catch (error) {
+    console.error('\n✗ Command failed:', error instanceof Error ? error.message : error);
+    process.exit(1);
+  }
+}
+
+main();

package/src/logger.ts

@@ -0,0 +1,60 @@
+export class Logger {
+  private debugEnabled: boolean;
+  private lastDebugTime: number;
+  
+  // ANSI color codes
+  private colors = {
+    reset: '\x1b[0m',
+    gray: '\x1b[90m',
+    cyan: '\x1b[36m',
+    yellow: '\x1b[33m',
+    red: '\x1b[31m',
+    green: '\x1b[32m',
+    blue: '\x1b[34m',
+    magenta: '\x1b[35m',
+    white: '\x1b[37m'
+  };
+  
+  constructor(debug = false) {
+    this.debugEnabled = debug || !!process.env.DEBUG;
+    this.lastDebugTime = Date.now();
+  }
+
+  debug(...args: unknown[]) {
+    if (this.debugEnabled) {
+      const now = Date.now();
+      const elapsed = now - this.lastDebugTime;
+      this.lastDebugTime = now;
+      // eslint-disable-next-line no-console
+      console.debug(`${this.colors.cyan}[DEBUG +${elapsed}ms]${this.colors.reset}`, ...args);
+    }
+  }
+
+  setDebug(enabled: boolean) {
+    this.debugEnabled = !!enabled;
+  }
+
+  info(...args: unknown[]) {
+    // eslint-disable-next-line no-console
+    console.log(`${this.colors.blue}[INFO]${this.colors.reset}`, ...args);
+  }
+
+  warn(...args: unknown[]) {
+    // eslint-disable-next-line no-console
+    console.warn(`${this.colors.yellow}[WARN]${this.colors.reset}`, ...args);
+  }
+
+  error(...args: unknown[]) {
+    // eslint-disable-next-line no-console
+    console.error(`${this.colors.red}[ERROR]${this.colors.reset}`, ...args);
+  }
+
+  success(...args: unknown[]) {
+    // eslint-disable-next-line no-console
+    console.log(`${this.colors.green}[SUCCESS]${this.colors.reset}`, ...args);
+  }
+}
+
+const logger = new Logger();
+export { logger };
+export default logger;

package/src/test.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Test script for minimal Confluence exporter
+# This script demonstrates how to test the exporter
+
+echo "╔════════════════════════════════════════════════════╗"
+echo "║   Testing Minimal Confluence Exporter             ║"
+echo "╚════════════════════════════════════════════════════╝"
+echo ""
+
+# Check if Node.js version is adequate
+NODE_VERSION=$(node --version | cut -d'v' -f2 | cut -d'.' -f1)
+if [ "$NODE_VERSION" -lt 18 ]; then
+    echo "❌ Error: Node.js 18+ required (current: $(node --version))"
+    exit 1
+fi
+
+echo "✓ Node.js version: $(node --version)"
+echo ""
+
+# Build the project
+echo "📦 Building TypeScript..."
+npm run build
+
+if [ $? -ne 0 ]; then
+    echo "❌ Build failed!"
+    exit 1
+fi
+
+echo "✓ Build successful"
+echo ""
+
+# Check if environment variables are set
+if [ -z "$CONFLUENCE_BASE_URL" ] || [ -z "$CONFLUENCE_USERNAME" ] || [ -z "$CONFLUENCE_PASSWORD" ] || [ -z "$CONFLUENCE_SPACE_KEY" ]; then
+    echo "⚠️  Environment variables not set. Please set:"
+    echo ""
+    echo "  export CONFLUENCE_BASE_URL='https://your-instance.atlassian.net'"
+    echo "  export CONFLUENCE_USERNAME='your-email@example.com'"
+    echo "  export CONFLUENCE_PASSWORD='your-api-token'"
+    echo "  export CONFLUENCE_SPACE_KEY='YOURSPACE'"
+    echo "  export OUTPUT_DIR='./test-output'  # optional"
+    echo ""
+    echo "Or provide as command line arguments:"
+    echo "  npm run start -- <baseUrl> <username> <password> <spaceKey> [outputDir]"
+    echo ""
+    exit 1
+fi
+
+# Run the exporter
+echo "🚀 Running export..."
+echo "   Space: $CONFLUENCE_SPACE_KEY"
+echo "   Output: ${OUTPUT_DIR:-./output}"
+echo ""
+
+npm run start
+
+if [ $? -eq 0 ]; then
+    echo ""
+    echo "✓ Test completed successfully!"
+    echo ""
+    echo "Check the output directory for exported markdown files."
+else
+    echo ""
+    echo "❌ Test failed!"
+    exit 1
+fi

package/src/types.ts

@@ -0,0 +1,176 @@
+/**
+ * Minimal type definitions for Confluence export
+ */
+
+// ============================================================================
+// Configuration Types
+// ============================================================================
+
+export interface ConfluenceConfig {
+  baseUrl: string;
+  username: string;
+  password: string;
+  spaceKey: string;
+  outputDir: string;
+  pageId?: string; // Optional: if specified, export only this page
+  pageSize?: number; // Optional: number of items per API page (default: 25)
+  limit?: number; // Optional: maximum number of pages to process
+  clear?: boolean; // Optional: if specified, clears the output directory before export
+  force?: boolean; // Optional: if specified, forces re-download of all pages regardless of status
+  debug?: boolean; // Optional: enable debug logging
+  parallel?: number; // Optional: number of concurrent operations (default: 5)
+}
+
+// ============================================================================
+// Core Domain Types
+// ============================================================================
+
+export interface Page {
+  id: string;
+  title: string;
+  body: string;
+  version?: number;
+  parentId?: string;
+  modifiedDate?: string;
+}
+
+export interface User {
+  userKey: string;
+  username: string;
+  displayName: string;
+  email?: string;
+}
+
+// ============================================================================
+// API Response Types
+// ============================================================================
+
+export interface PaginatedResponse<T> {
+  results: T[];
+  start: number;
+  limit: number;
+  size: number;
+  _links?: {
+    next?: string;
+  };
+}
+
+export interface PageResponse {
+  id: string;
+  title: string;
+  body?: { storage?: { value: string } };
+  version?: { number: number; when?: string };
+  ancestors?: Array<{ id: string }>;
+  history?: { lastUpdated?: { when?: string } };
+}
+
+export interface RawPage {
+  id: string;
+  title: string;
+  body?: { storage?: { value: string } };
+  version?: { number: number; when?: string };
+  ancestors?: Array<{ id: string }>;
+  history?: { lastUpdated?: { when?: string } };
+}
+
+export interface ListPagesResponse {
+  results: RawPage[];
+  start: number;
+  limit: number;
+  size: number;
+  _links?: {
+    next?: string;
+  };
+}
+
+export interface ChildPageResponse {
+  id: string;
+  title: string;
+  version?: { number: number };
+}
+
+export interface ChildPagesResponse {
+  results: ChildPageResponse[];
+}
+
+export interface AttachmentResult {
+  id: string;
+  title: string;
+  _links: {
+    download: string;
+  };
+}
+
+export interface AttachmentResponse {
+  results: AttachmentResult[];
+}
+
+// ============================================================================
+// Index & Export Types
+// ============================================================================
+
+export interface PageMetadata {
+  id: string;
+  title: string;
+  version?: number;
+  parentId?: string;
+  modifiedDate?: string;
+}
+
+export interface PageIndexEntry {
+  id: string;
+  title: string;
+  version?: number;
+  parentId?: string;
+  modifiedDate?: string;
+  indexedDate: string;
+  pageNumber: number;
+  downloadedVersion?: number;  // Last downloaded version
+  downloadedAt?: string;       // Last download timestamp (ISO 8601)
+  queueReason?: 'new' | 'updated';
+}
+
+export interface PageIndex {
+  spaceKey: string;
+  exportDate: string;
+  totalPages: number;
+  pages: PageIndexEntry[];
+}
+
+export interface PageTreeNode {
+  id: string;
+  title: string;
+  version?: number;
+  parentId?: string;
+  modifiedDate?: string;
+  children?: PageTreeNode[];
+}
+
+// ============================================================================
+// Transformation Types
+// ============================================================================
+
+export interface MarkdownResult {
+  content: string;
+  frontMatter: {
+    title: string;
+    id: string;
+    version?: number;
+    parentId?: string;
+  };
+  images: Array<{
+    filename: string;
+    data: Buffer;
+  }>;
+}
+
+/**
+ * Metadata sidecar file for tracking download state
+ * Stored as .meta.json alongside each .html file
+ */
+export interface PageMeta {
+  pageId: string;
+  version: number;
+  modifiedDate: string;
+  downloadedAt: string;
+}