shoko 0.1.0

data/lib/shoko/adapters/book_sources/epub_document.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require_relative '../monitoring/logger.rb'
+require_relative '../monitoring/performance_monitor.rb'
+require_relative '../monitoring/perf_tracer.rb'
+require_relative '../storage/book_cache_pipeline.rb'
+require_relative 'epub/parsers/html_processor.rb'
+require_relative '../output/terminal/terminal_sanitizer.rb'
+require_relative '../../core/models/chapter.rb'
+require_relative '../../core/models/toc_entry.rb'
+
+module Shoko
+  module Adapters::BookSources
+    # Represents an EPUB document backed by the cache pipeline.
+    # The document always operates on in-memory chapter objects; no temporary
+    # extraction to disk is required.
+    class EPUBDocument
+    attr_reader :title, :chapters, :language, :source_path,
+                :cache_path, :cache_sha, :toc_entries, :metadata, :resources
+
+    def initialize(path, formatting_service: nil, background_worker: nil, progress_reporter: nil)
+      @open_path = File.expand_path(path)
+      @formatting_service = formatting_service
+      @background_worker = background_worker
+      @progress_reporter = progress_reporter
+      @formatting_pending = {}
+      @formatting_pending_mutex = Mutex.new
+
+      @title = fallback_title(@open_path)
+      @language = 'en_US'
+      @chapters = []
+      @toc_entries = []
+      @metadata = {}
+      @resources = {}
+      @chapter_hrefs = []
+      @spine_relative_paths = []
+      @opf_path = nil
+      @cache_path = nil
+      @source_path = @open_path
+      @loaded_from_cache = false
+      @book_payload = nil
+
+      load_via_pipeline!
+    rescue Shoko::Error => e
+      create_error_chapter(e)
+    rescue StandardError => e
+      Adapters::Monitoring::Logger.error('EPUBDocument initialization failed', path: @open_path, error: e.message)
+      create_error_chapter(e)
+    end
+
+    def chapter_count
+      @chapters.size
+    end
+
+    def get_chapter(index)
+      return nil unless index.is_a?(Integer) && index >= 0 && index < @chapters.length
+
+      chapter = @chapters[index]
+      return nil unless chapter
+
+      ensure_formatted_chapter(chapter, index)
+      chapter.lines = fallback_plain_lines(chapter.raw_content) if chapter.lines.nil? || chapter.lines.empty?
+      chapter
+    end
+
+    def cached?
+      @loaded_from_cache
+    end
+
+    def canonical_path
+      @source_path || @open_path
+    end
+
+    # Backwards compatibility for components that previously expected a cache
+    # directory. Returns the directory containing the `.cache` file.
+    def cache_dir
+      return nil unless @cache_path
+
+      File.dirname(@cache_path)
+    end
+
+    private
+
+    def load_via_pipeline!
+      result = Adapters::Monitoring::PerformanceMonitor.time('import.pipeline') do
+        Adapters::Monitoring::PerfTracer.measure('cache.pipeline') do
+          Adapters::Storage::BookCachePipeline.new(progress_reporter: @progress_reporter)
+                                           .load(@open_path, formatting_service: @formatting_service)
+        end
+      end
+
+      apply_pipeline_result(result)
+    end
+
+    def apply_pipeline_result(result)
+      book = result.book
+      if defined?(Adapters::Monitoring::PerfTracer)
+        Adapters::Monitoring::PerfTracer.annotate(
+          cache_hit: result.loaded_from_cache,
+          chapters: Array(book&.chapters).size,
+          book: result.source_path || @open_path
+        )
+      end
+      @cache_path = result.cache_path
+      @cache_sha = derive_cache_sha(@cache_path)
+      @source_path = result.source_path || @open_path
+      @loaded_from_cache = result.loaded_from_cache
+      @book_payload = result.payload
+
+      @title = present_or_fallback(book.title, fallback_title(@source_path))
+      @language = book.language || @language
+      @metadata = book.metadata || {}
+      @chapters = Array(book.chapters).dup
+      @toc_entries = Array(book.toc_entries).dup
+      @resources = (book.resources || {}).dup
+      @chapter_hrefs = Array(book.chapter_hrefs).dup
+      @spine_relative_paths = Array(book.spine).dup
+      @opf_path = book.opf_path
+
+      ensure_chapters_exist
+    end
+
+    def derive_cache_sha(path)
+      return nil unless path && !path.to_s.empty?
+
+      File.basename(path.to_s, File.extname(path.to_s))
+    rescue StandardError
+      nil
+    end
+
+    def present_or_fallback(value, fallback)
+      str = value.to_s.strip
+      str.empty? ? fallback : value
+    end
+
+    def ensure_chapters_exist
+      return unless @chapters.empty?
+
+      @chapters << Core::Models::Chapter.new(
+        number: '1',
+        title: 'Empty Book',
+        lines: ['This EPUB appears to be empty.'],
+        metadata: nil,
+        blocks: nil,
+        raw_content: nil
+      )
+      @toc_entries = []
+    end
+
+    def create_error_chapter(error)
+      @chapters = [
+        Core::Models::Chapter.new(
+          number: '1',
+          title: 'Error Loading',
+          lines: ["Error: #{error.message}"],
+          metadata: nil,
+          blocks: nil,
+          raw_content: nil
+        ),
+      ]
+      @toc_entries = []
+    end
+
+    def ensure_formatted_chapter(chapter, index)
+      return unless @formatting_service && chapter
+
+      chapter_index = index.to_i
+      return if chapter.blocks && !chapter.blocks.empty?
+
+      # Always format synchronously so rendering/pagination receives structured lines immediately.
+      format_chapter_sync(chapter_index, chapter, raise_on_error: true)
+    end
+
+    def fallback_plain_lines(content)
+      return [] unless content
+
+      text = Adapters::BookSources::Epub::Parsers::HTMLProcessor.html_to_text(content.to_s)
+      text.split("\n").map(&:rstrip)
+    end
+
+    def fallback_title(path)
+      raw = File.basename(path, File.extname(path)).tr('_', ' ')
+      Shoko::Adapters::Output::Terminal::TerminalSanitizer.sanitize(raw, preserve_newlines: false, preserve_tabs: false)
+    end
+
+    def enqueue_async_formatting(index, chapter)
+      already_enqueued = false
+      @formatting_pending_mutex.synchronize do
+        already_enqueued = @formatting_pending[index]
+        @formatting_pending[index] = true unless already_enqueued
+      end
+      return if already_enqueued
+
+      @background_worker.submit do
+        format_chapter_sync(index, chapter, raise_on_error: false)
+      ensure
+        @formatting_pending_mutex.synchronize do
+          @formatting_pending.delete(index)
+        end
+      end
+    rescue StandardError => e
+      Adapters::Monitoring::Logger.debug('Async formatting enqueue failed', error: e.message)
+    end
+
+    def format_chapter_sync(index, chapter, raise_on_error:)
+      Adapters::Monitoring::PerfTracer.measure('formatting.ensure') do
+        @formatting_service.ensure_formatted!(self, index, chapter)
+      end
+    rescue Shoko::FormattingError => e
+      Adapters::Monitoring::Logger.error('Formatting error', error: e.message, chapter: index + 1)
+      raise if raise_on_error
+    rescue StandardError => e
+      Adapters::Monitoring::Logger.debug('Formatting service failed', error: e.message, chapter: index + 1)
+      nil
+    end
+
+    def assign_toc_entries(entries)
+      href_to_index = {}
+      Array(@chapter_hrefs).each_with_index do |href, idx|
+        href_to_index[href] = idx if href
+      end
+
+      @toc_entries = Array(entries).map do |entry|
+        title = entry[:title] || entry['title']
+        href = entry[:href] || entry['href']
+        level = (entry[:level] || entry['level']).to_i
+        resolved = normalize_toc_href(href)
+        chapter_index = href_to_index[resolved]
+
+        if chapter_index && (chapter = @chapters[chapter_index]) && chapter.respond_to?(:title=) && (chapter.title.nil? || chapter.title.to_s.strip.empty?)
+          chapter.title = title
+        end
+
+        Core::Models::TOCEntry.new(
+          title: title,
+          href: href,
+          level: level,
+          chapter_index: chapter_index,
+          navigable: !chapter_index.nil?
+        )
+      end
+    end
+
+    def normalize_toc_href(href)
+      return nil unless href
+
+      base = @opf_path ? File.dirname(@opf_path) : '.'
+      core = href.to_s.split('#', 2).first
+      File.expand_path(File.join('/', base, core), '/').sub(%r{^/}, '')
+    end
+    end
+  end
+end

data/lib/shoko/adapters/book_sources/epub_finder/directory_scanner.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require 'time'
+
+require_relative '../../output/terminal/terminal_sanitizer.rb'
+require_relative '../../storage/config_paths.rb'
+
+module Shoko
+  module Adapters::BookSources
+    class EPUBFinder
+      # Scans directories to locate EPUB files
+      class DirectoryScanner
+        def initialize(context)
+          @context = context
+        end
+
+        def scan_all_directories
+          all_dirs = build_directory_list
+          warn_debug "Scanning directories: #{all_dirs.join(', ')}"
+
+          all_dirs.each do |dir|
+            break if @context.epubs.length >= EPUBFinder::MAX_FILES
+
+            warn_debug "Scanning: #{dir}"
+            scan_directory(dir)
+          end
+        end
+
+        private
+
+        def build_directory_list
+          directories = priority_directories + other_directories
+          directories.uniq.select { |dir| safe_directory_exists?(dir) }
+        end
+
+        def priority_directories
+          [
+            Adapters::Storage::ConfigPaths.downloads_root,
+            '~/Books',
+            '~/Bücher', # German books directory
+            '~/Documents/Books',
+            '~/Downloads',
+            '~/Desktop',
+            '~/Documents',
+            '~/Library/Mobile Documents',
+          ].map { |dir| File.expand_path(dir) }
+        end
+
+        def other_directories
+          [
+            '~',
+            '~/Dropbox',
+            '~/Google Drive',
+            '~/OneDrive',
+          ].map { |dir| File.expand_path(dir) }
+        end
+
+        def safe_directory_exists?(dir)
+          Dir.exist?(dir)
+        rescue StandardError
+          false
+        end
+
+        def scan_directory(dir)
+          return unless @context.can_scan?(dir, EPUBFinder::MAX_DEPTH, EPUBFinder::MAX_FILES)
+
+          @context.mark_visited(dir)
+          process_entries(dir)
+        rescue Errno::EACCES, Errno::ENOENT, Errno::EPERM
+          # Skip directories we can't access
+        rescue StandardError => e
+          warn_debug "Error scanning #{dir}: #{e.message}"
+        end
+
+        def process_entries(dir)
+          Dir.entries(dir).each do |entry|
+            next if entry.start_with?('.')
+
+            path = File.join(dir, entry)
+            next if @context.visited_paths.include?(path)
+
+            process_path(path)
+          end
+        end
+
+        def process_path(path)
+          if File.directory?(path)
+            process_directory(path)
+          elsif epub_file?(path)
+            add_epub(path)
+          end
+        rescue StandardError
+          # Skip items we can't process
+        end
+
+        def process_directory(path)
+          return if skip_directory?(path)
+
+          DirectoryScanner.new(@context.with_deeper_depth).scan_directory(path)
+        end
+
+        def skip_directory?(path)
+          base = File.basename(path).downcase
+          EPUBFinder::SKIP_DIRS.map(&:downcase).include?(base)
+        end
+
+        def epub_file?(path)
+          path.downcase.end_with?('.epub') &&
+            File.readable?(path) &&
+            File.size(path).positive?
+        end
+
+        def add_epub(path)
+          raw_name = File.basename(path, '.epub').gsub(/[_-]/, ' ')
+          display_name = Shoko::Adapters::Output::Terminal::TerminalSanitizer.sanitize(raw_name, preserve_newlines: false, preserve_tabs: false)
+
+          @context.epubs << {
+            'path' => path,
+            'name' => display_name,
+            'size' => File.size(path),
+            'modified' => File.mtime(path).iso8601,
+            'dir' => File.dirname(path),
+          }
+        end
+
+        def debug? = EPUBFinder::DEBUG_MODE
+
+        def warn_debug(msg)
+          warn msg if debug?
+        end
+      end
+    end
+  end
+end

data/lib/shoko/adapters/book_sources/epub_finder/scanner_context.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Shoko
+  module Adapters::BookSources
+    class EPUBFinder
+      # Context for directory scanning operations
+      ScannerContext = Struct.new(:epubs, :visited_paths, :depth, keyword_init: true) do
+        def can_scan?(dir, max_depth, max_files)
+          depth <= max_depth &&
+            epubs.length < max_files &&
+            !visited_paths.include?(dir)
+        end
+
+        def mark_visited(dir)
+          visited_paths.add(dir)
+        end
+
+        def with_deeper_depth
+          self.class.new(
+            epubs: epubs,
+            visited_paths: visited_paths,
+            depth: depth + 1
+          )
+        end
+      end
+    end
+  end
+end

data/lib/shoko/adapters/book_sources/epub_finder.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'set'
+require 'time'
+require 'timeout'
+
+require_relative 'epub_finder/scanner_context'
+require_relative 'epub_finder/directory_scanner'
+require_relative '../storage/atomic_file_writer.rb'
+require_relative '../storage/config_paths.rb'
+
+module Shoko
+  module Adapters::BookSources
+    # EPUB file finder with robust error handling
+    class EPUBFinder
+      SCAN_TIMEOUT = 20
+      MAX_DEPTH = 3
+      MAX_FILES = 500
+      CACHE_DURATION = 86_400
+      CONFIG_DIR = Adapters::Storage::ConfigPaths.config_root
+      CACHE_FILE = File.join(CONFIG_DIR, 'epub_cache.json')
+      DEBUG_MODE = ARGV.include?('--debug') || ENV['DEBUG']
+      SKIP_DIRS = %w[
+        node_modules vendor cache tmp temp .git .svn
+        __pycache__ build dist bin obj debug release
+        .idea .vscode .atom .sublime library frameworks
+        applications system windows programdata appdata
+        .Trash .npm .gem .bundle .cargo .rustup .cache
+        .local .config backup backups old archive
+      ].freeze
+
+      class << self
+        def scan_system(force_refresh: false)
+          cache = load_cache
+          return cache['files'] if !force_refresh && cache_valid?(cache)
+
+          scan_with_timeout
+        end
+
+        def clear_cache
+          FileUtils.rm_f(CACHE_FILE)
+        rescue StandardError
+          nil
+        end
+
+        private
+
+        def cached_files
+          cache = load_cache
+          return [] unless cache_valid?(cache)
+
+          cache['files']
+        end
+
+        def cache_valid?(cache)
+          return false unless cache.is_a?(Hash)
+
+          files = cache['files']
+          ts = cache['timestamp']
+          files.is_a?(Array) && ts && !cache_expired?(ts)
+        end
+
+        def cache_expired?(timestamp = nil)
+          ts = timestamp.to_s.strip
+          return true if ts.empty?
+
+          Time.now - Time.parse(ts) >= CACHE_DURATION
+        rescue StandardError
+          true
+        end
+
+        def scan_with_timeout
+          epubs = []
+          epubs = perform_scan_with_timeout
+        rescue Timeout::Error
+          handle_timeout_error(epubs)
+        rescue StandardError
+          epubs = cached_files_fallback
+        ensure
+          save_and_return_epubs(epubs)
+        end
+
+        def perform_scan_with_timeout
+          Timeout.timeout(SCAN_TIMEOUT) { perform_scan }
+        end
+
+        def handle_timeout_error(epubs)
+          save_cache(epubs) unless epubs.empty?
+          epubs
+        end
+
+        def save_and_return_epubs(epubs)
+          save_cache(epubs)
+          epubs
+        end
+
+        def cached_files_fallback
+          cache = load_cache
+          cache && cache['files'] ? cache['files'] : []
+        end
+
+        def perform_scan
+          epubs = []
+          context = ScannerContext.new(
+            epubs: epubs,
+            visited_paths: Set.new,
+            depth: 0
+          )
+
+          scanner = DirectoryScanner.new(context)
+          scanner.scan_all_directories
+
+          warn_debug "Found #{epubs.length} EPUB files"
+          epubs
+        end
+
+        def load_cache
+          return nil unless File.exist?(CACHE_FILE)
+
+          parse_cache_file(CACHE_FILE)
+        rescue StandardError => e
+          warn_debug "Cache load error: #{e.message}"
+          delete_cache_file(CACHE_FILE)
+          nil
+        end
+
+        def parse_cache_file(path)
+          data = File.read(path)
+          json = JSON.parse(data)
+          json if json.is_a?(Hash)
+        end
+
+        def delete_cache_file(path)
+          File.delete(path)
+        rescue StandardError
+          nil
+        end
+
+        def save_cache(files)
+          FileUtils.mkdir_p(File.dirname(CACHE_FILE))
+          payload = JSON.pretty_generate({
+                                           'timestamp' => Time.now.iso8601,
+                                           'files' => files || [],
+                                           'version' => VERSION,
+                                         })
+          Adapters::Storage::AtomicFileWriter.write(CACHE_FILE, payload)
+        rescue StandardError => e
+          warn_debug "Cache save error: #{e.message}"
+        end
+
+        def debug? = !!DEBUG_MODE
+
+        def warn_debug(msg)
+          warn msg if debug?
+        end
+      end
+    end
+  end
+end