shoko 0.1.3 → 0.1.5

data/lib/shoko/core/book_formats/pdf/pdf_content_parser.rb

@@ -0,0 +1,687 @@
+# frozen_string_literal: true
+
+require 'json'
+
+require_relative '../../models/content_block'
+
+module Shoko
+  module Core
+    module BookFormats
+      module Pdf
+        # Converts extracted PDF chapter payloads into semantic content blocks.
+        #
+        # Supports:
+        # - legacy plain text payloads
+        # - structured layout payloads generated by PdfImporter/PdfTextExtractor
+        class PdfContentParser
+          LAYOUT_FORMAT = 'pdf-layout-v1'
+          private_constant :LAYOUT_FORMAT
+
+          # @param raw_text [String] text/layout payload extracted from PDF pages
+          # @param logger [Object, nil]
+          def initialize(raw_text, logger: nil)
+            @raw_text = raw_text.to_s
+            @logger = logger
+          end
+
+          # @return [Array<Core::Models::ContentBlock>]
+          def parse
+            return [] if @raw_text.strip.empty?
+
+            layout = parse_layout_payload(@raw_text)
+            if layout
+              blocks = parse_layout_blocks(layout)
+              return blocks unless blocks.empty?
+            end
+
+            paragraphs = split_paragraphs(@raw_text)
+            paragraphs.filter_map { |para| build_paragraph_block(para) }
+          rescue Shoko::Error => e
+            @logger&.debug('PDF content parse failed', error: e.message)
+            fallback_blocks
+          end
+
+          private
+
+          def parse_layout_payload(text)
+            parsed = parse_json_payload(text.to_s.strip)
+            normalized = normalize_layout_payload(parsed)
+            return normalized if normalized
+
+            recover_layout_from_jsonish_text(text.to_s)
+          end
+
+          def parse_layout_blocks(layout)
+            lines = normalize_layout_lines(layout[:lines])
+            return [] if lines.empty?
+
+            metrics = line_metrics(lines)
+            groups = build_layout_groups(lines, metrics)
+            groups.flat_map { |group| blocks_for_group(group) }
+          end
+
+          def normalize_layout_lines(lines)
+            normalized = []
+            Array(lines).each do |line|
+              if line.is_a?(String)
+                text = normalize_line_text(line)
+                if text.empty?
+                  normalized << { break: true }
+                  next
+                end
+
+                normalized << { text: text, x: nil, italic: false, italic_ratio: nil }
+                next
+              end
+
+              hash = line.is_a?(Hash) ? line : {}
+              break_flag = line_break?(hash)
+              text = normalize_line_text(extract_line_text(hash))
+              if break_flag || text.empty?
+                normalized << { break: true }
+                next
+              end
+
+              italic_ratio = extract_line_italic_ratio(hash)
+              normalized << {
+                text: text,
+                x: extract_line_x(hash),
+                italic: extract_line_italic(hash, italic_ratio: italic_ratio),
+                italic_ratio: italic_ratio,
+              }
+            end
+            normalized
+          end
+
+          def line_metrics(lines)
+            x_values = lines.filter_map { |line| line[:x] unless line[:break] }
+            min_x = x_values.min || 0.0
+            max_x = x_values.max || min_x
+            span = [max_x - min_x, 1.0].max
+            baseline_x = percentile_x(x_values, 0.15) || min_x
+            max_ref_x = percentile_x(x_values, 0.995) || max_x
+            ref_span = [max_ref_x - baseline_x, 1.0].max
+            {
+              min_x: min_x,
+              max_x: max_x,
+              baseline_x: baseline_x,
+              max_ref_x: max_ref_x,
+              span: span,
+              ref_span: ref_span,
+            }
+          end
+
+          def build_layout_groups(lines, metrics)
+            groups = []
+            current = nil
+            content_index = -1
+            preamble_open = true
+
+            lines.each_with_index do |line, idx|
+              if line[:break]
+                groups << current if current
+                current = nil
+                next
+              end
+
+              content_index += 1
+              align = alignment_for(line, metrics)
+              previous_kind = current ? current[:kind] : groups.last&.[](:kind)
+              next_line = next_content_line(lines, idx)
+              preamble_open = false if preamble_open && body_line_candidate?(line, align, content_index,
+                                                                              previous_kind: previous_kind,
+                                                                              next_line: next_line)
+              prev_break = previous_is_break?(lines, idx)
+              next_break = next_is_break?(lines, idx)
+              prev_line = previous_content_line(lines, idx)
+              heading = heading_line?(line[:text], align,
+                                      prev_break: prev_break,
+                                      next_break: next_break,
+                                      next_line: next_line,
+                                      prev_line: prev_line)
+              kind = if heading
+                       :heading
+                     elsif epigraph_line?(line, align, content_index,
+                                          previous_kind: previous_kind,
+                                          next_line: next_line,
+                                          preamble_open: preamble_open)
+                       :epigraph
+                     else
+                       :paragraph
+                     end
+              align = normalize_group_alignment(kind, align, preamble_open: preamble_open)
+
+              if mergeable_group?(current, kind, align)
+                current[:align] = :right if kind == :epigraph && (current[:align] == :right || align == :right)
+                current[:lines] << line
+              else
+                groups << current if current
+                current = { kind: kind, align: align, lines: [line] }
+              end
+            end
+
+            groups << current if current
+            groups
+          end
+
+          def blocks_for_group(group)
+            case group[:kind]
+            when :heading
+              [build_heading_block(group[:lines])]
+            when :epigraph
+              build_epigraph_blocks(group[:lines], align: group[:align])
+            else
+              [build_paragraph_from_lines(group[:lines], align: group[:align])]
+            end.compact
+          end
+
+          def build_heading_block(lines)
+            text = lines.map { |line| line[:text] }.join(' ').strip
+            return nil if text.empty?
+
+            Core::Models::ContentBlock.new(
+              type: :heading,
+              segments: [Core::Models::TextSegment.new(text: text, styles: {})],
+              level: 1,
+              metadata: { level: 1, align: :center }
+            )
+          end
+
+          def build_epigraph_blocks(lines, align:)
+            return [] if lines.empty?
+
+            quote_lines = lines.dup
+            attribution = nil
+            if quote_lines.length > 1 && attribution_signature_line?(quote_lines.last[:text])
+              attribution = quote_lines.pop
+            end
+
+            blocks = []
+            quote_block = build_epigraph_quote_block(quote_lines, align: align)
+            blocks << quote_block if quote_block
+            if attribution
+              blocks << Core::Models::ContentBlock.new(
+                type: :paragraph,
+                segments: [Core::Models::TextSegment.new(text: attribution[:text], styles: {})],
+                level: 0,
+                metadata: { style: :attribution, align: :right }
+              )
+            end
+            blocks
+          end
+
+          def build_epigraph_quote_block(lines, align:)
+            return nil if lines.empty?
+
+            force_italic = !lines.any? { |line| line[:italic] }
+            segments = []
+            lines.each_with_index do |line, index|
+              style = {}
+              style[:italic] = true if force_italic || line[:italic]
+              segments << Core::Models::TextSegment.new(text: line[:text], styles: style)
+              next if index >= lines.length - 1
+
+              segments << Core::Models::TextSegment.new(text: "\n", styles: style)
+            end
+
+            Core::Models::ContentBlock.new(
+              type: :quote,
+              segments: segments,
+              level: 1,
+              metadata: { style: :epigraph, align: normalize_epigraph_align(align) }
+            )
+          end
+
+          def build_paragraph_from_lines(lines, align:)
+            text = lines.map { |line| line[:text] }.join(' ').gsub(/\s+/, ' ').strip
+            return nil if text.empty?
+
+            metadata = {}
+            metadata[:align] = align if align && align != :left
+            Core::Models::ContentBlock.new(
+              type: :paragraph,
+              segments: [Core::Models::TextSegment.new(text: text, styles: {})],
+              level: 0,
+              metadata: metadata
+            )
+          end
+
+          def alignment_for(line, metrics)
+            x = line[:x]
+            return :left unless x
+
+            delta = x - metrics[:baseline_x].to_f
+            return :left if delta <= left_tolerance(metrics)
+            return :right if delta >= right_cutoff(metrics)
+            return :center if delta >= center_cutoff(metrics)
+
+            :left
+          end
+
+          def heading_line?(text, align, prev_break:, next_break:, next_line:, prev_line:)
+            compact = text.to_s.strip
+            return false if compact.empty?
+            return false unless prev_break || next_break
+            return false if attribution_signature_line?(compact)
+            return false if next_line && attribution_signature_line?(next_line[:text])
+            return false if likely_attribution_context?(compact, prev_line: prev_line)
+            return true if compact.match?(/\A[IVXLCDM\d]+\z/i)
+            return false unless align == :center || align == :right
+            return false if compact.length > 64
+            return false if compact.split(/\s+/).length > 8
+            return false if compact.match?(/[.!?:;]\z/)
+
+            true
+          end
+
+          def epigraph_line?(line, align, content_index, previous_kind:, next_line:, preamble_open:)
+            return false unless preamble_open
+            return false unless content_index <= 40
+            return false unless epigraph_length_candidate?(line[:text])
+
+            return true if attribution_signature_line?(line[:text]) && previous_kind == :epigraph && align != :left
+            return true if quote_line_followed_by_attribution?(line[:text], next_line, content_index)
+            return true if content_index <= 24 && italic_dominant?(line)
+            return true if align == :right
+            return false if align == :left
+
+            italic_dominant?(line)
+          end
+
+          def mergeable_group?(current, kind, align)
+            return false unless current
+            return false if kind == :heading
+            return false unless current[:kind] == kind
+
+            kind == :epigraph || current[:align] == align
+          end
+
+          def normalize_group_alignment(kind, align, preamble_open:)
+            return align unless kind == :paragraph
+            return align if preamble_open
+
+            :left
+          end
+
+          def attribution_signature_line?(text)
+            compact = text.to_s.strip
+            return false if compact.empty? || compact.length > 80
+
+            return true if author_source_signature?(compact)
+
+            letters = compact.scan(/[A-Za-z]/)
+            return false if letters.empty?
+
+            uppercase = letters.count { |ch| ch == ch.upcase }
+            ratio = uppercase.to_f / letters.length
+            has_year = compact.match?(/\b(1[5-9]\d{2}|20\d{2})\b/)
+            has_delimiter = compact.include?(',') || compact.include?(' - ') || compact.include?(' — ')
+
+            return true if ratio >= 0.58
+            return true if ratio >= 0.40 && has_delimiter
+            return true if ratio >= 0.30 && has_delimiter && has_year
+
+            false
+          end
+
+          def author_source_signature?(text)
+            compact = text.to_s.strip
+            return false unless compact.include?(',')
+
+            parts = compact.split(',').map(&:strip).reject(&:empty?)
+            return false if parts.length < 2
+
+            author = parts.first
+            return false if author.length < 4 || author.length > 48
+            return false if author.match?(/\d|[:;\/]/)
+
+            words = author.split(/\s+/)
+            return false unless words.length.between?(2, 6)
+            return false if words.any? { |word| attribution_author_stopword?(word) }
+
+            author_like_words = words.count { |word| author_name_token?(word) }
+            return false if author_like_words < [words.length - 1, 1].max
+
+            source = parts[1..].join(',').strip
+            return false if source.empty? || source.length > 72
+            return false if source.match?(/\A\d+\z/)
+
+            true
+          end
+
+          def attribution_author_stopword?(token)
+            normalized = token.to_s.downcase.gsub(/[^a-z]/, '')
+            return false if normalized.empty?
+
+            %w[copyright published printing edition registered offices group books ltd inc foundation].include?(normalized)
+          end
+
+          def author_name_token?(token)
+            compact = token.to_s.gsub(/\A[“"'\(]+|[”"'\)\.]+\z/, '')
+            return false if compact.empty?
+
+            compact.match?(/\A[A-Z]{2,}(?:-[A-Z]{2,})*\z/) ||
+              compact.match?(/\A[A-Z][a-z]+(?:[-'’][A-Z][a-z]+)*\z/) ||
+              compact.match?(/\A[A-Z]\z/)
+          end
+
+          def likely_attribution_context?(text, prev_line:)
+            return false unless prev_line
+
+            compact = text.to_s.strip
+            return false unless compact.include?(',')
+            return false unless sentence_like_epigraph_line?(prev_line[:text])
+
+            compact.length <= 80 && compact.split(/\s+/).length <= 10
+          end
+
+          def normalize_epigraph_align(align)
+            return :right if align == :right
+            return :center if align == :center
+
+            :right
+          end
+
+          def quote_line_followed_by_attribution?(text, next_line, content_index)
+            return false unless content_index <= 24
+            return false unless next_line
+            return false unless sentence_like_epigraph_line?(text)
+
+            attribution_signature_line?(next_line[:text])
+          end
+
+          def sentence_like_epigraph_line?(text)
+            compact = text.to_s.strip
+            return false if compact.empty?
+            return false unless compact.match?(/[.!?]\z/)
+
+            words = compact.split(/\s+/)
+            words.length >= 4 && words.length <= 20
+          end
+
+          def epigraph_length_candidate?(text)
+            compact = text.to_s.strip
+            return false if compact.empty?
+
+            words = compact.split(/\s+/).length
+            words <= 24 && compact.length <= 200
+          end
+
+          def italic_dominant?(line)
+            ratio = line[:italic_ratio]
+            if ratio
+              return ratio.to_f >= 0.65
+            end
+
+            !!line[:italic]
+          end
+
+          def body_line_candidate?(line, align, content_index, previous_kind:, next_line:)
+            return false unless content_index <= 160
+            return false unless align == :left
+            return false if previous_kind == :epigraph
+            return false if content_index <= 24 && italic_dominant?(line)
+            return false if quote_line_followed_by_attribution?(line[:text], next_line, content_index)
+
+            compact = line[:text].to_s.strip
+            return false if compact.empty?
+            return false if attribution_signature_line?(compact)
+            return false if compact.match?(/\A[IVXLCDM\d]+\z/i)
+
+            words = compact.split(/\s+/).length
+            words >= 8 || compact.length >= 48
+          end
+
+          def next_content_line(lines, start_idx)
+            idx = start_idx.to_i + 1
+            while idx < lines.length
+              line = lines[idx]
+              return line unless line[:break]
+
+              idx += 1
+            end
+            nil
+          end
+
+          def previous_content_line(lines, start_idx)
+            idx = start_idx.to_i - 1
+            while idx >= 0
+              line = lines[idx]
+              return line unless line[:break]
+
+              idx -= 1
+            end
+            nil
+          end
+
+          def previous_is_break?(lines, idx)
+            pos = idx.to_i - 1
+            return true if pos.negative?
+
+            lines[pos][:break] == true
+          end
+
+          def next_is_break?(lines, idx)
+            pos = idx.to_i + 1
+            return true if pos >= lines.length
+
+            lines[pos][:break] == true
+          end
+
+          def split_paragraphs(text)
+            # Split on two or more consecutive newlines (paragraph boundary)
+            text.split(/\n{2,}/).map do |para|
+              # Join single newlines within a paragraph (PDF wraps at fixed page width)
+              para.gsub(/\n/, ' ').gsub(/\s+/, ' ').strip
+            end.reject(&:empty?)
+          end
+
+          def build_paragraph_block(text)
+            return nil if text.empty?
+
+            segments = [Core::Models::TextSegment.new(text: text, styles: {})]
+            Core::Models::ContentBlock.new(
+              type: :paragraph,
+              segments: segments,
+              level: 0,
+              metadata: {}
+            )
+          end
+
+          def fallback_blocks
+            text = @raw_text.gsub(/\s+/, ' ').strip
+            return [] if text.empty?
+
+            [Core::Models::ContentBlock.new(
+              type: :paragraph,
+              segments: [Core::Models::TextSegment.new(text: text, styles: {})],
+              level: 0,
+              metadata: {}
+            )]
+          end
+
+          def truthy?(value)
+            value == true || value.to_s.casecmp('true').zero? || value.to_s == '1'
+          end
+
+          def parse_json_payload(text)
+            return nil if text.empty?
+            return nil unless json_candidate?(text)
+
+            payload = JSON.parse(text)
+            return payload unless payload.is_a?(String)
+
+            nested = payload.to_s.strip
+            return payload if nested.empty? || nested == text
+            return payload unless nested.start_with?('{', '[', '"')
+
+            parse_json_payload(nested)
+          end
+
+          def json_candidate?(text)
+            stripped = text.to_s.lstrip
+            return false if stripped.empty?
+
+            stripped.start_with?('{', '[', '"')
+          end
+
+          def normalize_layout_payload(payload)
+            case payload
+            when Array
+              { lines: payload }
+            when Hash
+              lines = extract_layout_lines(payload)
+              return nil unless lines.is_a?(Array)
+
+              { lines: lines }
+            else
+              nil
+            end
+          end
+
+          def extract_layout_lines(payload)
+            keys = %w[lines items runs]
+            keys.each do |key|
+              value = payload[key] || payload[key.to_sym]
+              return value if value.is_a?(Array)
+            end
+
+            nested = payload['layout'] || payload[:layout] || payload['payload'] || payload[:payload] || payload['data'] || payload[:data]
+            if nested
+              normalized = normalize_layout_payload(nested)
+              return normalized[:lines] if normalized
+            end
+
+            if payload_line_hash?(payload)
+              return [payload]
+            end
+
+            nil
+          end
+
+          def payload_line_hash?(payload)
+            return false unless payload.is_a?(Hash)
+
+            !extract_line_text(payload).strip.empty? || line_break?(payload)
+          end
+
+          def recover_layout_from_jsonish_text(text)
+            return nil unless text.include?('"text"') || text.include?('\\"text\\"')
+
+            raw_values = text.scan(/"text"\s*:\s*"((?:\\.|[^"\\])*)"/).flatten
+            if raw_values.empty?
+              raw_values = text.scan(/\\"text\\"\s*:\s*\\"((?:\\\\.|[^"\\])*)\\"/).flatten
+            end
+            decoded = raw_values.filter_map { |value| decode_json_text_fragment(value) }
+            decoded = decoded.map { |value| value.gsub(/\s+/, ' ').strip }.reject(&:empty?)
+            return nil if decoded.length < 2
+
+            { lines: decoded.map { |value| { 'text' => value } } }
+          end
+
+          def decode_json_text_fragment(fragment)
+            JSON.parse(%("#{fragment}"))
+          rescue JSON::ParserError, Shoko::Error
+            fragment.to_s
+                    .gsub('\\\\', '\\')
+                    .gsub('\\"', '"')
+                    .gsub('\\n', "\n")
+                    .gsub('\\r', "\r")
+          end
+
+          def line_break?(hash)
+            truthy?(hash[:break] || hash['break'] || hash[:line_break] || hash['line_break'] ||
+                    hash[:newline] || hash['newline'] || hash[:new_line] || hash['new_line'])
+          end
+
+          def extract_line_text(hash)
+            value = hash[:text] || hash['text'] ||
+                    hash[:value] || hash['value'] ||
+                    hash[:line] || hash['line'] ||
+                    hash[:content] || hash['content']
+            return value.to_s if value
+
+            segments = hash[:segments] || hash['segments']
+            return '' unless segments.is_a?(Array)
+
+            segments.map do |segment|
+              next segment.to_s unless segment.is_a?(Hash)
+
+              segment[:text] || segment['text'] || segment[:value] || segment['value']
+            end.join
+          end
+
+          def extract_line_x(hash)
+            parse_float(hash[:x] || hash['x'] ||
+                        hash[:left] || hash['left'] ||
+                        hash[:indent] || hash['indent'] ||
+                        hash[:start_x] || hash['start_x'])
+          end
+
+          def extract_line_italic_ratio(hash)
+            parse_float(hash[:italic_ratio] || hash['italic_ratio'] ||
+                        hash[:italic_pct] || hash['italic_pct'] ||
+                        hash[:italic_share] || hash['italic_share'])
+          end
+
+          def extract_line_italic(hash, italic_ratio:)
+            return italic_ratio >= 0.65 if italic_ratio
+
+            explicit = hash[:italic]
+            explicit = hash['italic'] if explicit.nil?
+            return truthy?(explicit) unless explicit.nil?
+
+            styles = hash[:styles] || hash['styles']
+            if styles.is_a?(Hash)
+              value = styles[:italic]
+              value = styles['italic'] if value.nil?
+              return truthy?(value) unless value.nil?
+            end
+
+            style_text = hash[:style] || hash['style'] ||
+                         hash[:font_style] || hash['font_style'] ||
+                         (styles.is_a?(Hash) ? (styles[:style] || styles['style'] || styles[:font_style] || styles['font_style']) : nil)
+            style_text.to_s.downcase.include?('italic')
+          end
+
+          def left_tolerance(metrics)
+            [[metrics[:ref_span].to_f * 0.10, 8.0].max, 22.0].min
+          end
+
+          def center_cutoff(metrics)
+            [[metrics[:ref_span].to_f * 0.22, 38.0].max, 180.0].min
+          end
+
+          def right_cutoff(metrics)
+            cutoff = [[metrics[:ref_span].to_f * 0.70, 64.0].max, 360.0].min
+            [cutoff, center_cutoff(metrics) + 18.0].max
+          end
+
+          def percentile_x(values, percentile)
+            sorted = Array(values).map(&:to_f).sort
+            return nil if sorted.empty?
+
+            p = [[percentile.to_f, 0.0].max, 1.0].min
+            index = ((sorted.length - 1) * p).floor
+            sorted[index]
+          end
+
+          def normalize_line_text(text)
+            text.to_s
+                .tr("\u00A0\u2007\u202F", ' ')
+                .gsub(/[[:space:]]+/, ' ')
+                .strip
+          end
+
+          def parse_float(value)
+            return nil if value.nil?
+
+            Float(value)
+          rescue Shoko::Error
+            raise
+          end
+        end
+      end
+    end
+  end
+end