invoice_extractor_arca 0.2.0

data/lib/invoice_extractor_arca/ocr/rapidocr_extract.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+
+import contextlib
+import json
+import sys
+from importlib.metadata import version
+
+ORIGINAL_STDOUT = sys.stdout
+
+
+def emit(payload):
+    print(
+        json.dumps(payload, ensure_ascii=False),
+        file=ORIGINAL_STDOUT,
+    )
+
+
+def failure(code, message):
+    return {
+        "success": False,
+        "engine": "rapidocr",
+        "version": version("rapidocr"),
+        "lines": [],
+        "raw_text": "",
+        "warnings": [
+            {
+                "code": code,
+                "message": str(message),
+            }
+        ],
+    }
+
+
+def box_metrics(box):
+    xs = [point[0] for point in box]
+    ys = [point[1] for point in box]
+
+    return {
+        "left": min(xs),
+        "top": min(ys),
+        "right": max(xs),
+        "bottom": max(ys),
+        "center_y": (min(ys) + max(ys)) / 2,
+        "height": max(ys) - min(ys),
+    }
+
+
+def build_visual_lines(items):
+    positioned = []
+
+    for item in items:
+        if not item["box"]:
+            continue
+
+        metrics = box_metrics(item["box"])
+        positioned.append({
+            **item,
+            **metrics,
+        })
+
+    positioned.sort(
+        key=lambda item: (item["center_y"], item["left"])
+    )
+
+    groups = []
+
+    for item in positioned:
+        matching_group = None
+
+        for group in reversed(groups):
+            tolerance = max(
+                10.0,
+                group["average_height"] * 0.55,
+                item["height"] * 0.55,
+            )
+
+            if abs(item["center_y"] - group["center_y"]) <= tolerance:
+                matching_group = group
+                break
+
+            if item["center_y"] - group["center_y"] > tolerance * 2:
+                break
+
+        if matching_group is None:
+            groups.append({
+                "center_y": item["center_y"],
+                "average_height": item["height"],
+                "items": [item],
+            })
+            continue
+
+        matching_group["items"].append(item)
+
+        count = len(matching_group["items"])
+        matching_group["center_y"] = (
+            sum(child["center_y"] for child in matching_group["items"])
+            / count
+        )
+        matching_group["average_height"] = (
+            sum(child["height"] for child in matching_group["items"])
+            / count
+        )
+
+    groups.sort(key=lambda group: group["center_y"])
+
+    visual_lines = []
+
+    for group in groups:
+        children = sorted(
+            group["items"],
+            key=lambda item: item["left"],
+        )
+
+        visual_lines.append({
+            "text": "  ".join(
+                child["text"].strip()
+                for child in children
+                if child["text"].strip()
+            ),
+            "items": [
+                {
+                    "text": child["text"],
+                    "score": child["score"],
+                    "box": child["box"],
+                }
+                for child in children
+            ],
+        })
+
+    return visual_lines
+
+
+def main():
+    if len(sys.argv) != 2:
+        emit(failure(
+            "rapidocr_invalid_arguments",
+            "Expected exactly one image path",
+        ))
+        return 2
+
+    image_path = sys.argv[1]
+
+    try:
+        with contextlib.redirect_stdout(sys.stderr):
+            from rapidocr import RapidOCR
+
+            engine = RapidOCR()
+            result = engine(image_path)
+
+    except Exception as exc:
+        emit(failure("rapidocr_failed", exc))
+        return 1
+
+    texts = list(result.txts or [])
+    scores = list(result.scores or [])
+    boxes = (
+        result.boxes.tolist()
+        if result.boxes is not None
+        else []
+    )
+
+    items = []
+
+    for index, text in enumerate(texts):
+        items.append({
+            "text": text,
+            "score": (
+                float(scores[index])
+                if index < len(scores)
+                else None
+            ),
+            "box": (
+                boxes[index]
+                if index < len(boxes)
+                else None
+            ),
+        })
+
+    visual_lines = build_visual_lines(items)
+
+    emit({
+        "success": True,
+        "engine": "rapidocr",
+        "version": version("rapidocr"),
+        "lines": visual_lines,
+        "raw_text": "\n".join(
+            line["text"]
+            for line in visual_lines
+            if line["text"]
+        ),
+        "warnings": [],
+    })
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

data/lib/invoice_extractor_arca/ocr/rapidocr_text_extractor.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../text/normalizer"
+
+module InvoiceExtractorArca
+  module Ocr
+    class RapidocrTextExtractor
+      DEFAULT_HELPER_PATH =
+        File.expand_path("rapidocr_extract.py", __dir__)
+
+      def initialize(
+        resolver:,
+        runner:,
+        python_command: :python3,
+        helper_path: DEFAULT_HELPER_PATH,
+        normalizer: Text::Normalizer.new
+      )
+        @resolver = resolver
+        @runner = runner
+        @python_command = python_command
+        @helper_path = helper_path
+        @normalizer = normalizer
+      end
+
+      def extract(image_path)
+        python = @resolver.resolve!(@python_command)
+
+        process = @runner.run(
+          python,
+          @helper_path,
+          image_path.to_s
+        )
+
+        payload = parse_json(process[:stdout])
+
+        unless process[:success] && payload["success"]
+          return failure_result(payload, process[:stderr])
+        end
+
+        raw = payload.fetch("raw_text", "")
+        normalized = @normalizer.normalize(raw)
+
+        {
+          extracted: !normalized.empty?,
+          source: rapidocr_source(payload),
+          raw: raw,
+          normalized: normalized,
+          pages: nil,
+          lines: Array(payload["lines"]),
+          warnings: normalized.empty? ? [empty_warning] : []
+        }
+      rescue Errors::MissingDependency => e
+        failure_result(
+          {
+            "warnings" => [{
+              "code" => "rapidocr_missing_dependency",
+              "message" => e.message
+            }]
+          },
+          ""
+        )
+      rescue JSON::ParserError => e
+        failure_result(
+          {
+            "warnings" => [{
+              "code" => "rapidocr_invalid_output",
+              "message" => e.message
+            }]
+          },
+          ""
+        )
+      end
+
+      private
+
+      def parse_json(stdout)
+        JSON.parse(stdout.to_s.strip)
+      end
+
+      def empty_warning
+        {
+          code: "image_ocr_empty",
+          message: "RapidOCR returned no recognized text",
+          component: "text"
+        }
+      end
+
+      def failure_result(payload, stderr)
+        warnings = Array(payload["warnings"]).map do |warning|
+          {
+            code: warning.fetch("code", "rapidocr_failed"),
+            message: warning.fetch("message", stderr.to_s.strip),
+            component: "text"
+          }
+        end
+
+        {
+          extracted: false,
+          source: "rapidocr-3.9.2",
+          raw: "",
+          normalized: "",
+          pages: nil,
+          lines: [],
+          warnings: warnings
+        }
+      end
+
+      def rapidocr_source(payload)
+        engine = payload.fetch("engine", "rapidocr")
+        version = payload["version"]
+
+        version ? "#{engine}-#{version}" : engine
+      end
+    end
+  end
+end

data/lib/invoice_extractor_arca/pdf/ocr_text_extractor.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module InvoiceExtractorArca
+  module Pdf
+    class OcrTextExtractor
+      def initialize(renderer:, image_text_extractor:, resolution: 300)
+        @renderer = renderer
+        @image_text_extractor = image_text_extractor
+        @resolution = resolution
+      end
+
+      def extract(pdf_path, page_range: nil)
+        @renderer.render(
+          pdf_path,
+          page_range: page_range,
+          resolution: @resolution
+        ) do |image_paths|
+          build_result(
+            Array(image_paths),
+            page_numbers(page_range, image_paths.length)
+          )
+        end
+      rescue Errors::UnsupportedFileType => e
+        failure_result(e, page_range)
+      end
+
+      private
+
+      def build_result(image_paths, pages)
+        results = image_paths.map do |image_path|
+          @image_text_extractor.extract(image_path)
+        end
+
+        raw = results
+          .map { |result| result[:raw].to_s }
+          .reject(&:empty?)
+          .join("\f")
+
+        normalized = results
+          .map { |result| result[:normalized].to_s }
+          .reject(&:empty?)
+          .join("\n\n")
+
+        warnings = results.flat_map do |result|
+          Array(result[:warnings])
+        end
+
+        warnings.unshift(
+          {
+            code: "pdf_ocr_fallback",
+            message: "pdftotext returned no text; PDF pages were processed with OCR",
+            component: "text"
+          }
+        )
+
+        unless normalized.empty?
+          return {
+            extracted: true,
+            source: ocr_source(results),
+            raw: raw,
+            normalized: normalized,
+            pages: pages,
+            lines: results.flat_map { |result| Array(result[:lines]) },
+            warnings: warnings
+          }
+        end
+
+        warnings << {
+          code: "pdf_ocr_empty",
+          message: "PDF OCR returned no recognized text",
+          component: "text"
+        }
+
+        {
+          extracted: false,
+          source: ocr_source(results),
+          raw: raw,
+          normalized: normalized,
+          pages: pages,
+          lines: [],
+          warnings: warnings
+        }
+      end
+
+      def ocr_source(results)
+        sources = results
+          .map { |result| result[:source] }
+          .compact
+          .uniq
+
+        sources.one? ? sources.first : "rapidocr"
+      end
+
+      def page_numbers(page_range, rendered_count)
+        pages =
+          case page_range
+          when nil
+            [1]
+          when Integer
+            [page_range]
+          when Range
+            page_range.to_a
+          when Array
+            page_range.dup
+          else
+            []
+          end
+
+        pages.first(rendered_count)
+      end
+
+      def failure_result(error, page_range)
+        {
+          extracted: false,
+          source: "rapidocr",
+          raw: "",
+          normalized: "",
+          pages: page_numbers(page_range, 0),
+          lines: [],
+          warnings: [
+            {
+              code: "pdf_ocr_failed",
+              message: error.message,
+              component: "text"
+            }
+          ]
+        }
+      end
+    end
+  end
+end

data/lib/invoice_extractor_arca/pdf/renderer.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+require_relative "../errors"
+
+module InvoiceExtractorArca
+  module Pdf
+    class Renderer
+      DEFAULT_PAGE = 1
+      DEFAULT_RESOLUTION = 300
+
+      def initialize(resolver:, runner:)
+        @resolver = resolver
+        @runner = runner
+      end
+
+      def render(
+        pdf_path,
+        page_range: nil,
+        resolution: DEFAULT_RESOLUTION
+      )
+        temp_dir = Dir.mktmpdir("invoice_extractor_arca-")
+
+        image_paths = pages_from(page_range).map do |page|
+          render_page(
+            pdf_path: pdf_path,
+            temp_dir: temp_dir,
+            page: page,
+            resolution: resolution
+          )
+        end
+
+        yield image_paths
+      ensure
+        FileUtils.rm_rf(temp_dir) if temp_dir
+      end
+
+      private
+
+      def pages_from(page_range)
+        pages =
+          case page_range
+          when nil
+            [DEFAULT_PAGE]
+          when Integer
+            [page_range]
+          when Range
+            page_range.to_a
+          when Array
+            page_range
+          else
+            raise Errors::UnsupportedFileType,
+              "Invalid page_range: #{page_range.inspect}"
+          end
+
+        pages.each { |page| validate_page!(page) }
+        pages
+      end
+
+      def validate_page!(page)
+        return if page.is_a?(Integer) && page.positive?
+
+        raise Errors::UnsupportedFileType,
+          "Invalid PDF page number: #{page.inspect}"
+      end
+
+      def render_page(pdf_path:, temp_dir:, page:, resolution:)
+        pdftoppm = @resolver.resolve!(:pdftoppm)
+        output_prefix = File.join(temp_dir, "page-#{page}")
+
+        result = @runner.run(
+          pdftoppm,
+          "-png",
+          "-r",
+          Integer(resolution).to_s,
+          "-f",
+          page.to_s,
+          "-l",
+          page.to_s,
+          "-singlefile",
+          pdf_path,
+          output_prefix
+        )
+
+        unless result[:success]
+          raise Errors::UnsupportedFileType, result[:stderr]
+        end
+
+        "#{output_prefix}.png"
+      rescue ArgumentError
+        raise Errors::UnsupportedFileType,
+          "Invalid PDF render resolution: #{resolution.inspect}"
+      end
+    end
+  end
+end

data/lib/invoice_extractor_arca/pdf/text_extractor.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+require_relative "../errors"
+require_relative "../text/normalizer"
+
+module InvoiceExtractorArca
+  module Pdf
+    class TextExtractor
+      def initialize(
+        resolver:,
+        runner:,
+        normalizer: Text::Normalizer.new
+      )
+        @resolver = resolver
+        @runner = runner
+        @normalizer = normalizer
+      end
+
+      def extract(pdf_path, page_range: nil)
+        raw = extract_raw(pdf_path, page_range)
+        normalized = @normalizer.normalize(raw)
+        warnings = []
+
+        if normalized.empty?
+          warnings << {
+            code: "pdf_text_empty",
+            message: "PDF contains no extractable text",
+            component: "text"
+          }
+        end
+
+        {
+          extracted: !normalized.empty?,
+          source: "pdftotext",
+          raw: raw,
+          normalized: normalized,
+          pages: page_numbers(page_range),
+          warnings: warnings
+        }
+      rescue Errors::MissingDependency,
+        Errors::PdfTextExtractionFailure => e
+        failure_result(e, page_range)
+      end
+
+      private
+
+      def extract_raw(pdf_path, page_range)
+        pdftotext = @resolver.resolve!(:pdftotext)
+
+        extraction_ranges(page_range)
+          .map do |range|
+            extract_range(
+              pdftotext,
+              pdf_path,
+              first_page: range[:first],
+              last_page: range[:last]
+            )
+          end
+          .join("\f")
+      end
+
+      def extract_range(
+        pdftotext,
+        pdf_path,
+        first_page:,
+        last_page:
+      )
+        command = [
+          pdftotext,
+          "-layout",
+          "-enc",
+          "UTF-8"
+        ]
+
+        if first_page
+          command.concat([
+            "-f",
+            first_page.to_s,
+            "-l",
+            last_page.to_s
+          ])
+        end
+
+        command.concat([
+          pdf_path.to_s,
+          "-"
+        ])
+
+        result = @runner.run(*command)
+
+        unless result[:success]
+          message = result[:stderr].to_s.strip
+          message = "pdftotext command failed" if message.empty?
+
+          raise Errors::PdfTextExtractionFailure, message
+        end
+
+        result[:stdout].to_s
+      end
+
+      def extraction_ranges(page_range)
+        case page_range
+        when nil
+          [
+            {
+              first: nil,
+              last: nil
+            }
+          ]
+        when Integer
+          validate_page!(page_range)
+
+          [
+            {
+              first: page_range,
+              last: page_range
+            }
+          ]
+        when Range
+          pages = page_range.to_a
+          pages.each { |page| validate_page!(page) }
+
+          [
+            {
+              first: pages.first,
+              last: pages.last
+            }
+          ]
+        when Array
+          page_range.map do |page|
+            validate_page!(page)
+
+            {
+              first: page,
+              last: page
+            }
+          end
+        else
+          raise Errors::PdfTextExtractionFailure,
+            "Invalid page_range: #{page_range.inspect}"
+        end
+      end
+
+      def page_numbers(page_range)
+        case page_range
+        when nil
+          nil
+        when Integer
+          [page_range]
+        when Range
+          page_range.to_a
+        when Array
+          page_range.dup
+        end
+      end
+
+      def validate_page!(page)
+        return if page.is_a?(Integer) && page.positive?
+
+        raise Errors::PdfTextExtractionFailure,
+          "Invalid PDF page number: #{page.inspect}"
+      end
+
+      def failure_result(error, page_range)
+        {
+          extracted: false,
+          source: "pdftotext",
+          raw: "",
+          normalized: "",
+          pages: page_numbers(page_range),
+          warnings: [
+            {
+              code: "pdf_text_extraction_failed",
+              message: error.message,
+              component: "text"
+            }
+          ]
+        }
+      end
+    end
+  end
+end