chupa-text-decomposer-webkit 1.0.2 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 9cfa2e299e210eed4fcdafa1f85934d377eb56f1
-  data.tar.gz: a4658b1c5b840ad94e6185ba1d41e4ba703e83c4
+SHA256:
+  metadata.gz: 1e38649e31b6142b1b907ecdc15fc109d66293c3408b55dca5fa5596d77a38a0
+  data.tar.gz: 4a85f2e3a9a21c280469c53bfb1f54f70ca1c80b7d35eb2093d524f6ab64a0f3
 SHA512:
-  metadata.gz: 5ff6407a451e09c91b670b1ee85361bffa7c3ed166d15ef7cd88d1cbc16af1ee0381f7c011d116346090325f6d0237703b331f9d692ca25d5de04d0ed9c283e1
-  data.tar.gz: 122344f3cfcda27ec94131a933aed1836289c8565c439d09b6e5b37962718db211dece223af7c565fefd4d94fe50627493bbf26ad2b5288ca55224b92353a0c4
+  metadata.gz: 20637409e930276464defaddf3a1d7018bc67b3c524509322c4b599dda0f217a7c0a8a2053d105520c47e9a1bd8e949870dbc381d598b12103dc1b05bdb46fdc
+  data.tar.gz: 9f03f9ed5efb9d242d4770a884afe7f85b1547e72f6ebb1703cae45bc4fc544c5b00f0126cc5558aaaf8023ede3e829b86773e2e6c0b73ae3149d633cf484f4d

data/bin/chupa-text-decomposer-webkit-screenshoter

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+#
+# Copyright (C) 2017-2021  Sutou Kouhei <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "chupa-text-decomposer-webkit/screenshoter"
+
+logger = Object.new
+def logger.debug
+  $stdout.puts("debug: #{yield}")
+  $stdout.flush
+end
+
+def logger.error
+  $stdout.puts("error: #{yield}")
+  $stdout.flush
+end
+
+path, uri, output_path, width, height = ARGV
+screenshoter = ChupaTextDecomposerWebKit::Screenshoter.new(logger)
+File.open(path, encoding: "UTF-8") do |input|
+  screenshoter.run(input.read,
+                   uri,
+                   output_path,
+                   Integer(width),
+                   Integer(height))
+end

data/chupa-text-decomposer-webkit.gemspec

@@ -22,7 +22,7 @@ end
 
 Gem::Specification.new do |spec|
   spec.name = "chupa-text-decomposer-webkit"
-  spec.version = "1.0.2"
+  spec.version = "1.0.7"
   spec.homepage = "https://github.com/ranguba/chupa-text-decomposer-webkit"
   spec.authors = ["Kouhei Sutou"]
   spec.email = ["kou@clear-code.com"]
@@ -38,6 +38,9 @@ Gem::Specification.new do |spec|
   spec.files += Dir.glob("lib/**/*.rb")
   spec.files += Dir.glob("doc/text/*")
   spec.files += Dir.glob("test/**/*")
+  Dir.chdir("bin") do
+    spec.executables = Dir.glob("*")
+  end
 
   spec.add_runtime_dependency("chupa-text", ">= 1.0.7")
   spec.add_runtime_dependency("webkit2-gtk", ">= 3.1.7")

data/doc/text/news.md

@@ -1,10 +1,42 @@
 # News
 
+## 1.0.7: 2021-03-04
+
+### Improvements
+
+  * screenshoter: Stopped to depend on locale.
+
+## 1.0.6: 2017-07-12
+
+### Fixes
+
+  * Added missing executable files.
+
+## 1.0.5: 2017-07-12
+
+### Improvements
+
+  * Added workaround for crash.
+
+## 1.0.4: 2017-07-12
+
+### Improvements
+
+  * Added more logs.
+
+  * Supported proxy.
+
+## 1.0.3: 2017-07-11
+
+### Improvements
+
+  * Supported timeout.
+
 ## 1.0.2: 2017-07-11
 
 ### Improvements
 
-  * Reduces required resources.
+  * Reduced required resources.
 
 ## 1.0.1: 2017-07-11
 

data/lib/chupa-text-decomposer-webkit/screenshoter.rb

@@ -0,0 +1,216 @@
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "webkit2-gtk"
+
+module ChupaTextDecomposerWebKit
+  class Screenshoter
+    def initialize(logger)
+      @logger = logger
+      @view_context = create_view_context
+      @view = create_view
+      @window = create_window
+      @main_loop = GLib::MainLoop.new(nil, false)
+      @timeout_second = compute_timeout_second
+      @screenshot_cancellable = nil
+      @on_snapshot = nil
+    end
+
+    def run(body, uri, output_path, width, height)
+      @on_snapshot = lambda do |snapshot_surface|
+        scaled_surface = scale_snapshot(snapshot_surface, width, height)
+        scaled_surface.write_to_png(output_path)
+      end
+
+      begin
+        timeout do
+          debug do
+            "#{log_tag}[load][HTML] #{uri}"
+          end
+          @view.load_html(body, uri)
+          @main_loop.run
+        end
+      ensure
+        @on_snapshot = nil
+      end
+    end
+
+    private
+    def create_view_context
+      context = WebKit2Gtk::WebContext.new(ephemeral: true)
+      http_proxy = ENV["http_proxy"]
+      https_proxy = ENV["https_proxy"]
+      ftp_proxy = ENV["ftp_proxy"]
+      if http_proxy or https_proxy or ftp_proxy
+        proxy_settings = WebKit2Gtk::NetworkProxySettings.new
+        if http_proxy
+          proxy_settings.add_proxy_for_scheme("http", http_proxy)
+        end
+        if https_proxy
+          proxy_settings.add_proxy_for_scheme("https", https_proxy)
+        end
+        if ftp_proxy
+          proxy_settings.add_proxy_for_scheme("ftp", ftp_proxy)
+        end
+        context.set_network_proxy_settings(:custom, proxy_settings)
+      end
+      context
+    end
+
+    def create_view
+      view = WebKit2Gtk::WebView.new(context: @view_context)
+
+      view.signal_connect("load-changed") do |_, load_event|
+        debug do
+          "#{log_tag}[load][#{load_event.nick}] #{view.uri}"
+        end
+
+        case load_event
+        when WebKit2Gtk::LoadEvent::FINISHED
+          debug do
+            "#{log_tag}[screenshot][start] #{view.uri}"
+          end
+          cancel_screenshot
+          @screenshot_cancellable = Gio::Cancellable.new
+          view.get_snapshot(:full_document,
+                            :none,
+                            @screenshot_cancellable) do |_, result|
+            @screenshot_cancellable = nil
+            @main_loop.quit
+            begin
+              snapshot_surface = view.get_snapshot_finish(result)
+            rescue
+              error do
+                message = "failed to create snapshot: #{view.uri}: "
+                message << "#{$!.class}: #{$!.message}"
+                "#{log_tag}[screenshot][failed] #{message}"
+              end
+            else
+              debug do
+                size = "#{snapshot_surface.width}x#{snapshot_surface.height}"
+                "#{log_tag}[screenshot][finish] #{view.uri}: #{size}"
+              end
+              unless snapshot_surface.width.zero?
+                @on_snapshot.call(snapshot_surface) if @on_snapshot
+              end
+            end
+          end
+        end
+      end
+
+      view.signal_connect("load-failed") do |_, _, failed_uri, error|
+        cancel_screenshot
+        @main_loop.quit
+        error do
+          message = "failed to load URI: #{failed_uri}: "
+          message << "#{error.class}(#{error.code}): #{error.message}"
+          "#{log_tag}[load][failed] #{message}"
+        end
+        true
+      end
+
+      view
+    end
+
+    def scale_snapshot(snapshot_surface, width, height)
+      scaled_surface = Cairo::ImageSurface.new(:argb32, width, height)
+
+      context = Cairo::Context.new(scaled_surface)
+      context.set_source_color(:white)
+      context.paint
+
+      ratio = width.to_f / snapshot_surface.width
+      context.scale(ratio, ratio)
+      context.set_source(snapshot_surface)
+      context.paint
+
+      scaled_surface
+    end
+
+    def create_window
+      window = Gtk::OffscreenWindow.new
+      window.set_default_size(800, 600)
+      window.add(@view)
+      window.show_all
+      window
+    end
+
+    def cancel_screenshot
+      return if @screenshot_cancellable.nil?
+
+      debug do
+        "#{log_tag}[snapshot][cancel] cancel screenshot: #{@view.uri}"
+      end
+      @screenshot_cancellable.cancel
+      @screenshot_cancellable = nil
+    end
+
+    def timeout
+      timeout_id = GLib::Timeout.add_seconds(@timeout_second) do
+        timeout_id = nil
+        error do
+          message = "timeout to load URI: #{@timeout_second}s: #{@view.uri}"
+          message << ": loading" if @view.loading?
+          "#{log_tag}[load][timeout] #{message}"
+        end
+        cancel_screenshot
+        if @view.loading?
+          close_id = @view.signal_connect("close") do
+            @view.signal_handler_disconnect(close_id)
+            @main_loop.quit
+            error do
+              "#{log_tag}[load][closed] #{@view.uri}"
+            end
+          end
+          @view.try_close
+        else
+          @main_loop.quit
+        end
+        GLib::Source::REMOVE
+      end
+
+      begin
+        yield
+      ensure
+        GLib::Source.remove(timeout_id) if timeout_id
+      end
+    end
+
+    def compute_timeout_second
+      default_timeout = 5
+      timeout_string =
+        ENV["CHUPA_TEXT_DECOMPOSER_WEBKIT_TIMEOUT"] || default_timeout.to_s
+      begin
+        Integer(timeout_string)
+      rescue ArgumentError
+        default_timeout
+      end
+    end
+
+    private
+    def log_tag
+      "[decomposer][webkit]"
+    end
+
+    def debug(*args, &block)
+      @logger.debug(*args, &block)
+    end
+
+    def error(*args, &block)
+      @logger.error(*args, &block)
+    end
+  end
+end

data/lib/chupa-text/decomposers/webkit.rb

@@ -14,12 +14,21 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
-require "webkit2-gtk"
+require "English"
+require "rbconfig"
 
 module ChupaText
   module Decomposers
     class WebKit < Decomposer
+      module LogTag
+        private
+        def log_tag
+          "[decomposer][webkit]"
+        end
+      end
+
       include Loggable
+      include LogTag
 
       registry.register("webkit", self)
 
@@ -49,88 +58,92 @@ module ChupaText
         false
       end
 
+      IN_PROCESS = ENV["CHUPA_TEXT_DECOMPOSER_WEBKIT_IN_PROCESS"] == "yes"
+      if IN_PROCESS
+        require "chupa-text-decomposer-webkit/screenshoter"
+      end
+
       def decompose(data)
-        data.screenshot = create_screenshot(data.source)
+        body = data.source.body
+        uri = data.source.uri.to_s
+        output = Tempfile.new(["chupa-text-decomposer-webkit", ".png"])
+        width, height = data.expected_screenshot_size
+        if IN_PROCESS
+          screenshoter = ChupaTextDecomposerWebKit::Screenshoter.new(logger)
+          screenshoter.run(body, uri, output.path, width, height)
+        else
+          screenshoter = ExternalScreenshoter.new
+          screenshoter.run(data.source.path, uri, output.path, width, height)
+        end
+        unless File.size(output.path).zero?
+          png = output.read
+          data.screenshot = Screenshot.new("image/png",
+                                           [png].pack("m*"),
+                                           "base64")
+        end
         data[AVAILABLE_ATTRIBUTE_NAME] = !data.screenshot.nil?
         yield(data)
       end
 
-      private
-      def create_screenshot(data)
-        screenshot = nil
-
-        @@view_context ||= WebKit2Gtk::WebContext.new(ephemeral: true)
-        view = WebKit2Gtk::WebView.new(context: @@view_context)
-        window = Gtk::OffscreenWindow.new
-        window.set_default_size(800, 600)
-        window.add(view)
-        window.show_all
-
-        finished = false
-        view.signal_connect("load-changed") do |_, load_event|
-          debug do
-            "#{log_tag}[load][#{load_event.nick}] #{view.uri}"
-          end
+      class ExternalScreenshoter
+        include Loggable
+        include LogTag
+
+        def initialize
+          @screenshoter = File.join(__dir__,
+                                    "..",
+                                    "..",
+                                    "..",
+                                    "bin",
+                                    "chupa-text-decomposer-webkit-screenshoter")
+          @command = ExternalCommand.new(RbConfig.ruby)
+        end
 
-          case load_event
-          when WebKit2Gtk::LoadEvent::FINISHED
-            view.get_snapshot(:full_document, :none) do |_, result|
-              finished = true
-              snapshot_surface = view.get_snapshot_finish(result)
-              unless snapshot_surface.width.zero?
-                png = convert_snapshot_surface_to_png(data, snapshot_surface)
-                screenshot = Screenshot.new("image/png",
-                                            [png].pack("m*"),
-                                            "base64")
+        def run(html_path, uri, output_path, width, height)
+          output_read, output_write = IO.pipe
+          error_output = Tempfile.new("chupa-text-decomposer-webkit-error")
+          output_reader = Thread.new do
+            loop do
+              IO.select([output_read])
+              line = output_read.gets
+              break if line.nil?
+
+              case line.chomp
+              when /\Adebug: /
+                debug($POSTMATCH)
+              when /\Aerror: /
+                error($POSTMATCH)
               end
             end
           end
-        end
-        view.signal_connect("load-failed") do |_, _, failed_uri, error|
-          finished = true
-          error do
-            message = "failed to load URI: #{failed_uri}: "
-            message << "#{error.class}(#{error.code}): #{error.message}"
-            "#{log_tag}[load][failed] #{message}"
+          successed = @command.run(@screenshoter,
+                                   html_path,
+                                   uri,
+                                   output_path,
+                                   width.to_s,
+                                   height.to_s,
+                                   {
+                                     :spawn_options => {
+                                       :out => output_write,
+                                       :err => error_output.path,
+                                     },
+                                   })
+          output_write.close
+          output_reader.join
+
+          unless successed
+            error do
+              message = "failed to external screenshoter: #{uri}: "
+              message << "#{@command.path} #{@screenshoter}"
+              "#{log_tag}[external-screenshoter][run][failed] #{message}"
+            end
+          end
+          unless error_output.size.zero?
+            error_output.each_line do |line|
+              error(line)
+            end
           end
-          true
-        end
-        debug do
-          "#{log_tag}[load][html] #{data.uri}"
-        end
-        view.load_html(data.body, data.uri.to_s)
-
-        main_context = GLib::MainContext.default
-        until finished
-          main_context.iteration(true)
         end
-        window.destroy
-
-        screenshot
-      end
-
-      def convert_snapshot_surface_to_png(data, snapshot_surface)
-        screenshot_width, screenshot_height = data.expected_screenshot_size
-
-        screenshot_surface = Cairo::ImageSurface.new(:argb32,
-                                                     screenshot_width,
-                                                     screenshot_height)
-        context = Cairo::Context.new(screenshot_surface)
-        context.set_source_color(:white)
-        context.paint
-
-        ratio = screenshot_width.to_f / snapshot_surface.width
-        context.scale(ratio, ratio)
-        context.set_source(snapshot_surface)
-        context.paint
-
-        png = StringIO.new
-        screenshot_surface.write_to_png(png)
-        png.string
-      end
-
-      def log_tag
-        "[decomposer][webkit]"
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chupa-text-decomposer-webkit
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.7
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-11 00:00:00.000000000 Z
+date: 2021-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chupa-text
@@ -115,7 +115,8 @@ description: |
   You can use `webkit` decomposer.
 email:
 - kou@clear-code.com
-executables: []
+executables:
+- chupa-text-decomposer-webkit-screenshoter
 extensions: []
 extra_rdoc_files: []
 files:
@@ -124,8 +125,10 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/chupa-text-decomposer-webkit-screenshoter
 - chupa-text-decomposer-webkit.gemspec
 - doc/text/news.md
+- lib/chupa-text-decomposer-webkit/screenshoter.rb
 - lib/chupa-text/decomposers/webkit.rb
 - test/run-test.rb
 - test/test-webkit.rb
@@ -148,8 +151,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.2
+rubygems_version: 3.3.0.dev
 signing_key: 
 specification_version: 4
 summary: This is a ChupaText decomposer plugin for to extract text and meta-data from