chupa-text 1.0.6 → 1.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3af110814417d579cde496c50b05d6d49964dd97
-  data.tar.gz: 2609179c9e817e33a82df59c53834236dc63da3d
+  metadata.gz: 9a50556c287fd5148a3032b4fd8518679237e7be
+  data.tar.gz: 0d58a4948b3df081449e67211e23168b539be83b
 SHA512:
-  metadata.gz: 6ae9ed51763e954b6b99973dfb2c020e046ccfb57f6670bfe96e935319055a6f898e71f0ed30ba25f3d3c8a4180ea6dbdfcb80337829659fdfaeb6a08841d873
-  data.tar.gz: a114dd75066faaf9501cd1fd4891df6a8ed46adf99711721a4d4d4cfaf4b1a63bb852aba578d7c46f5c63535bd8d5a19508b65c44fa953087ff8ac301cc85535
+  metadata.gz: 298a7416122a757fbb6018c73327f768adaa6b18402f7725e5c9dde12b320d00a7015c04a5eb90580b4c6f822678acded64e132a8ba9d8d1b1de792804d13103
+  data.tar.gz: 3aaaa8b120e085434094163d3e582cb4bb33c5478a2916a268cd5b03cfff10b48cd05f6912acb26185e288650cc656a3f38260a48468fd988de74f0dff59bc1e

data/doc/text/news.md

@@ -1,5 +1,21 @@
 # News
 
+## 1.0.7: 2017-07-06
+
+### Improvements
+
+  * Supported screenshot.
+
+  * `chupa-text`: Added new options:
+
+    * `--need-screenshot`
+
+    * `--expected-screenshot-size=WIDTHxHEIGHT`
+
+### Fixes
+
+  * CSV decomposer: Fixed a infinite loop bug.
+
 ## 1.0.6: 2017-07-05
 
 ### Improvements

data/lib/chupa-text.rb

@@ -41,6 +41,8 @@ require "chupa-text/formatters"
 require "chupa-text/file-content"
 require "chupa-text/virtual-content"
 
+require "chupa-text/screenshot"
+
 require "chupa-text/attributes"
 require "chupa-text/data"
 require "chupa-text/input-data"

data/lib/chupa-text/command/chupa-text.rb

@@ -28,11 +28,24 @@ module ChupaText
 
       AVAILABLE_FORMATS = [:json, :text]
 
+      SIZE = /\A\d+x\d+\z/o
+      OptionParser.accept(SIZE, SIZE) do |value|
+        if value
+           begin
+             value.split("x").collect {|number| Integer(number)}
+           rescue ArgumentError
+             raise OptionParser::InvalidArgument, value
+           end
+        end
+      end
+
       def initialize
         @input = nil
-        @configuration = Configuration.default
+        @configuration = Configuration.load_default
         @enable_gems = true
         @format = :json
+        @need_screenshot = true
+        @expected_screenshot_size = [200, 200]
       end
 
       def run(*arguments)
@@ -92,12 +105,25 @@ module ChupaText
                   "Appends PATH to decomposer load path.") do |path|
           $LOAD_PATH << path
         end
+
+        parser.separator("")
+        parser.separator("Output related options")
         parser.on("--format=FORMAT", AVAILABLE_FORMATS,
                   "Output FORMAT.",
                   "[#{AVAILABLE_FORMATS.join(', ')}]",
-                  "(default: json)") do |format|
+                  "(default: #{@format})") do |format|
           @format = format
         end
+        parser.on("--[no-]need-screenshot",
+                  "Generate screenshot if available.",
+                  "(default: #{@need_screenshot})") do |boolean|
+          @need_screenshot = boolean
+        end
+        parser.on("--expected-screenshot-size=WIDTHxHEIGHT", SIZE,
+                  "Expected screenshot size.",
+                  "(default: #{@expected_screenshot_size.join("x")})") do |size|
+          @expected_screenshot_size = size
+        end
 
         parser.separator("")
         parser.separator("Log related options:")
@@ -152,7 +178,7 @@ module ChupaText
 
       def create_data
         if @input.nil?
-          VirtualFileData.new(nil, $stdin)
+          data = VirtualFileData.new(nil, $stdin)
         else
           case @input
           when /\A[a-z]+:\/\//i
@@ -160,8 +186,11 @@ module ChupaText
           else
             input = Pathname(@input)
           end
-          InputData.new(input)
+          data = InputData.new(input)
         end
+        data.need_screenshot = @need_screenshot
+        data.expected_screenshot_size = @expected_screenshot_size
+        data
       end
 
       def create_formatter

data/lib/chupa-text/configuration.rb

@@ -18,11 +18,10 @@ module ChupaText
   class Configuration
     class << self
       def default
-        @default ||= create_default
+        @default ||= load_default
       end
 
-      private
-      def create_default
+      def load_default
         configuration = new
         loader = ConfigurationLoader.new(configuration)
         loader.load("chupa-text.conf")

data/lib/chupa-text/data.rb

@@ -52,6 +52,17 @@ module ChupaText
     #   archive data in {#source}.
     attr_accessor :source
 
+    # @return [Screenshot, nil] The screenshot of the data. For example,
+    #   the first page image for PDF file.text.
+    attr_accessor :screenshot
+
+    # @param [Bool] value `true` when screenshot is needed.
+    # @return [Bool] the specified value
+    attr_writer :need_screenshot
+
+    # @return [Array<Integer, Integer>] the expected screenshot size.
+    attr_accessor :expected_screenshot_size
+
     def initialize(options={})
       @uri = nil
       @body = nil
@@ -60,9 +71,15 @@ module ChupaText
       @mime_type = nil
       @attributes = Attributes.new
       @source = nil
+      @screenshot = nil
+      @need_screenshot = true
+      @expected_screenshot_size = [200, 200]
       @options = options || {}
       source_data = @options[:source_data]
-      merge!(source_data) if source_data
+      if source_data
+        merge!(source_data)
+        @source = source_data
+      end
     end
 
     def initialize_copy(object)
@@ -86,6 +103,8 @@ module ChupaText
         self["source-mime-types"] ||= []
         self["source-mime-types"].unshift(data.mime_type)
       end
+      self.need_screenshot = data.need_screenshot?
+      self.expected_screenshot_size = data.expected_screenshot_size
     end
 
     # @param [String, URI, nil] uri The URI for the data. If `uri` is
@@ -162,6 +181,11 @@ module ChupaText
       mime_type == "text/plain"
     end
 
+    # @return [Bool] `true` when screenshot is needed if available.
+    def need_screenshot?
+      @need_screenshot
+    end
+
     private
     def guess_mime_type
       guess_mime_type_from_uri or

data/lib/chupa-text/decomposers/csv.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,6 +14,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "cgi/util"
 require "csv"
 
 module ChupaText
@@ -22,8 +23,14 @@ module ChupaText
       registry.register("csv", self)
 
       def target?(data)
-        data.extension == "csv" or
-          data.mime_type == "text/csv"
+        return true if data.mime_type == "text/csv"
+
+        if data.text_plain? and
+            (data["source-mime-types"] || []).include?("text/csv")
+          return false
+        end
+
+        data.extension == "csv"
       end
 
       def decompose(data)
@@ -35,10 +42,42 @@ module ChupaText
             text << "\n"
           end
         end
-        text_data = TextData.new(text)
-        text_data.uri = data.uri
+
+        text_data = TextData.new(text, :source_data => data)
+        if data.need_screenshot?
+          text_data.screenshot = create_screenshot(data, text)
+        end
+
         yield(text_data)
       end
+
+      private
+      def create_screenshot(data, text)
+        width, height = data.expected_screenshot_size
+        max_n_lines = 10
+        font_size = height / max_n_lines
+        target_text = ""
+        text.each_line.with_index do |line, i|
+          break if i == max_n_lines
+          target_text << line
+        end
+        mime_type = "image/svg+xml"
+        data = <<-SVG
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+  xmlns="http://www.w3.org/2000/svg"
+  width="#{width}"
+  height="#{height}"
+  viewBox="0 0 #{width} #{height}">
+  <text
+    x="0"
+    y="#{font_size}"
+    style="font-size: #{font_size}px; white-space: pre-wrap;"
+    xml:space="preserve">#{CGI.escapeHTML(target_text)}</text>
+</svg>
+        SVG
+        Screenshot.new(mime_type, data)
+      end
     end
   end
 end

data/lib/chupa-text/decomposers/gzip.rb

@@ -42,8 +42,7 @@ module ChupaText
         when "tgz"
           uri = data.uri.to_s.gsub(/\.tgz\z/i, ".tar")
         end
-        extracted = VirtualFileData.new(uri, reader)
-        extracted.source = data
+        extracted = VirtualFileData.new(uri, reader, :source_data => data)
         yield(extracted)
       end
     end

data/lib/chupa-text/decomposers/tar.rb

@@ -32,8 +32,8 @@ module ChupaText
           reader.each do |entry|
             next unless entry.file?
             entry.extend(CopyStreamable)
-            extracted = VirtualFileData.new(entry.full_name, entry)
-            extracted.source = data
+            extracted = VirtualFileData.new(entry.full_name, entry,
+                                            :source_data => data)
             yield(extracted)
           end
         end

data/lib/chupa-text/decomposers/xml.rb

@@ -34,8 +34,7 @@ module ChupaText
           parser = REXML::Parsers::StreamParser.new(input, listener)
           parser.parse
         end
-        text_data = TextData.new(text)
-        text_data.uri = data.uri
+        text_data = TextData.new(text, :source_data => data)
         yield(text_data)
       end
 

data/lib/chupa-text/formatters/hash.rb

@@ -28,6 +28,16 @@ module ChupaText
         text = {}
         format_headers(data, text)
         text["body"] = data.body
+        screenshot = data.screenshot
+        if screenshot
+          text["screenshot"] = {
+            "mime-type" => screenshot.mime_type,
+            "data" => screenshot.data,
+          }
+          if screenshot.encoding
+            text["screenshot"]["encoding"] = screenshot.encoding
+          end
+        end
         @texts << text
       end
 

data/lib/chupa-text/screenshot.rb

@@ -0,0 +1,46 @@
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module ChupaText
+  class Screenshot
+    # @return [String] The MIME type of the screenshot.
+    attr_reader :mime_type
+
+    # @return [String] The data of the screenshot.
+    attr_reader :data
+
+    # @return [String, nil] The encoding of the screenshot data.
+    #   `nil` means that the data is raw data. It's used for SVG data
+    #   because it's text data. `"base64"` means that the data is encoded
+    #   by Base64. It's used for PNG data because it's binary data.
+    attr_reader :encoding
+
+    def initialize(mime_type, data, encoding=nil)
+      @mime_type = mime_type
+      @data = data
+      @encoding = encoding
+    end
+
+    def decoded_data
+      case @encoding
+      when "base64"
+        @data.unpack("m*")[0]
+      else
+        @data
+      end
+    end
+  end
+end

data/lib/chupa-text/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module ChupaText
-  VERSION = "1.0.6"
+  VERSION = "1.0.7"
 end

data/test/command/test-chupa-text.rb

@@ -182,4 +182,52 @@ class TestCommandChupaText < Test::Unit::TestCase
                                path.to_s))
     end
   end
+
+  sub_test_case("extract") do
+    def test_csv
+      fixture_name = "numbers.csv"
+      uri = fixture_uri(fixture_name)
+      path = fixture_path(fixture_name)
+      assert_equal([
+                     true,
+                     {
+                       "uri"       => uri.to_s,
+                       "path"      => path.to_s,
+                       "mime-type" => "text/csv",
+                       "size"      => path.stat.size,
+                       "texts"     => [
+                         {
+                           "uri"       => uri.to_s,
+                           "path"      => path.to_s,
+                           "mime-type" => "text/plain",
+                           "source-mime-types" => ["text/csv"],
+                           "body"      => "1 2 3\n4 5 6\n7 8 9\n",
+                           "size"      => 18,
+                           "screenshot" => {
+                             "mime-type" => "image/svg+xml",
+                             "data" => <<-SVG
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+  xmlns="http://www.w3.org/2000/svg"
+  width="200"
+  height="200"
+  viewBox="0 0 200 200">
+  <text
+    x="0"
+    y="20"
+    style="font-size: 20px; white-space: pre-wrap;"
+    xml:space="preserve">1 2 3
+4 5 6
+7 8 9
+</text>
+</svg>
+                             SVG
+                           },
+                         },
+                       ],
+                     },
+                   ],
+                   run_command(path.to_s))
+    end
+  end
 end

data/test/fixture/command/chupa-text/numbers.csv

@@ -0,0 +1,3 @@
+1,2,3
+4,5,6
+7,8,9

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chupa-text
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.7
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-05 00:00:00.000000000 Z
+date: 2017-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -132,6 +132,7 @@ files:
 - lib/chupa-text/logger.rb
 - lib/chupa-text/mime-type-registry.rb
 - lib/chupa-text/mime-type.rb
+- lib/chupa-text/screenshot.rb
 - lib/chupa-text/size-parser.rb
 - lib/chupa-text/text-data.rb
 - lib/chupa-text/version.rb
@@ -145,6 +146,7 @@ files:
 - test/fixture/command/chupa-text/hello.txt
 - test/fixture/command/chupa-text/hello.txt.gz
 - test/fixture/command/chupa-text/no-decomposer.conf
+- test/fixture/command/chupa-text/numbers.csv
 - test/fixture/extractor/hello.txt
 - test/fixture/gzip/hello.tar.gz
 - test/fixture/gzip/hello.tgz