chupa-text 1.0.5 → 1.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7bb1a550ae5f986d102ceb5b13c542201a1d630
-  data.tar.gz: 0ca5560f911c3f19ab018f8b0cb37780a2738e25
+  metadata.gz: 3af110814417d579cde496c50b05d6d49964dd97
+  data.tar.gz: 2609179c9e817e33a82df59c53834236dc63da3d
 SHA512:
-  metadata.gz: 0bfc9e586bfbcd10ba60fdfa982fc3027e0e3b2433874fd5870ece5396201e42871de772d6eb10c6e39b32172ea18e21cc84944e48182f70dc72918b949719a8
-  data.tar.gz: 40e10ff0742ae1a5b98733c8a64ea3f10934da02c30b834e8bfba0914271b31bbc04a151ddef19f8f6a11101f4de7590afc7f3d4901caca37e3d7d7c1ce4207a
+  metadata.gz: 6ae9ed51763e954b6b99973dfb2c020e046ccfb57f6670bfe96e935319055a6f898e71f0ed30ba25f3d3c8a4180ea6dbdfcb80337829659fdfaeb6a08841d873
+  data.tar.gz: a114dd75066faaf9501cd1fd4891df6a8ed46adf99711721a4d4d4cfaf4b1a63bb852aba578d7c46f5c63535bd8d5a19508b65c44fa953087ff8ac301cc85535

data/doc/text/news.md

@@ -1,5 +1,11 @@
 # News
 
+## 1.0.6: 2017-07-05
+
+### Improvements
+
+  * Supported non ASCII characters in file name.
+
 ## 1.0.5: 2017-05-02
 
 ### Improvements

data/lib/chupa-text/command/chupa-text.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -154,7 +154,13 @@ module ChupaText
         if @input.nil?
           VirtualFileData.new(nil, $stdin)
         else
-          InputData.new(@input)
+          case @input
+          when /\A[a-z]+:\/\//i
+            input = URI.parse(@input)
+          else
+            input = Pathname(@input)
+          end
+          InputData.new(input)
         end
       end
 

data/lib/chupa-text/configuration-loader.rb

@@ -25,24 +25,25 @@ module ChupaText
       @decomposer = DecomposerLoader.new(@configuration.decomposer)
       @mime_types = MIMETypesLoader.new(@configuration.mime_type_registry)
       @load_paths = []
-      data_dir = File.join(File.dirname(__FILE__), "..", "..", "data")
-      @load_paths << File.expand_path(data_dir)
+      data_dir = Pathname(__dir__) + ".." + ".." + "data"
+      @load_paths << data_dir.expand_path
     end
 
     def load(path)
       path = resolve_path(path)
       File.open(path) do |file|
-        instance_eval(file.read, path, 1)
+        instance_eval(file.read, path.to_path, 1)
       end
     end
 
     private
     def resolve_path(path)
-      return path if File.exist?(path)
-      return path if Pathname(path).absolute?
+      path = Pathname(path) unless path.is_a?(Pathname)
+      return path if path.exist?
+      return path if path.absolute?
       @load_paths.each do |load_path|
-        resolved_path = File.join(load_path, path)
-        return resolved_path if File.exist?(resolved_path)
+        resolved_path = path.expand_path(load_path)
+        return resolved_path if resolved_path.exist?
       end
       path
     end

data/lib/chupa-text/data.rb

@@ -14,6 +14,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "cgi/util"
 require "uri"
 require "open-uri"
 
@@ -91,10 +92,27 @@ module ChupaText
     #   `nil`, it means that the data isn't associated with any URIs.
     def uri=(uri)
       case uri
-      when String, Pathname
-        uri = URI.parse(uri.to_s)
+      when Pathname
+        file_uri = ""
+        target = uri.expand_path
+        loop do
+          target, base = target.split
+          file_uri = "/#{CGI.escape(base.to_s)}#{file_uri}"
+          break if target.root?
+        end
+        file_uri = "file://#{file_uri}"
+        @uri = URI.parse(file_uri)
+        self.path ||= uri
+      when NilClass
+        @uri = nil
+        self.path = nil
+      else
+        unless uri.is_a?(URI)
+          uri = URI.parse(uri)
+        end
+        @uri = uri
+        self.path ||= @uri.path
       end
-      @uri = uri
     end
 
     def open

data/lib/chupa-text/decomposers/tar.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -31,12 +31,35 @@ module ChupaText
         Gem::Package::TarReader.new(StringIO.new(data.body)) do |reader|
           reader.each do |entry|
             next unless entry.file?
+            entry.extend(CopyStreamable)
             extracted = VirtualFileData.new(entry.full_name, entry)
             extracted.source = data
             yield(extracted)
           end
         end
       end
+
+      # TODO: Supporting output buffer in #read and #readpartial
+      # should be done in RubyGems' tar implementation.
+      module CopyStreamable
+        def readpartial(max_length, buffer=nil)
+          data = super(max_length)
+          if data.nil?
+            if max_length.zero?
+              return ""
+            else
+              raise EOFError
+            end
+          end
+
+          if buffer.nil?
+            data
+          else
+            buffer.replace(data)
+            buffer
+          end
+        end
+      end
     end
   end
 end

data/lib/chupa-text/formatters.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -14,5 +14,6 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "chupa-text/formatters/hash"
 require "chupa-text/formatters/json"
 require "chupa-text/formatters/text"

data/lib/chupa-text/formatters/hash.rb

@@ -0,0 +1,60 @@
+# Copyright (C) 2017  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module ChupaText
+  module Formatters
+    class Hash
+      def initialize
+        @texts = []
+      end
+
+      def format_start(data)
+      end
+
+      def format_extracted(data)
+        text = {}
+        format_headers(data, text)
+        text["body"] = data.body
+        @texts << text
+      end
+
+      def format_finish(data)
+        formatted = {}
+        format_headers(data, formatted)
+        formatted["texts"] = @texts
+        formatted
+      end
+
+      private
+      def format_headers(data, target)
+        format_header("mime-type", data.mime_type, target)
+        format_header("uri",       data.uri,       target)
+        if data.uri.class == URI::Generic
+          format_header("path",      data.path,      target)
+        end
+        format_header("size",      data.size,      target)
+        data.attributes.each do |name, value|
+          format_header(name, value, target)
+        end
+      end
+
+      def format_header(name, value, target)
+        return if value.nil?
+        target[name] = value
+      end
+    end
+  end
+end

data/lib/chupa-text/formatters/json.rb

@@ -16,46 +16,21 @@
 
 require "json"
 
+require "chupa-text/formatters/hash"
+
 module ChupaText
   module Formatters
-    class JSON
+    class JSON < Hash
       def initialize(output)
+        super()
         @output = output
-        @texts = []
-      end
-
-      def format_start(data)
-      end
-
-      def format_extracted(data)
-        text = {}
-        format_headers(data, text)
-        text["body"] = data.body
-        @texts << text
       end
 
       def format_finish(data)
-        formatted = {}
-        format_headers(data, formatted)
-        formatted["texts"] = @texts
+        formatted = super
         @output << ::JSON.pretty_generate(formatted)
         @output << "\n"
       end
-
-      private
-      def format_headers(data, target)
-        format_header("mime-type", data.mime_type, target)
-        format_header("uri",       data.uri,       target)
-        format_header("size",      data.size,      target)
-        data.attributes.each do |name, value|
-          format_header(name, value, target)
-        end
-      end
-
-      def format_header(name, value, target)
-        return if value.nil?
-        target[name] = value
-      end
     end
   end
 end

data/lib/chupa-text/input-data.rb

@@ -14,6 +14,7 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
+require "cgi/util"
 require "uri"
 require "open-uri"
 
@@ -23,9 +24,10 @@ module ChupaText
       super(options)
       self.uri = uri
       if @uri.class == URI::Generic
-        @content = FileContent.new(@uri.path)
+        @content = FileContent.new(path)
       else
         @content = download
+        self.path = @content.path
       end
     end
 
@@ -37,10 +39,6 @@ module ChupaText
       @content.size
     end
 
-    def path
-      @content.path
-    end
-
     def open(&block)
       @content.open(&block)
     end

data/lib/chupa-text/version.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013-2014  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module ChupaText
-  VERSION = "1.0.5"
+  VERSION = "1.0.6"
 end

data/lib/chupa-text/virtual-content.rb

@@ -26,30 +26,14 @@ module ChupaText
     def initialize(input, original_path=nil)
       @file = nil
       @base_name = compute_base_name(original_path)
-      chunk = input.read(BUFFER_SIZE) || ""
-      if chunk.bytesize != BUFFER_SIZE
-        @path = nil
-        @body = chunk
-        @size = @body.bytesize
-      else
-        @body = nil
-        @size = chunk.bytesize
-        setup_file do |file|
-          file.write(chunk)
-          while (chunk = input.read(BUFFER_SIZE))
-            @size += chunk.bytesize
-            file.write(chunk)
-          end
-        end
+      @body = nil
+      setup_file do |file|
+        @size = IO.copy_stream(input, file)
       end
     end
 
     def open(&block)
-      if @body
-        yield(StringIO.new(@body))
-      else
-        File.open(path, "rb", &block)
-      end
+      File.open(path, "rb", &block)
     end
 
     def body
@@ -57,9 +41,6 @@ module ChupaText
     end
 
     def path
-      ensure_setup_file do |file|
-        file.write(@body)
-      end
       @path
     end
 
@@ -77,10 +58,6 @@ module ChupaText
       end
     end
 
-    def ensure_setup_file(&block)
-      setup_file(&block) unless @file
-    end
-
     def setup_file
       @file = Tempfile.new(@base_name)
       @path = @file.path

data/lib/chupa-text/virtual-file-data.rb

@@ -19,11 +19,6 @@ module ChupaText
     def initialize(uri, input, options={})
       super(options)
       self.uri = uri
-      if @uri
-        path = @uri.path
-      else
-        path = nil
-      end
       @content = VirtualContent.new(input, path)
     end
 
@@ -35,10 +30,6 @@ module ChupaText
       @content.size
     end
 
-    def path
-      @content.path
-    end
-
     def open(&block)
       @content.open(&block)
     end

data/test/command/test-chupa-text.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -57,17 +57,21 @@ class TestCommandChupaText < Test::Unit::TestCase
     sub_test_case("file") do
       def test_single
         body = "Hello\n"
-        path = fixture_path("hello.txt").to_s
+        fixture_name = "hello.txt"
+        uri = fixture_uri(fixture_name).to_s
+        path = fixture_path(fixture_name).to_s
         assert_equal([
                        true,
                        {
                          "mime-type" => "text/plain",
-                         "uri"       => path,
+                         "uri"       => uri,
+                         "path"      => path,
                          "size"      => body.bytesize,
                          "texts"     => [
                            {
                              "mime-type" => "text/plain",
-                             "uri"       => path,
+                             "uri"       => uri,
+                             "path"      => path,
                              "size"      => body.bytesize,
                              "body"      => body,
                            },
@@ -161,18 +165,21 @@ class TestCommandChupaText < Test::Unit::TestCase
   sub_test_case("configuration") do
     def test_no_decomposer
       conf = fixture_path("no-decomposer.conf")
-      gz = fixture_path("hello.txt.gz")
+      fixture_name = "hello.txt.gz"
+      uri = fixture_uri(fixture_name)
+      path = fixture_path(fixture_name)
       assert_equal([
                      true,
                      {
-                       "uri"       => gz.to_s,
+                       "uri"       => uri.to_s,
+                       "path"      => path.to_s,
                        "mime-type" => "application/x-gzip",
-                       "size"      => gz.stat.size,
+                       "size"      => path.stat.size,
                        "texts"     => [],
                      },
                    ],
                    run_command("--configuration", conf.to_s,
-                               gz.to_s))
+                               path.to_s))
     end
   end
 end

data/test/decomposers/test-gzip.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -41,8 +41,8 @@ class TestDecomposersGzip < Test::Unit::TestCase
         @data = ChupaText::InputData.new(fixture_path("hello.txt.gz"))
       end
 
-      def test_path
-        assert_equal([URI.parse(fixture_path("hello.txt").to_s)],
+      def test_uri
+        assert_equal([fixture_uri("hello.txt")],
                      decompose(@data).collect(&:uri))
       end
 
@@ -64,7 +64,7 @@ class TestDecomposersGzip < Test::Unit::TestCase
       end
 
       def test_uri
-        assert_equal([URI.parse(fixture_path("hello.tar").to_s)],
+        assert_equal([fixture_uri("hello.tar")],
                      decompose(@data).collect(&:uri))
       end
 
@@ -91,7 +91,7 @@ class TestDecomposersGzip < Test::Unit::TestCase
       end
 
       def test_uri
-        assert_equal([URI.parse(fixture_path("hello.tar").to_s)],
+        assert_equal([fixture_uri("hello.tar")],
                      decompose(@data).collect(&:uri))
       end
 

data/test/helper.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,10 +16,16 @@
 
 require "pathname"
 require "tempfile"
+require "uri"
 
 module Helper
   def fixture_path(*components)
     base_path = Pathname(__FILE__).dirname + "fixture"
     base_path.join(*components)
   end
+
+  def fixture_uri(*components)
+    path = fixture_path(*components)
+    URI.parse("file://#{path}")
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chupa-text
 version: !ruby/object:Gem::Version
-  version: 1.0.5
+  version: 1.0.6
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-02 00:00:00.000000000 Z
+date: 2017-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -124,6 +124,7 @@ files:
 - lib/chupa-text/extractor.rb
 - lib/chupa-text/file-content.rb
 - lib/chupa-text/formatters.rb
+- lib/chupa-text/formatters/hash.rb
 - lib/chupa-text/formatters/json.rb
 - lib/chupa-text/formatters/text.rb
 - lib/chupa-text/input-data.rb