chupa-text 1.0.4 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b085d3a77d1092447c542a34302dcecaa12ff8a2
-  data.tar.gz: a6eca07515400f3440ef5395197ca1e87a8049a3
+  metadata.gz: b7bb1a550ae5f986d102ceb5b13c542201a1d630
+  data.tar.gz: 0ca5560f911c3f19ab018f8b0cb37780a2738e25
 SHA512:
-  metadata.gz: 4bbb30d051f5f7035ea64fe80585191038d372e3b9c65138e5e3f45d10443929839a8934981a14a494988d59a3b99ea3442f9ab90c846f6e67827f12fd5549aa
-  data.tar.gz: 08829033f8ab4d5ae0b004c8b81857605a2014f5fed058b0f181357916bff35a4927b05fc1becf5e4dd7cc81a682063675584e4f19c1baaf5819fbfc95d78799
+  metadata.gz: 0bfc9e586bfbcd10ba60fdfa982fc3027e0e3b2433874fd5870ece5396201e42871de772d6eb10c6e39b32172ea18e21cc84944e48182f70dc72918b949719a8
+  data.tar.gz: 40e10ff0742ae1a5b98733c8a64ea3f10934da02c30b834e8bfba0914271b31bbc04a151ddef19f8f6a11101f4de7590afc7f3d4901caca37e3d7d7c1ce4207a

data/Rakefile

@@ -34,7 +34,9 @@ end
 helper.install
 spec = helper.gemspec
 
-Packnga::DocumentTask.new(spec) do
+Packnga::DocumentTask.new(spec) do |task|
+  task.original_language = "en"
+  task.translate_language = "ja"
 end
 
 Packnga::ReleaseTask.new(spec) do

data/chupa-text.gemspec

@@ -1,6 +1,6 @@
-# -*- mode: ruby; coding: utf-8 -*-
+# -*- ruby -*-
 #
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
   entries = readme.split(/^\#\#\s(.*)$/)
   description = clean_white_space.call(entries[entries.index("Description") + 1])
   spec.summary, spec.description, = description.split(/\n\n+/, 3)
-  spec.license = "LGPLv2.1 or later"
+  spec.license = "LGPL-2.1+"
   spec.files = ["#{spec.name}.gemspec"]
   spec.files += ["README.md", "LICENSE.txt", "Rakefile", "Gemfile"]
   spec.files += [".yardopts"]

data/data/mime-types.conf

@@ -34,3 +34,6 @@ mime_types["xlsx"] =
   "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
 mime_types["pptx"] =
   "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+
+mime_types["eml"] = "message/rfc822"
+mime_types["mew"] = "message/rfc822"

data/doc/text/news.md

@@ -1,5 +1,20 @@
 # News
 
+## 1.0.5: 2017-05-02
+
+### Improvements
+
+  * Added `message/rfc822` MIME type association with `.eml` and
+    `.mew` into the default MIME type list.
+
+  * Searched decomposer even if MIME type is `text/plain`.
+
+  * `ChupaText::Data#initialize`: Accepted source data.
+
+  * `ChupaText::UnknownEncodingError`: Added.
+
+  * Added plain text formatter.
+
 ## 1.0.4: 2014-02-17
 
   * Removed a needless optimization.

data/lib/chupa-text/command/chupa-text-generate-decomposer.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -76,7 +76,7 @@ module ChupaText
       end
 
       def lgplv2_1_or_later_license
-        "LGPLv2.1 or later"
+        "LGPL-2.1+"
       end
 
       def create_option_parser
@@ -154,7 +154,7 @@ module ChupaText
       def generate_gemspec
         create_file("#{gem_name}.gemspec") do |file|
           file.puts(<<-GEMSPEC)
-# -*- mode: ruby; coding: utf-8 -*-
+# -*- ruby -*-
 
 Gem::Specification.new do |spec|
   spec.name = "#{gem_name}"
@@ -183,7 +183,7 @@ end
       def generate_gemfile
         create_file("Gemfile") do |file|
           file.puts(<<-Gemfile)
-# -*- mode: ruby; coding: utf-8 -*-
+# -*- ruby -*-
 
 source "https://rubygems.org/"
 
@@ -195,7 +195,7 @@ gemspec
       def generate_rakefile
         create_file("Rakefile") do |file|
           file.puts(<<-RAKEFILE)
-# -*- mode: ruby; coding: utf-8 -*-
+# -*- ruby -*-
 
 require "bundler/gem_tasks"
 

data/lib/chupa-text/command/chupa-text.rb

@@ -26,10 +26,13 @@ module ChupaText
         end
       end
 
+      AVAILABLE_FORMATS = [:json, :text]
+
       def initialize
         @input = nil
         @configuration = Configuration.default
         @enable_gems = true
+        @format = :json
       end
 
       def run(*arguments)
@@ -89,6 +92,12 @@ module ChupaText
                   "Appends PATH to decomposer load path.") do |path|
           $LOAD_PATH << path
         end
+        parser.on("--format=FORMAT", AVAILABLE_FORMATS,
+                  "Output FORMAT.",
+                  "[#{AVAILABLE_FORMATS.join(', ')}]",
+                  "(default: json)") do |format|
+          @format = format
+        end
 
         parser.separator("")
         parser.separator("Log related options:")
@@ -150,7 +159,12 @@ module ChupaText
       end
 
       def create_formatter
-        Formatters::JSON.new($stdout)
+        case @format
+        when :json
+          Formatters::JSON.new($stdout)
+        when :text
+          Formatters::Text.new($stdout)
+        end
       end
     end
   end

data/lib/chupa-text/data.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -51,7 +51,7 @@ module ChupaText
     #   archive data in {#source}.
     attr_accessor :source
 
-    def initialize
+    def initialize(options={})
       @uri = nil
       @body = nil
       @size = nil
@@ -59,6 +59,9 @@ module ChupaText
       @mime_type = nil
       @attributes = Attributes.new
       @source = nil
+      @options = options || {}
+      source_data = @options[:source_data]
+      merge!(source_data) if source_data
     end
 
     def initialize_copy(object)
@@ -67,6 +70,23 @@ module ChupaText
       self
     end
 
+    # Merges metadata from data.
+    #
+    # @param [Data] data The data to be merged.
+    #
+    # @return [void]
+    def merge!(data)
+      self.uri = data.uri
+      self.path = data.path
+      data.attributes.each do |name, value|
+        self[name] = value
+      end
+      if data.mime_type
+        self["source-mime-types"] ||= []
+        self["source-mime-types"].unshift(data.mime_type)
+      end
+    end
+
     # @param [String, URI, nil] uri The URI for the data. If `uri` is
     #   `nil`, it means that the data isn't associated with any URIs.
     def uri=(uri)

data/lib/chupa-text/default-logger.rb

@@ -137,7 +137,11 @@ module ChupaText
       def format_message(message)
         case message
         when String
-          message
+          if message.end_with?("\n")
+            message
+          else
+            "#{message}\n"
+          end
         when Exception
           "#{message.message}(#{message.class})\n" +
             (message.backtrace || []).join("\n")

data/lib/chupa-text/error.rb

@@ -22,7 +22,7 @@ module ChupaText
     attr_reader :data
     def initialize(data)
       @data = data
-      super("Encrypted data: <#{data.path}>(#{data.mime_type})")
+      super("Encrypted data: <#{data.uri}>(#{data.mime_type})")
     end
   end
 
@@ -31,7 +31,16 @@ module ChupaText
     def initialize(data, detail)
       @data = data
       @detail = detail
-      super("Invalid data: <#{data.path}>(#{data.mime_type}): <#{detail}>")
+      super("Invalid data: <#{data.uri}>(#{data.mime_type}): <#{detail}>")
+    end
+  end
+
+  class UnknownEncodingError < Error
+    attr_reader :data, :encoding
+    def initialize(data, encoding)
+      @data = data
+      @encoding = encoding
+      super("Unknown encoding data: <#{data.uri}>(#{data.mime_type}): <#{encoding}>")
     end
   end
 end

data/lib/chupa-text/extractor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -60,26 +60,28 @@ module ChupaText
     def extract(input)
       targets = [ensure_data(input)]
       until targets.empty?
-        target = targets.pop
+        target = targets.shift
         debug do
-          "#{log_tag}[extract][target] <#{target.path}>:<#{target.mime_type}>"
-        end
-        if target.text_plain?
-          yield(target)
-          next
+          "#{log_tag}[extract][target] <#{target.uri}>:<#{target.mime_type}>"
         end
         decomposer = find_decomposer(target)
         if decomposer.nil?
-          debug {"#{log_tag}[extract][decomposer] not found"}
-          yield(target) if target.text?
-          next
+          if target.text_plain?
+            debug {"#{log_tag}[extract][text-plain]"}
+            yield(target)
+            next
+          else
+            debug {"#{log_tag}[extract][decomposer] not found"}
+            yield(target) if target.text?
+            next
+          end
         end
         debug {"#{log_tag}[extract][decomposer] #{decomposer.class}"}
         decomposer.decompose(target) do |decomposed|
           debug do
             "#{log_tag}[extract][decomposed] " +
               "#{decomposer.class}: " +
-              "<#{target.path}>:<#{target.mime_type}> -> " +
+              "<#{target.uri}>:<#{target.mime_type}> -> " +
               "<#{decomposed.mime_type}>"
           end
           targets.push(decomposed)

data/lib/chupa-text/formatters.rb

@@ -15,3 +15,4 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 require "chupa-text/formatters/json"
+require "chupa-text/formatters/text"

data/lib/chupa-text/formatters/json.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -21,23 +21,24 @@ module ChupaText
     class JSON
       def initialize(output)
         @output = output
-        @formatted = {}
+        @texts = []
       end
 
       def format_start(data)
-        format_headers(data, @formatted)
-        @formatted["texts"] = []
       end
 
       def format_extracted(data)
         text = {}
         format_headers(data, text)
         text["body"] = data.body
-        @formatted["texts"] << text
+        @texts << text
       end
 
       def format_finish(data)
-        @output << ::JSON.pretty_generate(@formatted)
+        formatted = {}
+        format_headers(data, formatted)
+        formatted["texts"] = @texts
+        @output << ::JSON.pretty_generate(formatted)
         @output << "\n"
       end
 

data/lib/chupa-text/formatters/text.rb

@@ -0,0 +1,22 @@
+module ChupaText
+  module Formatters
+    class Text
+      def initialize(output)
+        @output = output
+        @texts = []
+      end
+
+      def format_start(data)
+      end
+
+      def format_extracted(data)
+        @texts << data.body
+      end
+
+      def format_finish(data)
+        @output << @texts.join("\n\x0c\n")
+        @output << "\n"
+      end
+    end
+  end
+end

data/lib/chupa-text/input-data.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -19,8 +19,8 @@ require "open-uri"
 
 module ChupaText
   class InputData < Data
-    def initialize(uri)
-      super()
+    def initialize(uri, options={})
+      super(options)
       self.uri = uri
       if @uri.class == URI::Generic
         @content = FileContent.new(@uri.path)

data/lib/chupa-text/text-data.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,8 +16,8 @@
 
 module ChupaText
   class TextData < Data
-    def initialize(text)
-      super()
+    def initialize(text, options={})
+      super(options)
       self.mime_type = "text/plain"
       self.body = text
       self.size = text.bytesize

data/lib/chupa-text/version.rb

@@ -15,5 +15,5 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 
 module ChupaText
-  VERSION = "1.0.4"
+  VERSION = "1.0.5"
 end

data/lib/chupa-text/virtual-file-data.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+# Copyright (C) 2013-2017  Kouhei Sutou <kou@clear-code.com>
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -16,8 +16,8 @@
 
 module ChupaText
   class VirtualFileData < Data
-    def initialize(uri, input)
-      super()
+    def initialize(uri, input, options={})
+      super(options)
       self.uri = uri
       if @uri
         path = @uri.path

metadata

@@ -1,83 +1,83 @@
 --- !ruby/object:Gem::Specification
 name: chupa-text
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.5
 platform: ruby
 authors:
 - Kouhei Sutou
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-17 00:00:00.000000000 Z
+date: 2017-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: packnga
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: redcarpet
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: ''
@@ -89,85 +89,86 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- chupa-text.gemspec
-- README.md
+- ".yardopts"
+- Gemfile
 - LICENSE.txt
+- README.md
 - Rakefile
-- Gemfile
-- .yardopts
+- bin/chupa-text
+- bin/chupa-text-generate-decomposer
+- chupa-text.gemspec
 - data/chupa-text.conf
 - data/mime-types.conf
-- lib/chupa-text/data.rb
-- lib/chupa-text/version.rb
-- lib/chupa-text/logger.rb
-- lib/chupa-text/extractor.rb
-- lib/chupa-text/command/chupa-text.rb
+- doc/text/command-line.md
+- doc/text/decomposer.md
+- doc/text/library.md
+- doc/text/news.md
+- lib/chupa-text.rb
+- lib/chupa-text/attributes.rb
+- lib/chupa-text/command.rb
 - lib/chupa-text/command/chupa-text-generate-decomposer.rb
+- lib/chupa-text/command/chupa-text.rb
+- lib/chupa-text/configuration-loader.rb
 - lib/chupa-text/configuration.rb
+- lib/chupa-text/data.rb
 - lib/chupa-text/decomposer-registry.rb
-- lib/chupa-text/text-data.rb
-- lib/chupa-text/configuration-loader.rb
-- lib/chupa-text/virtual-file-data.rb
-- lib/chupa-text/size-parser.rb
-- lib/chupa-text/formatters/json.rb
-- lib/chupa-text/virtual-content.rb
+- lib/chupa-text/decomposer.rb
+- lib/chupa-text/decomposers.rb
 - lib/chupa-text/decomposers/csv.rb
+- lib/chupa-text/decomposers/gzip.rb
 - lib/chupa-text/decomposers/tar.rb
 - lib/chupa-text/decomposers/xml.rb
-- lib/chupa-text/decomposers/gzip.rb
+- lib/chupa-text/default-logger.rb
 - lib/chupa-text/error.rb
-- lib/chupa-text/formatters.rb
-- lib/chupa-text/decomposers.rb
-- lib/chupa-text/command.rb
-- lib/chupa-text/file-content.rb
-- lib/chupa-text/decomposer.rb
 - lib/chupa-text/external-command.rb
-- lib/chupa-text/mime-type.rb
-- lib/chupa-text/loggable.rb
+- lib/chupa-text/extractor.rb
+- lib/chupa-text/file-content.rb
+- lib/chupa-text/formatters.rb
+- lib/chupa-text/formatters/json.rb
+- lib/chupa-text/formatters/text.rb
 - lib/chupa-text/input-data.rb
+- lib/chupa-text/loggable.rb
+- lib/chupa-text/logger.rb
 - lib/chupa-text/mime-type-registry.rb
-- lib/chupa-text/default-logger.rb
-- lib/chupa-text/attributes.rb
-- lib/chupa-text.rb
-- doc/text/decomposer.md
-- doc/text/command-line.md
-- doc/text/news.md
-- doc/text/library.md
-- test/test-decomposers.rb
-- test/test-default-logger.rb
-- test/test-decomposer.rb
-- test/test-virtual-content.rb
+- lib/chupa-text/mime-type.rb
+- lib/chupa-text/size-parser.rb
+- lib/chupa-text/text-data.rb
+- lib/chupa-text/version.rb
+- lib/chupa-text/virtual-content.rb
+- lib/chupa-text/virtual-file-data.rb
 - test/command/test-chupa-text.rb
-- test/helper.rb
-- test/test-configuration-loader.rb
-- test/test-mime-type-registry.rb
-- test/test-decomposer-registry.rb
-- test/fixture/gzip/hello.txt.gz
-- test/fixture/gzip/hello.tgz
-- test/fixture/gzip/hello.tar.gz
-- test/fixture/tar/top-level.tar
-- test/fixture/tar/directory.tar
-- test/fixture/command/chupa-text/hello.txt.gz
-- test/fixture/command/chupa-text/no-decomposer.conf
-- test/fixture/command/chupa-text/hello.txt
-- test/fixture/extractor/hello.txt
-- test/test-attributes.rb
-- test/test-external-command.rb
-- test/test-size-parser.rb
 - test/decomposers/test-csv.rb
 - test/decomposers/test-gzip.rb
 - test/decomposers/test-tar.rb
 - test/decomposers/test-xml.rb
+- test/fixture/command/chupa-text/hello.txt
+- test/fixture/command/chupa-text/hello.txt.gz
+- test/fixture/command/chupa-text/no-decomposer.conf
+- test/fixture/extractor/hello.txt
+- test/fixture/gzip/hello.tar.gz
+- test/fixture/gzip/hello.tgz
+- test/fixture/gzip/hello.txt.gz
+- test/fixture/tar/directory.tar
+- test/fixture/tar/top-level.tar
+- test/helper.rb
+- test/run-test.rb
+- test/test-attributes.rb
+- test/test-configuration-loader.rb
 - test/test-data.rb
-- test/test-file-content.rb
+- test/test-decomposer-registry.rb
+- test/test-decomposer.rb
+- test/test-decomposers.rb
+- test/test-default-logger.rb
+- test/test-external-command.rb
 - test/test-extractor.rb
-- test/run-test.rb
+- test/test-file-content.rb
+- test/test-mime-type-registry.rb
+- test/test-size-parser.rb
 - test/test-text-data.rb
-- bin/chupa-text
-- bin/chupa-text-generate-decomposer
+- test/test-virtual-content.rb
 homepage: http://ranguba.org/#about-chupa-text
 licenses:
-- LGPLv2.1 or later
+- LGPL-2.1+
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -175,20 +176,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: ChupaText is an extensible text extractor. You can plug your custom text
   extractor in ChupaText. You can write your plugin by Ruby.
 test_files: []
-has_rdoc: