chupa-text 1.0.0

data/lib/chupa-text/decomposers.rb

@@ -0,0 +1,59 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module ChupaText
+  module Decomposers
+    class << self
+      def load
+        paths = []
+        $LOAD_PATH.each do |load_path|
+          next unless File.directory?(load_path)
+          Dir.chdir(load_path) do
+            Dir.glob("chupa-text/decomposers/*.rb") do |decomposer_path|
+              paths << decomposer_path.gsub(/\.rb\z/, "")
+            end
+          end
+        end
+        paths.each do |path|
+          require path
+        end
+      end
+
+      def create(registry, configuration)
+        enabled_names = resolve_names(registry, configuration.names)
+        enabled_names.collect do |enabled_name|
+          decomposer_class = registry.find(enabled_name)
+          options = configuration.options[name] || {}
+          decomposer_class.new(options)
+        end
+      end
+
+      private
+      def resolve_names(registry, enabled_names)
+        resolved_names = []
+        flag = 0
+        flag |= File::FNM_EXTGLOB if File.const_defined?(:FNM_EXTGLOB)
+        enabled_names.each do |enabled_name|
+          registry.each do |name,|
+            next unless File.fnmatch(enabled_name, name, flag)
+            resolved_names << name
+          end
+        end
+        resolved_names
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/csv.rb

@@ -0,0 +1,44 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "csv"
+
+module ChupaText
+  module Decomposers
+    class CSV < Decomposer
+      registry.register("csv", self)
+
+      def target?(data)
+        data.extension == "csv" or
+          data.mime_type == "text/csv"
+      end
+
+      def decompose(data)
+        text = ""
+        data.open do |input|
+          csv = ::CSV.new(input)
+          csv.each do |row|
+            text << row.join(" ")
+            text << "\n"
+          end
+        end
+        text_data = TextData.new(text)
+        text_data.uri = data.uri
+        yield(text_data)
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/gzip.rb

@@ -0,0 +1,51 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "stringio"
+require "zlib"
+
+module ChupaText
+  module Decomposers
+    class Gzip < Decomposer
+      registry.register("gzip", self)
+
+      TARGET_EXTENSIONS = ["gz", "tgz"]
+      TARGET_MIME_TYPES = [
+        "application/gzip",
+        "application/x-gzip",
+        "application/x-gtar-compressed",
+      ]
+      def target?(data)
+        TARGET_EXTENSIONS.include?(data.extension) or
+          TARGET_MIME_TYPES.include?(data.mime_type)
+      end
+
+      def decompose(data)
+        reader = Zlib::GzipReader.new(StringIO.new(data.body))
+        uri = nil
+        case data.extension
+        when "gz"
+          uri = data.uri.to_s.gsub(/\.gz\z/i, "")
+        when "tgz"
+          uri = data.uri.to_s.gsub(/\.tgz\z/i, ".tar")
+        end
+        extracted = VirtualFileData.new(uri, reader)
+        extracted.source = data
+        yield(extracted)
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/tar.rb

@@ -0,0 +1,42 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "stringio"
+require "rubygems/package"
+
+module ChupaText
+  module Decomposers
+    class Tar < Decomposer
+      registry.register("tar", self)
+
+      def target?(data)
+        data.extension == "tar" or
+          data.mime_type == "application/x-tar"
+      end
+
+      def decompose(data)
+        Gem::Package::TarReader.new(StringIO.new(data.body)) do |reader|
+          reader.each do |entry|
+            next unless entry.file?
+            extracted = VirtualFileData.new(entry.full_name, entry)
+            extracted.source = data
+            yield(extracted)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposers/xml.rb

@@ -0,0 +1,55 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "rexml/document"
+require "rexml/streamlistener"
+
+module ChupaText
+  module Decomposers
+    class XML < Decomposer
+      registry.register("xml", self)
+
+      def target?(data)
+        data.extension == "xml" or
+          data.mime_type == "text/xml"
+      end
+
+      def decompose(data)
+        text = ""
+        listener = Listener.new(text)
+        data.open do |input|
+          parser = REXML::Parsers::StreamParser.new(input, listener)
+          parser.parse
+        end
+        text_data = TextData.new(text)
+        text_data.uri = data.uri
+        yield(text_data)
+      end
+
+      class Listener
+        include REXML::StreamListener
+
+        def initialize(output)
+          @output = output
+        end
+
+        def text(text)
+          @output << text
+        end
+      end
+    end
+  end
+end

data/lib/chupa-text/extractor.rb

@@ -0,0 +1,91 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "pathname"
+require "uri"
+
+module ChupaText
+  class Extractor
+    def initialize
+      @decomposers = []
+    end
+
+    # Sets the extractor up by the configuration. It adds decomposers
+    # enabled in the configuration.
+    #
+    # @param [Configuration] configuration The configuration to be
+    #   applied.
+    #
+    # @return [void]
+    def apply_configuration(configuration)
+      decomposers = Decomposers.create(Decomposer.registry,
+                                       configuration.decomposer)
+      decomposers.each do |decomposer|
+        add_decomposer(decomposer)
+      end
+    end
+
+    def add_decomposer(decomposer)
+      @decomposers << decomposer
+    end
+
+    # Extracts texts from input. Each extracted text is passes to the
+    # given block.
+    #
+    # @param [Data, String] input The input to be extracted texts.
+    #   If `input` is `String`, it is treated as the local file path or URI
+    #   of input data.
+    #
+    # @yield [text_data] Gives extracted text data to the block.
+    #   The block may be called zero or more times.
+    # @yieldparam [Data] text_data The extracted text data.
+    #   You can get text data by `text_data.body`.
+    #
+    # @return [void]
+    def extract(input)
+      targets = [ensure_data(input)]
+      until targets.empty?
+        target = targets.pop
+        decomposer = find_decomposer(target)
+        if decomposer.nil?
+          yield(target) if target.text?
+          next
+        end
+        decomposer.decompose(target) do |decomposed|
+          targets.push(decomposed)
+        end
+      end
+    end
+
+    private
+    def ensure_data(input)
+      case input
+      when String, Pathname, URI::Generic
+        data = Data.new
+        data.uri = input.to_s
+        data
+      else
+        input
+      end
+    end
+
+    def find_decomposer(data)
+      @decomposers.find do |decomposer|
+        decomposer.target?(data)
+      end
+    end
+  end
+end

data/lib/chupa-text/file-content.rb

@@ -0,0 +1,35 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module ChupaText
+  class FileContent
+    attr_reader :size
+    attr_reader :path
+
+    def initialize(path)
+      @path = path
+      @size = File.size(@path)
+    end
+
+    def open(&block)
+      File.open(@path, "rb", &block)
+    end
+
+    def body
+      @body ||= open {|file| file.read}
+    end
+  end
+end

data/lib/chupa-text/formatters.rb

@@ -0,0 +1,17 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "chupa-text/formatters/json"

data/lib/chupa-text/formatters/json.rb

@@ -0,0 +1,60 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "json"
+
+module ChupaText
+  module Formatters
+    class JSON
+      def initialize(output)
+        @output = output
+        @formatted = {}
+      end
+
+      def format_start(data)
+        format_headers(data, @formatted)
+        @formatted["texts"] = []
+      end
+
+      def format_extracted(data)
+        text = {}
+        format_headers(data, text)
+        text["body"] = data.body
+        @formatted["texts"] << text
+      end
+
+      def format_finish(data)
+        @output << ::JSON.pretty_generate(@formatted)
+        @output << "\n"
+      end
+
+      private
+      def format_headers(data, target)
+        format_header("mime-type", data.mime_type, target)
+        format_header("uri",       data.uri,       target)
+        format_header("size",      data.size,      target)
+        data.attributes.each do |name, value|
+          format_header(name, value, target)
+        end
+      end
+
+      def format_header(name, value, target)
+        return if value.nil?
+        target[name] = value
+      end
+    end
+  end
+end