chupa-text 1.0.0

data/lib/chupa-text/command/chupa-text.rb

@@ -0,0 +1,102 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "optparse"
+
+module ChupaText
+  module Command
+    class ChupaText
+      class << self
+        def run(*arguments)
+          chupa_text = new
+          chupa_text.run(*arguments)
+        end
+      end
+
+      def initialize
+        @input = nil
+        @configuration = Configuration.default
+      end
+
+      def run(*arguments)
+        return false unless parse_arguments(arguments)
+
+        Decomposers.load
+        extractor = create_extractor
+        data = create_data
+        formatter = create_formatter
+        formatter.format_start(data)
+        extractor.extract(data) do |extracted|
+          formatter.format_extracted(extracted)
+        end
+        formatter.format_finish(data)
+        true
+      end
+
+      private
+      def load_configuration(path)
+        loader = ConfigurationLoader.new(@configuration)
+        loader.load(path)
+      end
+
+      def parse_arguments(arguments)
+        parser = create_option_parser
+        rest = nil
+        begin
+          rest = parser.parse!(arguments)
+        rescue OptionParser::ParseError
+          puts($!.message)
+          return false
+        end
+        if rest.size > 1
+          puts(parser.help)
+          return false
+        end
+        @input, = rest
+        true
+      end
+
+      def create_option_parser
+        parser = OptionParser.new
+        parser.banner += " [FILE_OR_URI]"
+        parser.version = VERSION
+        parser.on("--configuration=FILE",
+                  "Read configuration from FILE.") do |path|
+          load_configuration(path)
+        end
+        parser
+      end
+
+      def create_extractor
+        extractor = Extractor.new
+        extractor.apply_configuration(@configuration)
+        extractor
+      end
+
+      def create_data
+        if @input.nil?
+          VirtualFileData.new(nil, $stdin)
+        else
+          InputData.new(@input)
+        end
+      end
+
+      def create_formatter
+        Formatters::JSON.new($stdout)
+      end
+    end
+  end
+end

data/lib/chupa-text/configuration-loader.rb

@@ -0,0 +1,95 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "pathname"
+
+module ChupaText
+  class ConfigurationLoader
+    attr_reader :decomposer
+    attr_reader :mime_type
+    def initialize(configuration)
+      @configuration = configuration
+      @decomposer = DecomposerLoader.new(@configuration.decomposer)
+      @mime_type = MIMETypeLoader.new(@configuration.mime_type_registry)
+      @load_paths = []
+      data_dir = File.join(File.dirname(__FILE__), "..", "..", "data")
+      @load_paths << File.expand_path(data_dir)
+    end
+
+    def load(path)
+      path = resolve_path(path)
+      File.open(path) do |file|
+        instance_eval(file.read, path, 1)
+      end
+    end
+
+    private
+    def resolve_path(path)
+      return path if File.exist?(path)
+      return path if Pathname(path).absolute?
+      @load_paths.each do |load_path|
+        resolved_path = File.join(load_path, path)
+        return resolved_path if File.exist?(resolved_path)
+      end
+      path
+    end
+
+    class DecomposerLoader
+      def initialize(configuration)
+        @configuration = configuration
+      end
+
+      def names
+        @configuration.names
+      end
+
+      def names=(names)
+        @configuration.names = names
+      end
+
+      def [](name)
+        @configuration.options[name]
+      end
+
+      def []=(name, options)
+        @configuration.options[name] = options
+      end
+
+      def method_missing(name, *arguments)
+        return super if block_given?
+
+        if name.to_s.end_with?("=") and arguments.size == 1
+          value = arguments.first
+          self[name.to_s.gsub(/=\z/, "")] = value
+        elsif arguments.empty?
+          self[name.to_s]
+        else
+          super
+        end
+      end
+    end
+
+    class MIMETypeLoader
+      def initialize(registry)
+        @registry = registry
+      end
+
+      def []=(extension, mime_type)
+        @registry.register(extension, mime_type)
+      end
+    end
+  end
+end

data/lib/chupa-text/configuration.rb

@@ -0,0 +1,49 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module ChupaText
+  class Configuration
+    class << self
+      def default
+        @default ||= create_default
+      end
+
+      private
+      def create_default
+        configuration = new
+        loader = ConfigurationLoader.new(configuration)
+        loader.load("chupa-text.conf")
+        configuration
+      end
+    end
+
+    attr_reader :decomposer
+    attr_accessor :mime_type_registry
+    def initialize
+      @decomposer = DecomposerConfiguration.new
+      @mime_type_registry = MIMEType.registry
+    end
+
+    class DecomposerConfiguration
+      attr_accessor :names
+      attr_accessor :options
+      def initialize
+        @names = ["*"]
+        @options = {}
+      end
+    end
+  end
+end

data/lib/chupa-text/data.rb

@@ -0,0 +1,149 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+require "uri"
+require "open-uri"
+
+module ChupaText
+  class Data
+    # @return [URI, nil] The URI of the data if the data is for remote
+    #   or local file, `nil` if the data isn't associated with any
+    #   URIs.
+    attr_reader :uri
+
+    # @return [String, nil] The content of the data, `nil` if the data
+    #   doesn't have any content.
+    attr_accessor :body
+
+    # @return [Integer, nil] The byte size of the data, `nil` if the data
+    #   doesn't have any content.
+    attr_accessor :size
+
+    # @return [String, nil] The path associated with the content of
+    #   the data, `nil` if the data doesn't associated with any file.
+    #
+    #   The path may not be related with the original content. For
+    #   example, `"/tmp/XXX.txt"` may be returned for the data of
+    #   `"http://example.com/XXX.txt"`.
+    #
+    #   This value is useful to use an external command to extract
+    #   text and meta-data.
+    attr_accessor :path
+
+    attr_accessor :attributes
+
+    # @return [Data, nil] The source of the data. For example, text
+    #   data (`hello.txt`) in archive data (`hello.tar`) have the
+    #   archive data in {#source}.
+    attr_accessor :source
+
+    def initialize
+      @uri = nil
+      @body = nil
+      @size = nil
+      @path = nil
+      @mime_type = nil
+      @attributes = {}
+      @source = nil
+    end
+
+    def initialize_copy(object)
+      super
+      @attributes = @attributes.dup
+      self
+    end
+
+    # @param [String, URI, nil] uri The URI for the data. If `uri` is
+    #   `nil`, it means that the data isn't associated with any URIs.
+    def uri=(uri)
+      case uri
+      when String, Pathname
+        uri = URI.parse(uri.to_s)
+      end
+      @uri = uri
+    end
+
+    def open
+      yield(StringIO.new(body))
+    end
+
+    def [](name)
+      @attributes[name]
+    end
+
+    def []=(name, value)
+      @attributes[name] = value
+    end
+
+    # @return [String] The MIME type of the data. If MIME type
+    #   isn't set, guesses MIME type from path and body.
+    # @return [nil] If MIME type isn't set and it can't guess MIME type
+    #   from path and body.
+    def mime_type
+      @mime_type || guess_mime_type
+    end
+
+    # @param [String, nil] type The MIME type of the data. You can
+    #   unset MIME type by `nil`. If you unset MIME type, MIME type
+    #   is guessed from path and body of the data.
+    def mime_type=(type)
+      @mime_type = type
+    end
+
+    # @return [String, nil] Normalized extension as String if {#uri}
+    #   is not `nil`, `nil` otherwise. The normalized extension uses
+    #   lower case like `pdf` not `PDF`.
+    def extension
+      return nil if @uri.nil?
+      File.extname(@uri.path).downcase.gsub(/\A\./, "")
+    end
+
+    # @return [Bool] true if MIME type is "text/XXX", false
+    #   otherwise.
+    def text?
+      (mime_type || "").start_with?("text/")
+    end
+
+    private
+    def guess_mime_type
+      guess_mime_type_from_uri or
+        guess_mime_type_from_body
+    end
+
+    def guess_mime_type_from_uri
+      MIMEType.registry.find(extension)
+    end
+
+    def guess_mime_type_from_body
+      mime_type = nil
+      change_encoding(body, "UTF-8") do |utf8_body|
+        mime_type = "text/plain" if utf8_body.valid_encoding?
+      end
+      mime_type
+    end
+
+    def change_encoding(string, encoding)
+      return if string.nil?
+      begin
+        original_encoding = string.encoding
+        string.force_encoding(encoding)
+        yield(string)
+      ensure
+        string.force_encoding(original_encoding)
+      end
+    end
+  end
+end

data/lib/chupa-text/decomposer-registry.rb

@@ -0,0 +1,37 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module ChupaText
+  class DecomposerRegistry
+    include Enumerable
+
+    def initialize
+      @decomposer_classes = {}
+    end
+
+    def register(name, decomposer_class)
+      @decomposer_classes[name] = decomposer_class
+    end
+
+    def find(name)
+      @decomposer_classes[name]
+    end
+
+    def each(&block)
+      @decomposer_classes.each(&block)
+    end
+  end
+end

data/lib/chupa-text/decomposer.rb

@@ -0,0 +1,37 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+module ChupaText
+  class Decomposer
+    class << self
+      def registry
+        @@registry ||= DecomposerRegistry.new
+      end
+    end
+
+    def initialize(options)
+      @options = options
+    end
+
+    def target?(data)
+      raise NotImplementedError, "must implement #{self.class}\##{__method__}"
+    end
+
+    def decompose(data)
+      raise NotImplementedError, "must implement #{self.class}\##{__method__}"
+    end
+  end
+end