chupa-text 1.0.0

data/test/test-decomposer-registry.rb

@@ -0,0 +1,30 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestDecomposerRegistry < Test::Unit::TestCase
+  class CSVDecomposer < ChupaText::Decomposer
+  end
+
+  def setup
+    @registry = ChupaText::DecomposerRegistry.new
+  end
+
+  def test_register
+    assert_equal([], @registry.to_a)
+    @registry.register("csv", CSVDecomposer)
+    assert_equal([["csv", CSVDecomposer]], @registry.to_a)
+  end
+end

data/test/test-decomposer.rb

@@ -0,0 +1,41 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestDecomposer < Test::Unit::TestCase
+  sub_test_case("not-implemented") do
+    class NotImplementedDecomposer < ChupaText::Decomposer
+    end
+
+    def setup
+      @decomposer = NotImplementedDecomposer.new({})
+      @data = ChupaText::Data.new
+    end
+
+    def test_target?
+      message = "must implement #{NotImplementedDecomposer}\#target?"
+      assert_raise(NotImplementedError.new(message)) do
+        @decomposer.target?(@data)
+      end
+    end
+
+    def test_decompose
+      message = "must implement #{NotImplementedDecomposer}\#decompose"
+      assert_raise(NotImplementedError.new(message)) do
+        @decomposer.decompose(@data)
+      end
+    end
+  end
+end

data/test/test-decomposers.rb

@@ -0,0 +1,59 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestDecomposers < Test::Unit::TestCase
+  class CSVDecomposer < ChupaText::Decomposer
+  end
+
+  def setup
+    @registry = ChupaText::DecomposerRegistry.new
+    @registry.register("csv", CSVDecomposer)
+    @configuration = ChupaText::Configuration.new
+  end
+
+  sub_test_case("create") do
+    def test_default
+      decomposers = create
+      assert_equal([CSVDecomposer], decomposers.collect(&:class))
+    end
+
+    def test_no_match
+      @configuration.decomposer.names = []
+      decomposers = create
+      assert_equal([], decomposers.collect(&:class))
+    end
+
+    def test_glob
+      @configuration.decomposer.names = ["*sv"]
+      decomposers = create
+      assert_equal([CSVDecomposer], decomposers.collect(&:class))
+    end
+
+    def test_ext_glob
+      unless File.const_defined?(:FNM_EXTGLOB)
+        omit("File::FNM_EXTGLOB is required")
+      end
+      @configuration.decomposer.names = ["{a,b,c}sv"]
+      decomposers = create
+      assert_equal([CSVDecomposer], decomposers.collect(&:class))
+    end
+
+    private
+    def create
+      ChupaText::Decomposers.create(@registry, @configuration.decomposer)
+    end
+  end
+end

data/test/test-extractor.rb

@@ -0,0 +1,125 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestExtractor < Test::Unit::TestCase
+  include Helper
+
+  def setup
+    @extractor = ChupaText::Extractor.new
+  end
+
+  private
+  def fixture_path(*components)
+    super("extractor", *components)
+  end
+
+  sub_test_case("extract") do
+    private
+    def extract(data)
+      texts = []
+      @extractor.extract(data) do |extracted_data|
+        texts << extracted_data.body
+      end
+      texts
+    end
+
+    sub_test_case("input") do
+      def test_string
+        extract(fixture_path("hello.txt").to_s)
+      end
+
+      def test_uri
+        extract(URI.parse(fixture_path("hello.txt").to_s))
+      end
+
+      def test_path
+        extract(fixture_path("hello.txt"))
+      end
+    end
+
+    sub_test_case("no decomposers") do
+      def test_text
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "Hello"
+        assert_equal(["Hello"], extract(data))
+      end
+
+      def test_not_text
+        data = ChupaText::Data.new
+        data.mime_type = "application/x-javascript"
+        data.body = "alert('Hello');"
+        assert_equal([], extract(data))
+      end
+    end
+
+    sub_test_case("use decomposer") do
+      class HTMLDecomposer < ChupaText::Decomposer
+        def target?(data)
+          data.mime_type == "text/html"
+        end
+
+        def decompose(data)
+          extracted = ChupaText::Data.new
+          extracted.mime_type = "text/plain"
+          extracted.body = data.body.gsub(/<.+?>/, "")
+          yield(extracted)
+        end
+      end
+
+      def setup
+        super
+        decomposer = HTMLDecomposer.new({})
+        @extractor.add_decomposer(decomposer)
+      end
+
+      def test_decompose
+        data = ChupaText::Data.new
+        data.mime_type = "text/html"
+        data.body = "<html><body>Hello</body></html>"
+        assert_equal(["Hello"], extract(data))
+      end
+    end
+
+    sub_test_case("multi decomposed") do
+      class CopyDecomposer < ChupaText::Decomposer
+        def target?(data)
+          data["copied"].nil?
+        end
+
+        def decompose(data)
+          copied_data = data.dup
+          copied_data["copied"] = true
+          yield(copied_data.dup)
+          yield(copied_data.dup)
+        end
+      end
+
+      def setup
+        super
+        decomposer = CopyDecomposer.new({})
+        @extractor.add_decomposer(decomposer)
+      end
+
+      def test_decompose
+        data = ChupaText::Data.new
+        data.mime_type = "text/plain"
+        data.body = "Hello"
+        assert_equal(["Hello", "Hello"], extract(data))
+      end
+    end
+  end
+end

data/test/test-file-content.rb

@@ -0,0 +1,51 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestFileContent < Test::Unit::TestCase
+  def setup
+    @file = Tempfile.new(["test-file-content", ".txt"])
+  end
+
+  def test_size
+    body = "Hello"
+    assert_equal(body.bytesize, content(body).size)
+  end
+
+  def test_path
+    assert_equal(@file.path, content.path)
+  end
+
+  def test_body
+    body = "Hello"
+    assert_equal(body, content(body).body)
+  end
+
+  def test_open
+    body = "Hello"
+    assert_equal(body, content(body).open {|file| file.read})
+  end
+
+  private
+  def write(string)
+    @file.write(string)
+    @file.flush
+  end
+
+  def content(string=nil)
+    write(string) if string
+    ChupaText::FileContent.new(@file.path)
+  end
+end

data/test/test-mime-type-registry.rb

@@ -0,0 +1,48 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestMIMETypeRegistry < Test::Unit::TestCase
+  def setup
+    @registry = ChupaText::MIMETypeRegistry.new
+  end
+
+  sub_test_case("register") do
+    def test_multiple
+      @registry.register("csv", "text/csv")
+      @registry.register("txt", "text/plain")
+      assert_equal("text/csv", @registry.find("csv"))
+    end
+  end
+
+  sub_test_case("find") do
+    def setup
+      super
+      @registry.register("csv", "text/csv")
+    end
+
+    def test_nil
+      assert_nil(@registry.find(nil))
+    end
+
+    def test_nonexistent
+      assert_nil(@registry.find("txt"))
+    end
+
+    def test_existent
+      assert_equal("text/csv", @registry.find("csv"))
+    end
+  end
+end

data/test/test-text-data.rb

@@ -0,0 +1,36 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestTextData < Test::Unit::TestCase
+  def test_mime_type
+    assert_equal("text/plain", text_data("").mime_type)
+  end
+
+  def test_body
+    body = "Hello"
+    assert_equal(body, text_data(body).body)
+  end
+
+  def test_size
+    body = "Hello"
+    assert_equal(body.bytesize, text_data(body).size)
+  end
+
+  private
+  def text_data(text)
+    ChupaText::TextData.new(text)
+  end
+end

data/test/test-virtual-content.rb

@@ -0,0 +1,103 @@
+# Copyright (C) 2013  Kouhei Sutou <kou@clear-code.com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+class TestVirtualContent < Test::Unit::TestCase
+  private
+  def input(string)
+    StringIO.new(string)
+  end
+
+  def content(string, original_path=nil)
+    ChupaText::VirtualContent.new(input(string), original_path)
+  end
+
+  sub_test_case("small data") do
+    def setup
+      @body = "Hello"
+    end
+
+    def test_size
+      assert_equal(@body.bytesize, content.size)
+    end
+
+    def test_path
+      assert_equal(@body, File.read(content.path))
+    end
+
+    def test_body
+      assert_equal(@body, content.body)
+    end
+
+    def test_open
+      assert_equal(@body, content.open {|file| file.read})
+    end
+
+    private
+    def content
+      super(@body)
+    end
+  end
+
+  sub_test_case("large data") do
+    def setup
+      @body = "X" * (ChupaText::VirtualContent::BUFFER_SIZE + 1)
+    end
+
+    def test_size
+      assert_equal(@body.bytesize, content.size)
+    end
+
+    def test_path
+      assert_equal(@body, File.read(content.path))
+    end
+
+    def test_body
+      assert_equal(@body, content.body)
+    end
+
+    def test_open
+      assert_equal(@body, content.open {|file| file.read})
+    end
+
+    private
+    def content
+      super(@body)
+    end
+  end
+
+  sub_test_case("original path") do
+    def test_extension
+      assert_equal(".txt", File.extname(path("hello.txt")))
+    end
+
+    def test_extension_only
+      assert_equal(".txt", File.extname(path(".txt")))
+    end
+
+    def test_no_extension
+      assert_equal("", File.extname(path("hello")))
+    end
+
+    def test_nil
+      assert_equal("", File.extname(path(nil)))
+    end
+
+    private
+    def path(original_path)
+      content("", original_path).path
+    end
+  end
+end