fluent-plugin-droonga 0.9.0 → 0.9.9

data/test/unit/fixtures/catalog/version1.json

@@ -36,8 +36,5 @@
         }
       }
     }
-  },
-  "options": {
-    "plugins": ["for_global"]
   }
 }

data/test/unit/helper.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Droonga Project
+# Copyright (C) 2013-2014 Droonga Project
 #
 # This library is free software; you can redistribute it and/or
 # modify it under the terms of the GNU Lesser General Public
@@ -20,6 +20,7 @@ require_relative "helper/fixture"
 require_relative "helper/stub_worker"
 require_relative "helper/plugin_helper"
 require_relative "helper/watch_helper"
+require_relative "helper/distributed_search_planner_helper"
 
 class Test::Unit::TestCase
   include Sandbox

data/test/unit/helper/distributed_search_planner_helper.rb

@@ -0,0 +1,83 @@
+# Copyright (C) 2014 Droonga Project
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License version 2.1 as published by the Free Software Foundation.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+require "droonga/plugin/planner/distributed_search_planner"
+
+module DistributedSearchPlannerHelper
+  def plan(search_request)
+    planner = Droonga::DistributedSearchPlanner.new(search_request)
+    planner.plan
+  end
+
+  def messages
+    @messages ||= plan(@request)
+  end
+
+  def broadcast_message
+    messages.find do |message|
+      message["type"] == "broadcast"
+    end
+  end
+
+  def reduce_message
+    messages.find do |message|
+      message["type"] == "search_reduce"
+    end
+  end
+
+  def gather_message
+    messages.find do |message|
+      message["type"] == "search_gather"
+    end
+  end
+
+  def dependencies
+    dependencies = messages.collect do |message|
+      {
+        "type"    => message["type"],
+        "inputs"  => message["inputs"],
+        "outputs" => message["outputs"],
+      }
+    end
+    sort_dependencies(dependencies)
+  end
+
+  def sort_dependencies(dependencies)
+    dependencies.sort do |a, b|
+      a["type"] <=> b["type"]
+    end
+  end
+
+  def expected_dependencies(reduce_inputs, gather_inputs)
+    dependencies = [
+      {
+        "type"    => "search_reduce",
+        "inputs"  => reduce_inputs,
+        "outputs" => gather_inputs,
+      },
+      {
+        "type"    => "search_gather",
+        "inputs"  => gather_inputs,
+        "outputs" => nil,
+      },
+      {
+        "type"    => "broadcast",
+        "inputs"  => nil,
+        "outputs" => reduce_inputs,
+      },
+    ]
+    sort_dependencies(dependencies)
+  end
+end

data/test/unit/plugin/collector/test_basic.rb

@@ -40,32 +40,18 @@ class BasicCollectorTest < Test::Unit::TestCase
     end
   end
 
-  class GatherTest < self
+  class IOTest < self
     data(
       :simple_mapping => {
-        :expected => "result",
+        :expected => ["output_name", "result"],
         :source => "result",
-        :mapping => "string_name",
+        :mapping => "output_name",
       },
       :complex_mapping => {
-        :expected => {
-          "count" => 3,
-          "records" => [
-            create_record(0),
-            create_record(1),
-            create_record(2),
-          ],
-        },
-        :source => {
-          "count" => 3,
-          "records" => [
-            create_record(0),
-            create_record(1),
-            create_record(2),
-          ],
-        },
+        :expected => ["output_name", "result"],
+        :source => "result",
         :mapping => {
-          "output" => "search_result",
+          "output" => "output_name",
         },
       },
     )
@@ -73,7 +59,7 @@ class BasicCollectorTest < Test::Unit::TestCase
       request = {
         "task" => {
           "values" => nil,
-          "component" => {
+          "step" => {
             "body" => nil,
             "outputs" => nil,
           },
@@ -84,38 +70,23 @@ class BasicCollectorTest < Test::Unit::TestCase
         "descendants" => nil,
       }
       @plugin.process("collector_gather", request)
-      output_name = data[:mapping]
-      output_name = output_name["output"] if output_name.is_a?(Hash)
-      assert_equal([output_name, data[:expected]], @outputs.last)
+      assert_equal(data[:expected], @outputs.last)
     end
-  end
 
-  class ReduceTest < self
-    data(
-      :int => {
-        :expected => 1.5,
-        :value => 1,
-        :source => 2,
-      },
-      :float => {
-        :expected => 1.5,
-        :value => 1.0,
-        :source => 2.0,
-      },
-    )
-    def test_average(data)
+    def test_reduce
       input_name = "input_#{Time.now.to_i}"
       output_name = "output_#{Time.now.to_i}"
       request = {
         "task" => {
           "values" => {
-            output_name => data[:value],
+            output_name => [0, 1, 2],
           },
-          "component" => {
+          "step" => {
             "body" => {
               input_name => {
                 output_name => {
-                  "type" => "average",
+                  "type" => "sum",
+                  "limit" => -1,
                 },
               },
             },
@@ -123,77 +94,102 @@ class BasicCollectorTest < Test::Unit::TestCase
           },
         },
         "id" => nil,
-        "value" => data[:source],
+        "value" => [3, 4, 5],
         "name" => input_name,
         "descendants" => nil,
       }
       @plugin.process("collector_reduce", request)
       assert_equal([
                      output_name,
-                     data[:expected],
+                     [0, 1, 2, 3, 4, 5],
                    ],
                    @outputs.last)
     end
+  end
+
+  class ReduceTest < self
+    data(
+      :int => {
+        :expected => 1.5,
+        :left => 1,
+        :right => 2,
+      },
+      :float => {
+        :expected => 1.5,
+        :left => 1.0,
+        :right => 2.0,
+      },
+    )
+    def test_average(data)
+      reduced = @plugin.reduce({ "type" => "average" },
+                               data[:left],
+                               data[:right])
+      assert_equal(data[:expected], reduced)
+    end
 
     data(
       :true_and_false => {
         :expected => false,
-        :value => true,
-        :source => false,
+        :left => true,
+        :right => false,
       },
       :false_and_true => {
         :expected => false,
-        :value => false,
-        :source => true,
+        :left => false,
+        :right => true,
       },
       :both_true => {
         :expected => true,
-        :value => true,
-        :source => true,
+        :left => true,
+        :right => true,
       },
       :both_false => {
         :expected => false,
-        :value => false,
-        :source => false,
+        :left => false,
+        :right => false,
       },
     )
     def test_and(data)
-      input_name = "input_#{Time.now.to_i}"
-      output_name = "output_#{Time.now.to_i}"
-      request = {
-        "task" => {
-          "values" => {
-            output_name => data[:value],
-          },
-          "component" => {
-            "body" => {
-              input_name => {
-                output_name => {
-                  "type" => "and",
-                },
-              },
-            },
-            "outputs" => nil,
-          },
-        },
-        "id" => nil,
-        "value" => data[:source],
-        "name" => input_name,
-        "descendants" => nil,
-      }
-      @plugin.process("collector_reduce", request)
-      assert_equal([
-                     output_name,
-                     data[:expected],
-                   ],
-                   @outputs.last)
+      reduced = @plugin.reduce({ "type" => "and" },
+                               data[:left],
+                               data[:right])
+      assert_equal(data[:expected], reduced)
+    end
+
+    data(
+      :true_and_false => {
+        :expected => true,
+        :left => true,
+        :right => false,
+      },
+      :false_and_true => {
+        :expected => true,
+        :left => false,
+        :right => true,
+      },
+      :both_true => {
+        :expected => true,
+        :left => true,
+        :right => true,
+      },
+      :both_false => {
+        :expected => false,
+        :left => false,
+        :right => false,
+      },
+    )
+    def test_or(data)
+      reduced = @plugin.reduce({ "type" => "or" },
+                               data[:left],
+                               data[:right])
+      assert_equal(data[:expected], reduced)
     end
 
     data(
       :numeric_values => {
         :expected => 3,
-        :value => 1,
-        :source => 2,
+        :left => 1,
+        :right => 2,
       },
       :numeric_key_records => {
         :expected => [
@@ -204,12 +200,12 @@ class BasicCollectorTest < Test::Unit::TestCase
           create_record(5),
           create_record(6),
         ],
-        :value => [
+        :left => [
           create_record(1),
           create_record(2),
           create_record(3),
         ],
-        :source => [
+        :right => [
           create_record(4),
           create_record(5),
           create_record(6),
@@ -224,86 +220,51 @@ class BasicCollectorTest < Test::Unit::TestCase
           create_record("e"),
           create_record("f"),
         ],
-        :value => [
+        :left => [
           create_record("a"),
           create_record("b"),
           create_record("c"),
         ],
-        :source => [
+        :right => [
           create_record("d"),
           create_record("e"),
           create_record("f"),
         ],
       },
-    )
-    def test_sum(data)
-      input_name = "input_#{Time.now.to_i}"
-      output_name = "output_#{Time.now.to_i}"
-      request = {
-        "task" => {
-          "values" => {
-            output_name => data[:value],
-          },
-          "component" => {
-            "body" => {
-              input_name => {
-                output_name => {
-                  "type" => "sum",
-                  "limit" => -1,
-                },
-              },
-            },
-            "outputs" => nil,
-          },
-        },
-        "id" => nil,
-        "value" => data[:source],
-        "name" => input_name,
-        "descendants" => nil,
-      }
-      @plugin.process("collector_reduce", request)
-      assert_equal([
-                     output_name,
-                     data[:expected],
-                   ],
-                   @outputs.last)
-    end
-
-    data(
-      :numeric_values => {
+      :numeric_values_with_limit => {
         :expected => 3,
-        :value => 1,
-        :source => 2,
+        :left => 1,
+        :right => 2,
         :limit => 2,
       },
-      :numeric_key_records => {
+      :numeric_key_records_with_limit => {
         :expected => [
           create_record(1),
           create_record(2),
         ],
-        :value => [
+        :left => [
           create_record(1),
           create_record(2),
           create_record(3),
         ],
-        :source => [
+        :right => [
           create_record(4),
           create_record(5),
           create_record(6),
         ],
         :limit => 2,
       },
-      :string_key_records => {
+      :string_key_records_with_limit => {
         :expected => [
           create_record("a"),
           create_record("b"),
         ],
-        :value => [
+        :left => [
           create_record("a"),
           create_record("b"),
           create_record("c"),
         ],
-        :source => [
+        :right => [
           create_record("d"),
           create_record("e"),
           create_record("f"),
@@ -311,37 +272,12 @@ class BasicCollectorTest < Test::Unit::TestCase
         :limit => 2,
       },
     )
-    def test_sum_with_limit(data)
-      input_name = "input_#{Time.now.to_i}"
-      output_name = "output_#{Time.now.to_i}"
-      request = {
-        "task" => {
-          "values" => {
-            output_name => data[:value],
-          },
-          "component" => {
-            "body" => {
-              input_name => {
-                output_name => {
-                  "type" => "sum",
-                  "limit" => data[:limit],
-                },
-              },
-            },
-            "outputs" => nil,
-          },
-        },
-        "id" => nil,
-        "value" => data[:source],
-        "name" => input_name,
-        "descendants" => nil,
-      }
-      @plugin.process("collector_reduce", request)
-      assert_equal([
-                     output_name,
-                     data[:expected],
-                   ],
-                   @outputs.last)
+    def test_sum(data)
+      reduced = @plugin.reduce({ "type" => "sum",
+                                 "limit" => data[:limit] || -1 },
+                               data[:left],
+                               data[:right])
+      assert_equal(data[:expected], reduced)
     end
 
     data(
@@ -354,12 +290,12 @@ class BasicCollectorTest < Test::Unit::TestCase
           create_record(5),
           create_record(6),
         ],
-        :value => [
+        :left => [
           create_record(1),
           create_record(3),
           create_record(5),
         ],
-        :source => [
+        :right => [
           create_record(2),
           create_record(4),
           create_record(6),
@@ -374,83 +310,45 @@ class BasicCollectorTest < Test::Unit::TestCase
           create_record("e"),
           create_record("f"),
         ],
-        :value => [
+        :left => [
           create_record("a"),
           create_record("c"),
           create_record("e"),
         ],
-        :source => [
+        :right => [
           create_record("b"),
           create_record("d"),
           create_record("f"),
         ],
       },
-    )
-    def test_sort(data)
-      input_name = "input_#{Time.now.to_i}"
-      output_name = "output_#{Time.now.to_i}"
-      request = {
-        "task" => {
-          "values" => {
-            output_name => data[:value],
-          },
-          "component" => {
-            "body" => {
-              input_name => {
-                output_name => {
-                  "type" => "sort",
-                  "operators" => [
-                    { "column" => 0, "operator" => "<" },
-                  ],
-                  "limit" => -1,
-                },
-              },
-            },
-            "outputs" => nil,
-          },
-        },
-        "id" => nil,
-        "value" => data[:source],
-        "name" => input_name,
-        "descendants" => nil,
-      }
-      @plugin.process("collector_reduce", request)
-      assert_equal([
-                     output_name,
-                     data[:expected],
-                   ],
-                   @outputs.last)
-    end
-
-    data(
-      :numeric_key_records => {
+      :numeric_key_records_with_limit => {
         :expected => [
           create_record(1),
           create_record(2),
         ],
-        :value => [
+        :left => [
           create_record(1),
           create_record(3),
           create_record(5),
         ],
-        :source => [
+        :right => [
           create_record(2),
           create_record(4),
           create_record(6),
         ],
         :limit => 2,
       },
-      :string_key_records => {
+      :string_key_records_with_limit => {
         :expected => [
           create_record("a"),
           create_record("b"),
         ],
-        :value => [
+        :left => [
           create_record("a"),
           create_record("c"),
           create_record("e"),
         ],
-        :source => [
+        :right => [
           create_record("b"),
           create_record("d"),
           create_record("f"),
@@ -458,178 +356,137 @@ class BasicCollectorTest < Test::Unit::TestCase
         :limit => 2,
       },
     )
-    def test_sort_with_limit(data)
-      input_name = "input_#{Time.now.to_i}"
-      output_name = "output_#{Time.now.to_i}"
-      request = {
-        "task" => {
-          "values" => {
-            output_name => data[:value],
-          },
-          "component" => {
-            "body" => {
-              input_name => {
-                output_name => {
-                  "type" => "sort",
-                  "operators" => [
-                    { "column" => 0, "operator" => "<" },
-                  ],
-                  "limit" => 2,
-                },
-              },
-            },
-            "outputs" => nil,
-          },
-        },
-        "id" => nil,
-        "value" => data[:source],
-        "name" => input_name,
-        "descendants" => nil,
-      }
-      @plugin.process("collector_reduce", request)
-      assert_equal([
-                     output_name,
-                     data[:expected],
-                   ],
-                   @outputs.last)
+    def test_sort(data)
+      reduced = @plugin.reduce({ 
+                                 "type" => "sort",
+                                 "operators" => [
+                                   { "column" => 0, "operator" => "<" },
+                                 ],
+                                 "limit" => data[:limit] || -1,
+                               },
+                               data[:left],
+                               data[:right])
+      assert_equal(data[:expected], reduced)
     end
   end
 
   class MergeTest < self
     def test_grouped
-      input_name = "input_#{Time.now.to_i}"
-      output_name = "output_#{Time.now.to_i}"
-      request = {
-        "task" => {
-          "values" => {
-            output_name => [
-                [
-                  "group1",
-                  10,
-                  [
-                    create_record(1),
-                    create_record(3),
-                    create_record(5),
-                  ],
-                ],
-                [
-                  "group2",
-                  20,
-                  [
-                    create_record("a"),
-                    create_record("c"),
-                    create_record("e"),
-                  ],
-                ],
-                [
-                  "group3",
-                  30,
-                  [
-                    create_record("A"),
-                    create_record("B"),
-                    create_record("C"),
-                  ],
-                ],
-            ],
-          },
-          "component" => {
-            "body" => {
-              input_name => {
-                output_name => {
-                    "type" => "sort",
-                    "operators" => [
-                      { "column" => 1, "operator" => "<" },
-                    ],
-                    "key_column" => 0,
-                    "limit" => -1,
-                },
-              },
-            },
-            "outputs" => nil,
-          },
-        },
-        "id" => nil,
-        "value" => [
-            [
-              "group1",
-              30,
-              [
-                create_record(2),
-                create_record(4),
-                create_record(6),
-              ],
-            ],
-            [
-              "group2",
-              40,
-              [
-                create_record("b"),
-                create_record("d"),
-                create_record("f"),
-              ],
-            ],
-            [
-              "group4",
-              50,
-              [
-                create_record("D"),
-                create_record("E"),
-                create_record("F"),
-              ],
-            ],
+      expected = [
+        [
+          "group3",
+          30,
+          [
+            create_record("A"),
+            create_record("B"),
+            create_record("C"),
+          ],
         ],
-        "name" => input_name,
-        "descendants" => nil,
-      }
-      @plugin.process("collector_reduce", request)
-      assert_equal([
-                     output_name,
-                     [
-                         [
-                           "group3",
-                           30,
-                           [
-                             create_record("A"),
-                             create_record("B"),
-                             create_record("C"),
-                           ],
-                         ],
-                         [
-                           "group1",
-                           40,
-                           [
-                             create_record(2),
-                             create_record(4),
-                             create_record(6),
-                             create_record(1),
-                             create_record(3),
-                             create_record(5),
-                           ],
-                         ],
-                         [
-                           "group4",
-                           50,
-                           [
-                             create_record("D"),
-                             create_record("E"),
-                             create_record("F"),
-                           ],
-                         ],
-                         [
-                           "group2",
-                           60,
-                           [
-                             create_record("b"),
-                             create_record("d"),
-                             create_record("f"),
-                             create_record("a"),
-                             create_record("c"),
-                             create_record("e"),
-                           ],
-                         ],
-                     ],
-                   ],
-                   @outputs.last)
+        [
+          "group1",
+          40,
+          [
+            create_record(2),
+            create_record(4),
+            create_record(6),
+            create_record(1),
+            create_record(3),
+            create_record(5),
+          ],
+        ],
+        [
+          "group4",
+          50,
+          [
+            create_record("D"),
+            create_record("E"),
+            create_record("F"),
+          ],
+        ],
+        [
+          "group2",
+          60,
+          [
+            create_record("b"),
+            create_record("d"),
+            create_record("f"),
+            create_record("a"),
+            create_record("c"),
+            create_record("e"),
+          ],
+        ],
+      ]
+
+      left = [
+        [
+          "group1",
+          10,
+          [
+            create_record(1),
+            create_record(3),
+            create_record(5),
+          ],
+        ],
+        [
+          "group2",
+          20,
+          [
+            create_record("a"),
+            create_record("c"),
+            create_record("e"),
+          ],
+        ],
+        [
+          "group3",
+          30,
+          [
+            create_record("A"),
+            create_record("B"),
+            create_record("C"),
+          ],
+        ],
+      ]
+      right = [
+        [
+          "group1",
+          30,
+          [
+            create_record(2),
+            create_record(4),
+            create_record(6),
+          ],
+        ],
+        [
+          "group2",
+          40,
+          [
+            create_record("b"),
+            create_record("d"),
+            create_record("f"),
+          ],
+        ],
+        [
+          "group4",
+          50,
+          [
+            create_record("D"),
+            create_record("E"),
+            create_record("F"),
+          ],
+        ],
+      ]
+
+      reduced = @plugin.reduce({ 
+                                 "type" => "sort",
+                                 "operators" => [
+                                   { "column" => 1, "operator" => "<" },
+                                 ],
+                                 "key_column" => 0,
+                                 "limit" => -1,
+                               },
+                               left,
+                               right)
+      assert_equal(expected, reduced)
     end
   end
 end