embulk-input-elasticsearch 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e5dbc0a3f7f57fd192f65ba884bb66c6f3cc64e9
-  data.tar.gz: e2d7318385c5ef34c173a2a7e2a4f826ff526495
+  metadata.gz: 24e8943311e14d4c3ba8fbbdb490bed67a2974ba
+  data.tar.gz: 4f795401dfb26d3e5b58e39d4921a99eb1530445
 SHA512:
-  metadata.gz: 4c0ab03a6b629767ed2eaea29c349ed5541187667b5017806de60ed518dd4d34c840efc70bdc4f6fb5c7f126850985dc6d9c24cd0d92af6a118e63fac2fd58ff
-  data.tar.gz: 086e06172b34af6e8c88b7d8c02c03e8b2d2ee560ffab72cbe409b12fcf2aaad271029992d48d3443675b0239f5d4cd20670e5d624743b46f9bf3b8c7e54aae6
+  metadata.gz: 0b48169a99f7d055271aeecf213704bd1798d9b4f0bac14b28a56933c5ca3ce0bce48b849690abe8a133b7f2e06d4bd869c2cdb01351904fa9992bc7f429efa8
+  data.tar.gz: 1d2dafc4528c3d306f7bbbd08de973b3bdc2ec0ab4dc6ae0997aeb1adbe83e940d71e9becf0c9dcafeebf8169a76275fb9ce3bb4c0e3a6d9dd1a9d3ad4612a1d

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+cache: bundler
+rvm:
+  - jruby-9.0.5.0
+  - jruby-head
+jdk:
+  - openjdk7
+before_install:
+  - gem install bundler
+matrix:
+  allow_failures:
+    - rvm: jruby-head

data/README.md

@@ -1,4 +1,4 @@
-# Elasticsearch input plugin for Embulk
+# Elasticsearch input plugin for Embulk [![Build Status](https://secure.travis-ci.org/toyama0919/embulk-input-elasticsearch.png?branch=master)](http://travis-ci.org/toyama0919/embulk-input-elasticsearch)
 
 ## Overview
 
@@ -17,6 +17,7 @@
 - **request_timeout**: request_timeout (integer)
 - **per_size**: per_size (integer, required, default: `1000`)
 - **limit_size**: limit_size (integer, default: unlimit)
+- **num_threads**: number of threads for queries. (integer, default: 1)
 - **fields**: fields (array, required)
   - **name**: name (string, required)
   - **type**: type (string, required)
@@ -38,6 +39,7 @@ in:
   request_timeout: 60
   per_size: 1000
   limit_size: 200000
+  num_threads: 2
   fields:
     - { name: _id, type: string, metadata: true }
     - { name: _type, type: string, metadata: true }

data/Rakefile

@@ -1,10 +1,11 @@
 require "bundler/gem_tasks"
+require 'rake/testtask'
 
-task default: :build
-
-desc "Run tests"
-task :test do
-  require "test-unit"
-
-  Test::Unit::AutoRunner.run(true, './')
+desc 'Run test_unit based test'
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"].sort
+  t.verbose = true
+  t.warning = false
 end
+task :default => :test

data/embulk-input-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-elasticsearch"
-  spec.version       = "0.2.0"
+  spec.version       = "0.2.1"
   spec.authors       = ["toyama0919"]
   spec.summary       = "Elasticsearch input plugin for Embulk"
   spec.description   = "Loads records from Elasticsearch. parallel query support."

data/lib/embulk/input/elasticsearch.rb

@@ -20,7 +20,8 @@ module Embulk
           "limit_size" => config.param("limit_size", :integer, default: nil),
           "fields" => config.param("fields", :array, default: nil),
           "queries" => config.param("queries", :array),
-          "sort" => config.param("sort", :hash, default: nil)
+          "sort" => config.param("sort", :hash, default: nil),
+          "add_query_to_record" => config.param("add_query_to_record", :bool, default: false)
         }
         # TODO: want max_threads
         define_num_threads = config.param("num_threads", :integer, default: 1)
@@ -30,6 +31,9 @@ module Embulk
         task['fields'].each_with_index{ |field, i|
           columns << Column.new(i, field['name'], field['type'].to_sym)
         }
+        if task['add_query_to_record']
+          columns << Column.new(task['fields'].size, "query", :string)
+        end
 
         resume(task, columns, task['slice_queries'].size, &control)
       end
@@ -81,6 +85,7 @@ module Embulk
         @limit_size = task['limit_size']
         @fields = task['fields']
         @sort = task['sort']
+        @add_query_to_record = task['add_query_to_record']
       end
 
       def run
@@ -96,6 +101,9 @@ module Embulk
 
             results = get_sources(search(@index_type, query, size, now_results_size, @routing, @fields, @sort), @fields)
             results.each do |record|
+              if @add_query_to_record
+                record << query
+              end
               page_builder.add(record)
             end
             break if last_query?(next_results_size ,total_count)

data/test/helper.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+
+# require 'embulk/java/bootstrap'
+require 'embulk'
+Embulk.setup
+Embulk.logger = Embulk::Logger.new('/dev/null')
+
+APP_ROOT = File.expand_path('../', __dir__)
+TEST_ROOT = File.expand_path(File.dirname(__FILE__))

data/test/test_transaction.rb

@@ -0,0 +1,78 @@
+require_relative './helper'
+require 'embulk/input/elasticsearch'
+require 'yaml'
+
+Elasticsearch = Embulk::Input::Elasticsearch
+
+module Embulk
+  class Input::Elasticsearch
+    class TestTransaction < Test::Unit::TestCase
+      def control
+        Proc.new {|task| task_reports = [] }
+      end
+
+      sub_test_case "get_slice_from_num_threads" do
+        def test_normal
+          slice = Elasticsearch.get_slice_from_num_threads((1..10).to_a, 5)
+          assert_equal slice.size, 5
+          assert_equal slice.first.size, 2
+        end
+
+        def test_normal_same
+          slice = Elasticsearch.get_slice_from_num_threads((1..3).to_a, 3)
+          assert_equal slice.size, 3
+          assert_equal slice.first.size, 1
+        end
+
+        def test_num_threads_over_array_size
+          slice = Elasticsearch.get_slice_from_num_threads((1..3).to_a, 10)
+          assert_equal slice.size, 3
+          assert_equal slice.first.size, 1
+        end
+
+        def test_rest
+          slice = Elasticsearch.get_slice_from_num_threads((1..20).to_a, 8)
+          assert_equal slice.size, 7
+          assert_equal slice.first.size, 3
+        end
+      end
+
+      sub_test_case "transaction" do
+        def test_normal
+          yaml = YAML.load(%(
+              nodes:
+                - {host: localhost, port: 9200}
+              queries:
+                - 'title: 製函機'
+              index: crawl
+              index_type: m_corporation_page
+              request_timeout: 60
+              per_size: 1000
+              limit_size: 2000
+              num_threads: 20
+              fields:
+                - { name: title, type: string }
+            )
+          )
+          config = DataSource.new(yaml)
+          Elasticsearch.transaction(config, &control)
+        end
+
+        def test_minimum
+          yaml = YAML.load(%(
+              nodes:
+                - {host: localhost, port: 9200}
+              queries:
+                - 'title: 製函機'
+              index: crawl
+              fields:
+                - { name: title, type: string }
+            )
+          )
+          config = DataSource.new(yaml)
+          Elasticsearch.transaction(config, &control)
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - toyama0919
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-15 00:00:00.000000000 Z
+date: 2016-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -103,13 +103,15 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".ruby-version"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - embulk-input-elasticsearch.gemspec
 - lib/embulk/input/elasticsearch.rb
-- test/embulk/input/test_elasticsearch.rb
+- test/helper.rb
+- test/test_transaction.rb
 homepage: https://github.com/toyama0919/embulk-input-elasticsearch
 licenses:
 - MIT
@@ -135,4 +137,5 @@ signing_key:
 specification_version: 4
 summary: Elasticsearch input plugin for Embulk
 test_files:
-- test/embulk/input/test_elasticsearch.rb
+- test/helper.rb
+- test/test_transaction.rb

data/test/embulk/input/test_elasticsearch.rb

@@ -1,11 +0,0 @@
-require "embulk/command/embulk_run"
-require "embulk"
-Embulk.setup
-
-require "embulk/input/elasticsearch"
-module Embulk
-  module Input
-    class ElasticsearchInputPluginTest < Test::Unit::TestCase
-    end
-  end
-end