metacrunch-elasticsearch 4.0.0 → 4.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7fcdad44003a195fca6541d8c9156523dcde02ad
-  data.tar.gz: 964c93ec7c91b54ea76bf338c5997920546f3840
+  metadata.gz: 54f127cb7af8ebc5f9b7246d0241994a499fad0d
+  data.tar.gz: f7fca4e9e60e2d2c6951147bb57f177dc21350e2
 SHA512:
-  metadata.gz: 29bdc05cebb697ee8fbf6a434d2ad3861cf205c0a35324a77f1c1ef145599c72032f4adf2e5cc6e2cb395ac5bc744493942f87ddeb6f0310b342ac8285c04074
-  data.tar.gz: 60afb62d71c16d17404b5fd2b1bf735bce2011398a9bc014f0cb2c0e398df13d668c1601b9caa3e6beda1ed3f0d946da4bd37c0a59548159410182e86d3faf12
+  metadata.gz: 198b86f27ce9a318ea975201f95f837be6b56408d6c302d08ba29ba9ec3469868ff367959bcab670d147180a2cba9dae89a0c594ec0551ea807be938513aaa48
+  data.tar.gz: ef8828c5ff6eac6002edaaa0beca1cc7eb29125ca5251c53eab3a35ca21b25910dfd5a63afc3d2af70332f28a7142e8e5aa0b303720c932cd757d826031cfbd6

data/.circleci/config.yml

@@ -0,0 +1,46 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      - image: circleci/ruby:2.4.1-node-browsers
+      - image: docker.elastic.co/elasticsearch/elasticsearch:5.6.3
+        environment:
+          - "discovery.type=single-node"
+          - "xpack.security.enabled=false"
+          - "http.host=0.0.0.0"
+          - "transport.host=127.0.0.1"
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      - run:
+          name: Waiting for elasticsearch to start up (30 sec.)
+          command: sleep 30
+
+      - run:
+          name: Install dependencies
+          command: bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - run:
+          name: Install CodeClimate test coverage reporter
+          command: |
+            curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+            chmod +x ./cc-test-reporter
+            ./cc-test-reporter before-build
+
+      - run:
+          name: Run tests
+          command: |
+            mkdir /tmp/test-results
+            bundle exec rspec --format progress --format RspecJunitFormatter --out /tmp/test-results/rspec.xml
+
+      - run:
+          name: Upload test coverage report to CodeClimate
+          command: ./cc-test-reporter after-build --exit-code $?
+

data/Gemfile

@@ -5,7 +5,6 @@ gemspec
 group :development do
   gem "bundler", ">= 1.15"
   gem "rake",    ">= 12.1"
-  gem "rspec",   ">= 3.5.0", "< 4.0.0"
 
   if !ENV["CI"]
     gem "pry-byebug", ">= 3.5.0"
@@ -13,5 +12,8 @@ group :development do
 end
 
 group :test do
-  gem "simplecov", ">= 0.15.0"
+  gem "faker",                 ">= 1.8.4"
+  gem "rspec",                 ">= 3.5.0", "< 4.0.0"
+  gem "rspec_junit_formatter", ">= 0.3.0"
+  gem "simplecov",             ">= 0.15.0"
 end

data/License.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2013 René Sprotte, Michael Sievers
+Copyright (c) 2013 René Sprotte
 
 MIT License
 

data/Readme.md

@@ -0,0 +1,104 @@
+metacrunch-elasticsearch
+========================
+
+[![Gem Version](https://badge.fury.io/rb/metacrunch-elasticsearch.svg)](http://badge.fury.io/rb/metacrunch-elasticsearch)
+[![Code Climate](https://codeclimate.com/github/ubpb/metacrunch-elasticsearch/badges/gpa.svg)](https://codeclimate.com/github/ubpb/metacrunch-elasticsearch)
+[![Test Coverage](https://codeclimate.com/github/ubpb/metacrunch-elasticsearch/badges/coverage.svg)](https://codeclimate.com/github/ubpb/metacrunch-elasticsearch/coverage)
+[![CircleCI](https://circleci.com/gh/ubpb/metacrunch-elasticsearch.svg?style=svg)](https://circleci.com/gh/ubpb/metacrunch-elasticsearch)
+
+This is the official [Elasticsearch](https://www.elastic.co) package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch).
+
+Installation
+------------
+
+Include the gem in your `Gemfile`
+
+```ruby
+gem "metacrunch-elasticsearch", "~> 4.0.1"
+```
+
+and run `$ bundle install` to install it.
+
+Or install it manually
+
+```
+$ gem install metacrunch-elasticsearch
+```
+
+Usage
+-----
+
+*Note: For working examples on how to use this package check out our [demo repository](https://github.com/ubpb/metacrunch-demo).*
+
+### `Metacrunch::Elasticsearch::Source`
+
+This class provides a metacrunch `source` implementation that can be used to read data from Elasticsearch into a metacrunch job.
+
+```ruby
+# my_job.metacrunch
+
+# Create a Elasticsearch connection 
+elasticsearch = Elasticsearch::Client.new(...)
+
+# Set the source
+source Metacrunch::Elasticsearch::Source.new(elasticsearch, OPTIONS)
+```
+
+**Options**
+
+* `:search_options`: A hash with search options (including your query) as described [here](https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/search.rb). We have set some meaningful defaults though: `size: 100`, `scroll: 1m`, `sort: ["_doc"]`. Depending on your use-case it may be needed to modify `:size` and `:scroll` for optimal performance.
+* `:total_hits_callback`: You can set a `Proc` that gets called with the total number of hits your query will match. Use can use this callback to setup a progress bar for example. Defaults to `nil`.
+
+
+### `Metacrunch::Elasticsearch::Destination`
+
+This class provides a metacrunch `destination` implementation that can be used to write data from a metacrunch job to Elasticsearch.
+
+The data that gets passed to the destination, must be in a proper format. You can use a transformation to transform your data before it reaches the destination.
+
+As `Metacrunch::Elasticsearch::Destination` utilizes the Elasticsearch bulk API, the expected format must match one of the available options for the `body`parameter described [here](https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb). Please note that you can use the bulk API not only to index records. You can update or delete records as well.
+
+```ruby
+# my_job.metacrunch
+
+# Transform data into a format that the destination can understand.
+# In this example `data` is some hash.
+transformation ->(data) do
+  {
+    index: {
+      _index: "my-index",
+      _type: "my-type",
+      _id: data.delete(:id),
+      data: data
+    }
+  }
+end
+```
+
+It is not efficient to call Elasticsearch for every single record. Therefore we can use a transformation with a buffer, to create bulks of records. In this example we use a buffer size of 10. In production environments and depending on your data, larger buffers may be useful.
+
+```ruby
+# my_job.metacrunch
+
+transformation ->(data) { data }, buffer: 10
+```
+
+If these transformations are in place you can now use the `Metacrunch::Elasticsearch::Destination` class as a destination.
+
+```ruby
+# my_job.metacrunch
+
+# Write data into elasticsearch
+destination Metacrunch::Elasticsearch::Destination.new(elasticsearch [, OPTIONS])
+```
+
+**Options**
+
+* `:raise_on_result_errors`: If set to `true` an error is raised if one of the bulk operations return with an error. Defaults to `false`.
+* `:result_callback`: You can set a `Proc` that gets called with the result from the bulk operation. Defaults to `nil`.
+* `:bulk_options`: A hash of options for the Eleasticsearch bulk API as described [here](https://github.com/elastic/elasticsearch-ruby/blob/master/elasticsearch-api/lib/elasticsearch/api/actions/bulk.rb). Setting `body` here will be ignored. Defaults to `{}`.
+
+License
+-------
+
+metacrunch-elasticsearch is available at [github](https://github.com/ubpb/metacrunch-elasticsearch) under [MIT license](https://github.com/ubpb/metacrunch-elasticsearch/blob/master/License.txt).

data/lib/metacrunch/elasticsearch.rb

@@ -1,3 +1,5 @@
+require "active_support"
+require "active_support/core_ext"
 require "elasticsearch"
 
 module Metacrunch

data/lib/metacrunch/elasticsearch/version.rb

@@ -1,5 +1,5 @@
 module Metacrunch
   module Elasticsearch
-    VERSION = "4.0.0"
+    VERSION = "4.0.1"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.1
 platform: ruby
 authors:
 - René Sprotte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-27 00:00:00.000000000 Z
+date: 2017-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -44,11 +44,13 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
 - ".gitignore"
 - ".rspec"
 - Gemfile
 - License.txt
 - Rakefile
+- Readme.md
 - bin/console
 - lib/metacrunch/elasticsearch.rb
 - lib/metacrunch/elasticsearch/destination.rb