sweatpants 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: f74b60bb640fe707d0cae47be97c6b5dd611b3bb
4
+ data.tar.gz: 982cbf2bf2c4ab79fe6337885eed4a913f53ebb6
5
+ SHA512:
6
+ metadata.gz: 96454502b61b4807f32e9cb0b2b7d8acefb0a434d536418df30f5721376addb889c1cbb49fd4c93fe439e21349655c7d84c705a8c161938982a50e44582855df
7
+ data.tar.gz: a95a70d1305b3eb079c4c84336b0269b863888c118c60ab3f702b37a73f1ffa662ac99177efd091748110ce7da3c9fccf4b525d62bf4ed5500c4e8aec3dfff49
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 bentona
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,47 @@
1
+ # Sweatpants
2
+ [![Build Status](https://secure.travis-ci.org/bentona/sweatpants.png?branch=master)](http://travis-ci.org/bentona/sweatpants) [![Code Climate](https://codeclimate.com/github/bentona/sweatpants.png)](https://codeclimate.com/github/bentona/sweatpants) [![Coverage Status](https://coveralls.io/repos/bentona/sweatpants/badge.png)](https://coveralls.io/r/bentona/sweatpants)
3
+
4
+ Redis-backed elasticsearch-ruby wrapper that aggregates requests & dispatches them as bulk requests.
5
+
6
+ Inspired, in part, by https://github.com/nz/elasticmill
7
+
8
+
9
+ Create a sweatpants client.
10
+ ```
11
+ elasticsearch_options = {
12
+ host: 'localhost'
13
+ }
14
+
15
+ sweatpants_options = {
16
+ queue: SweatpantsQueue.new,
17
+ flush_frequency: 1,
18
+ actions_to_trap: [:index]
19
+ }
20
+
21
+ sweatpants = Sweatpants.new elasticsearch_options, sweatpants_options
22
+ ```
23
+
24
+ Send sweatpants a request that will be queued.
25
+ ```
26
+ sweatpants.index({
27
+ index: "matches",
28
+ type: "ExpertMatch",
29
+ id: "1234",
30
+ body: {some: 'stuff'}
31
+ })
32
+ ```
33
+
34
+ Send sweatpants a request that will be immediately executed.
35
+ ```
36
+ sweatpants.index(
37
+ {
38
+ index: "matches",
39
+ type: "ExpertMatch",
40
+ id: "5678",
41
+ body: {some: "really important stuff"}
42
+ },
43
+ {
44
+ immediate: true
45
+ }
46
+ )
47
+ ```
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
@@ -0,0 +1,23 @@
1
+ require "sweatpants/client"
2
+ require "sweatpants/queue"
3
+ require "sweatpants/timer"
4
+ require "sweatpants/queued_request"
5
+ require "sweatpants/configuration"
6
+
7
+ module Sweatpants
8
+ class << self
9
+ attr_writer :configuration
10
+ end
11
+
12
+ def self.configuration
13
+ @configuration ||= Configuration.new
14
+ end
15
+
16
+ def self.configure
17
+ yield(configuration)
18
+ end
19
+
20
+ def self.reset
21
+ @configuration = Configuration.new
22
+ end
23
+ end
@@ -0,0 +1,45 @@
1
+ require 'elasticsearch'
2
+
3
+ module Sweatpants
4
+ class Client
5
+
6
+ attr_reader :queue, :client, :actions_to_trap, :flush_frequency
7
+
8
+ def initialize es_params=nil
9
+ @client = es_params.nil? ? Sweatpants.configuration.client : Elasticsearch::Client.new(es_params)
10
+ @queue = Sweatpants.configuration.queue
11
+ @flush_frequency = Sweatpants.configuration.flush_frequency
12
+ @actions_to_trap = Sweatpants.configuration.actions_to_trap
13
+ @timer = Sweatpants::Timer.new(@flush_frequency)
14
+ @timer.on_tick { flush }
15
+ end
16
+
17
+ def flush
18
+ begin
19
+ @client.bulk @queue.dequeue
20
+ rescue Exception => e
21
+ $stderr.puts e # use a Logger, maybe @client's?
22
+ end
23
+ end
24
+
25
+ def method_missing(method_name, *args, &block)
26
+ if trap_request?(method_name, *args)
27
+ delay(method_name, *args)
28
+ else
29
+ @client.send(method_name, args[0])
30
+ end
31
+ end
32
+
33
+ private
34
+
35
+ def delay method_name, *args
36
+ request = Sweatpants::QueuedRequest.new method_name, args.first
37
+ @queue.enqueue request.to_bulk
38
+ end
39
+
40
+ def trap_request? action, *args
41
+ sweatpants_arguments = args[1] || {}
42
+ !sweatpants_arguments[:immediate] && @actions_to_trap.include?(action)
43
+ end
44
+ end
45
+ end
@@ -0,0 +1,29 @@
1
+ # mostly taken from http://brandonhilkert.com/blog/ruby-gem-configuration-patterns
2
+ module Sweatpants
3
+ class Configuration
4
+ attr_accessor :flush_frequency, :queue, :actions_to_trap, :client
5
+
6
+ def initialize
7
+ @flush_frequency = 1
8
+ @queue_type = :redis
9
+ @actions_to_trap = [:index]
10
+ @client = Elasticsearch::Client.new
11
+ @redis_config = {
12
+ host: 'localhost',
13
+ port: 6379,
14
+ list: 'sweatpants_queue',
15
+ database: 3
16
+ }
17
+ end
18
+
19
+ def queue
20
+ case @queue_type
21
+ when :redis
22
+ Sweatpants::RedisQueue.new(@redis_config)
23
+ else
24
+ Sweatpants::SimpleQueue.new
25
+ end
26
+ end
27
+
28
+ end
29
+ end
@@ -0,0 +1,60 @@
1
+ require 'redis'
2
+
3
+ module Sweatpants
4
+ class SimpleQueue < Array
5
+ def initialize
6
+ super
7
+ end
8
+
9
+ def enqueue request
10
+ true if self << request
11
+ end
12
+
13
+ def dequeue count=nil
14
+ count.nil? ? pop(self.size) : pop(count)
15
+ end
16
+
17
+ def peek
18
+ self
19
+ end
20
+ end
21
+ end
22
+
23
+ module Sweatpants
24
+ class RedisQueue
25
+ attr_reader :redis
26
+
27
+ def initialize params = {}
28
+ @list_name = params[:list]
29
+ @redis = Redis.new(host: params[:host], port: params[:port])
30
+ @redis.select params[:database]
31
+ end
32
+
33
+ def enqueue request
34
+ @redis.rpush @list_name, request.to_json
35
+ end
36
+
37
+ def dequeue count=nil
38
+ count = @redis.llen(@list_name) unless count
39
+ multi_pop count
40
+ end
41
+
42
+ def multi_pop count
43
+ items = @redis.multi {
44
+ peek count
45
+ @redis.ltrim(@list_name, count, -1)
46
+ }[0]
47
+ items
48
+ end
49
+
50
+ def flushall
51
+ @redis.flushall
52
+ end
53
+
54
+ def peek count = nil
55
+ count = @redis.llen(@list_name) unless count
56
+ @redis.lrange(@list_name, 0, count - 1)
57
+ end
58
+ end
59
+ end
60
+
@@ -0,0 +1,23 @@
1
+
2
+ module Sweatpants
3
+ class QueuedRequest
4
+
5
+ def initialize method, params
6
+ @method = method
7
+ @type = params[:type] || nil
8
+ @index = params[:index] || nil
9
+ @id = params[:id] || nil
10
+ @body = params[:body] || nil
11
+ end
12
+
13
+ def to_bulk
14
+ [bulk_header, @body].map(&:to_json).join("\n")
15
+ end
16
+
17
+ def bulk_header
18
+ header = { @method.to_sym => { _index: @index, _type: @type } }
19
+ header[:id] = @id if @id
20
+ header
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,28 @@
1
+ module Sweatpants
2
+ class Timer
3
+ attr_reader :blocks
4
+ def initialize frequency
5
+ @blocks = []
6
+ @frequency = frequency
7
+ spawn_tick_thread
8
+ end
9
+
10
+ def spawn_tick_thread
11
+ Thread.new do
12
+ while true do
13
+ sleep @frequency
14
+ call_blocks
15
+ end
16
+ end
17
+ end
18
+
19
+ def call_blocks
20
+ @blocks.each &:call
21
+ end
22
+
23
+ def on_tick &block
24
+ raise 'Timer#register requires a block' unless block
25
+ @blocks << block
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,81 @@
1
+ require 'spec_helper'
2
+ require 'redis'
3
+
4
+ module Sweatpants
5
+ describe Client do
6
+ let(:request_1){ {index: "matches", type: 'MyIndex', body: {stuff: 'some stuff'} } }
7
+
8
+ describe '#new' do
9
+ it "instantiates with a client and queue" do
10
+ sweatpants = Sweatpants::Client.new
11
+ sweatpants.instance_variable_get(:@client).should_not be_nil
12
+ sweatpants.instance_variable_get(:@queue).should be_a Sweatpants::RedisQueue
13
+ end
14
+ end
15
+
16
+ describe '#configure' do
17
+ before do
18
+ Sweatpants.configure do |config|
19
+ config.flush_frequency = 5
20
+ end
21
+ end
22
+
23
+ it "has a flush frequency of 5 seconds" do
24
+ client = Sweatpants::Client.new
25
+ expect(client.flush_frequency).to eq(5)
26
+ end
27
+ end
28
+
29
+ describe '#flush' do
30
+ before :each do
31
+ Sweatpants.reset
32
+ @es_client = double('es_client')
33
+ allow(@es_client).to receive(:bulk)
34
+ Sweatpants.configure do |config|
35
+ config.client = @es_client
36
+ config.flush_frequency = 10000
37
+ end
38
+ @client = Sweatpants::Client.new
39
+ @jobs = [{here: 'are'}, {some: 'jobs'}]
40
+ @jobs.map{|job| @client.queue.enqueue job}
41
+ @client.flush
42
+ end
43
+
44
+ it 'sends the requests with client.bulk' do
45
+ expect(@es_client).to have_received(:bulk).with(@jobs.map(&:to_json))
46
+ end
47
+
48
+ end
49
+
50
+ describe 'traps requests' do
51
+
52
+ before :each do
53
+ Sweatpants.reset
54
+ fake_client = double(search: nil, index: nil)
55
+ Sweatpants.configure do |config|
56
+ config.client = fake_client
57
+ config.flush_frequency = 10000
58
+ end
59
+ @sweatpants = Sweatpants::Client.new
60
+ @sweatpants.queue.flushall
61
+ end
62
+
63
+ it "traps an index request" do
64
+ @sweatpants.index(request_1)
65
+ expect(@sweatpants.queue.peek).to have(1).item
66
+ end
67
+
68
+ it "doesn't trap a search request" do
69
+ @sweatpants.search(request_1)
70
+ expect(@sweatpants.client).to have_received(:search).with(request_1)
71
+ expect(@sweatpants.queue.peek).to have(0).items
72
+ end
73
+
74
+ it "doesn't trap a request marked as immediate" do
75
+ @sweatpants.index(request_1, {immediate: true})
76
+ expect(@sweatpants.queue.peek).to have(0).items
77
+ expect(@sweatpants.client).to have_received(:index).with(request_1)
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,52 @@
1
+ # spec/mega_lotto/configuration_spec.rb
2
+
3
+ require "spec_helper"
4
+
5
+ # client, queue, flush_frequency, actions_to_trap, timer
6
+
7
+ module Sweatpants
8
+ describe Configuration do
9
+ describe "#flush_frequency" do
10
+ it "default value is 1 second" do
11
+ Configuration.new.flush_frequency = 1
12
+ end
13
+ end
14
+
15
+ describe "#actions_to_trap" do
16
+ it "default value is [:index]" do
17
+ Configuration.new.actions_to_trap = [:index]
18
+ end
19
+ end
20
+
21
+ describe "#flush_frequency=" do
22
+ it "can set value" do
23
+ config = Configuration.new
24
+ config.flush_frequency = 7
25
+ expect(config.flush_frequency).to eq(7)
26
+ end
27
+ end
28
+
29
+ describe "#actions_to_trap=" do
30
+ it "can set value" do
31
+ config = Configuration.new
32
+ config.actions_to_trap = [:index, :test_action]
33
+ expect(config.actions_to_trap).to eq([:index, :test_action])
34
+ end
35
+ end
36
+
37
+ describe ".reset" do
38
+ before :each do
39
+ Sweatpants.configure do |config|
40
+ config.flush_frequency = 5
41
+ end
42
+ end
43
+
44
+ it "resets the configuration" do
45
+ Sweatpants.reset
46
+ config = Sweatpants.configuration
47
+ expect(config.flush_frequency).to eq(1)
48
+ expect(config.actions_to_trap).to eq([:index])
49
+ end
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,54 @@
1
+ require 'spec_helper'
2
+ require 'fakeredis'
3
+
4
+ describe Sweatpants::RedisQueue do
5
+
6
+ let(:request_1) { ({foo: "bar", baz: "buzz"})}
7
+ let(:request_2) { ({some: "stuff"})}
8
+ let(:request_3) { ({other: "things"})}
9
+ let(:request_4) { ({fizz: 'bang'})}
10
+
11
+ describe '#initialize' do
12
+ it "works" do
13
+ Sweatpants::RedisQueue.new
14
+ end
15
+ end
16
+
17
+ describe '#enqueue' do
18
+
19
+ before :each do
20
+ Redis.new.flushall
21
+ @queue = Sweatpants::RedisQueue.new
22
+ end
23
+
24
+ it "can enqueue requests (json strings)" do
25
+ @queue.enqueue(request_1)
26
+ @queue.enqueue(request_2)
27
+ expect(@queue.peek(2)).to eq([request_1.to_json, request_2.to_json])
28
+ end
29
+ end
30
+
31
+ describe '#dequeue' do
32
+
33
+ before :each do
34
+ Redis.new.flushall
35
+ @queue = Sweatpants::RedisQueue.new
36
+ end
37
+
38
+ it "dequeues all requests by default" do
39
+ @queue.enqueue(request_1)
40
+ @queue.enqueue(request_2)
41
+ expect(@queue.dequeue).to eq([request_1.to_json, request_2.to_json])
42
+ end
43
+
44
+ it "dequeues the specified number of requests" do
45
+ @queue.enqueue(request_1)
46
+ @queue.enqueue(request_2)
47
+ @queue.enqueue(request_3)
48
+ @queue.enqueue(request_4)
49
+ expect(@queue.dequeue(1)).to eq([request_1.to_json])
50
+ expect(@queue.dequeue(2)).to eq([request_2.to_json, request_3.to_json])
51
+ end
52
+ end
53
+
54
+ end
@@ -0,0 +1,24 @@
1
+ require 'spec_helper'
2
+
3
+ describe Sweatpants::QueuedRequest do
4
+
5
+ let(:params_1) { { type: "MyType", index: "users", id: 123, body: {some: 'stuff'} } }
6
+ let(:request_1) { Sweatpants::QueuedRequest.new :index, params_1 }
7
+
8
+ describe '#initialize' do
9
+ it 'initializes with proper params' do
10
+ Sweatpants::QueuedRequest.new :index, params_1
11
+ end
12
+
13
+ it 'raises when initialized without proper params' do
14
+ expect{Sweatpants::QueuedRequest.new}.to raise_error
15
+ expect{Sweatpants::QueuedRequest.new :index }.to raise_error
16
+ end
17
+ end
18
+
19
+ describe '#to_bulk' do
20
+ it 'properly formats a bulk request' do
21
+ expect(request_1.to_bulk).to eq("{\"index\":{\"_index\":\"users\",\"_type\":\"MyType\"},\"id\":123}\n{\"some\":\"stuff\"}")
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,3 @@
1
+ require 'coveralls'
2
+ require 'sweatpants'
3
+ Coveralls.wear!
@@ -0,0 +1,9 @@
1
+ require 'spec_helper'
2
+
3
+ describe Sweatpants do
4
+ describe '::configuration' do
5
+ it 'has a configuration member' do
6
+ Sweatpants.configuration.should be_a Sweatpants::Configuration
7
+ end
8
+ end
9
+ end
@@ -0,0 +1,59 @@
1
+ require 'spec_helper'
2
+
3
+ describe Sweatpants::Timer do
4
+
5
+ before :each do
6
+ @timer = Sweatpants::Timer.new 1
7
+ end
8
+
9
+ describe '#on_tick' do
10
+
11
+ context 'block given' do
12
+
13
+ before :each do
14
+ 3.times{ @timer.on_tick {} }
15
+ end
16
+
17
+ it 'stores the given blocks' do
18
+ expect(@timer).to have(3).blocks
19
+ end
20
+ end
21
+
22
+ context 'block not given' do
23
+ it 'throws an exception when no block is given' do
24
+ expect{ @timer.on_tick }.to raise_error
25
+ end
26
+ end
27
+ end
28
+
29
+ describe '#call_blocks' do
30
+
31
+ context 'with blocks present' do
32
+
33
+ before :each do
34
+ @block_1_called = false
35
+ @block_2_called = false
36
+
37
+ @timer.on_tick do
38
+ @block_1_called = true
39
+ end
40
+
41
+ @timer.on_tick do
42
+ @block_2_called = true
43
+ end
44
+ end
45
+
46
+ it 'calls all blocks present' do
47
+ @timer.call_blocks
48
+ expect(@block_1_called).to be_true
49
+ expect(@block_2_called).to be_true
50
+ end
51
+ end
52
+
53
+ context 'with no blocks present' do
54
+ it 'should not raise when no blocks are present' do
55
+ @timer.call_blocks
56
+ end
57
+ end
58
+ end
59
+ end
metadata ADDED
@@ -0,0 +1,66 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: sweatpants
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.2
5
+ platform: ruby
6
+ authors:
7
+ - Benton Anderson
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-05-05 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: Sponsored by Originate(http://originate.com)
14
+ email: benton.anderson@gmail.com
15
+ executables: []
16
+ extensions: []
17
+ extra_rdoc_files: []
18
+ files:
19
+ - LICENSE.txt
20
+ - README.md
21
+ - Rakefile
22
+ - lib/sweatpants.rb
23
+ - lib/sweatpants/client.rb
24
+ - lib/sweatpants/configuration.rb
25
+ - lib/sweatpants/queue.rb
26
+ - lib/sweatpants/queued_request.rb
27
+ - lib/sweatpants/timer.rb
28
+ - spec/client_spec.rb
29
+ - spec/configuration_spec.rb
30
+ - spec/queue_spec.rb
31
+ - spec/request_spec.rb
32
+ - spec/spec_helper.rb
33
+ - spec/sweatpants_spec.rb
34
+ - spec/timer_spec.rb
35
+ homepage: http://github.com/bentona/sweatpants
36
+ licenses:
37
+ - MIT
38
+ metadata: {}
39
+ post_install_message:
40
+ rdoc_options: []
41
+ require_paths:
42
+ - lib
43
+ required_ruby_version: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ required_rubygems_version: !ruby/object:Gem::Requirement
49
+ requirements:
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: '0'
53
+ requirements: []
54
+ rubyforge_project:
55
+ rubygems_version: 2.2.2
56
+ signing_key:
57
+ specification_version: 4
58
+ summary: Wrapper for elasticsearch-ruby that bulks-up requests
59
+ test_files:
60
+ - spec/client_spec.rb
61
+ - spec/configuration_spec.rb
62
+ - spec/queue_spec.rb
63
+ - spec/request_spec.rb
64
+ - spec/spec_helper.rb
65
+ - spec/sweatpants_spec.rb
66
+ - spec/timer_spec.rb