sweatpants 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f74b60bb640fe707d0cae47be97c6b5dd611b3bb
+  data.tar.gz: 982cbf2bf2c4ab79fe6337885eed4a913f53ebb6
+SHA512:
+  metadata.gz: 96454502b61b4807f32e9cb0b2b7d8acefb0a434d536418df30f5721376addb889c1cbb49fd4c93fe439e21349655c7d84c705a8c161938982a50e44582855df
+  data.tar.gz: a95a70d1305b3eb079c4c84336b0269b863888c118c60ab3f702b37a73f1ffa662ac99177efd091748110ce7da3c9fccf4b525d62bf4ed5500c4e8aec3dfff49

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 bentona
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,47 @@
+# Sweatpants 
+[![Build Status](https://secure.travis-ci.org/bentona/sweatpants.png?branch=master)](http://travis-ci.org/bentona/sweatpants) [![Code Climate](https://codeclimate.com/github/bentona/sweatpants.png)](https://codeclimate.com/github/bentona/sweatpants) [![Coverage Status](https://coveralls.io/repos/bentona/sweatpants/badge.png)](https://coveralls.io/r/bentona/sweatpants)
+
+Redis-backed elasticsearch-ruby wrapper that aggregates requests & dispatches them as bulk requests. 
+
+Inspired, in part, by https://github.com/nz/elasticmill
+
+
+Create a sweatpants client.
+```
+elasticsearch_options = {
+	host: 'localhost'
+}
+
+sweatpants_options = {
+	queue: SweatpantsQueue.new,
+	flush_frequency: 1,
+	actions_to_trap: [:index]
+}
+
+sweatpants = Sweatpants.new elasticsearch_options, sweatpants_options
+```
+
+Send sweatpants a request that will be queued.
+```
+sweatpants.index({
+	index: "matches", 
+	type: "ExpertMatch", 
+	id: "1234",
+	body: {some: 'stuff'}
+})
+```
+
+Send sweatpants a request that will be immediately executed.
+```
+sweatpants.index(
+	{
+		index: "matches",
+		type: "ExpertMatch",
+		id: "5678",
+		body: {some: "really important stuff"}
+	}, 
+	{
+		immediate: true
+	}
+)
+```

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/sweatpants.rb

@@ -0,0 +1,23 @@
+require "sweatpants/client"
+require "sweatpants/queue"
+require "sweatpants/timer"
+require "sweatpants/queued_request"
+require "sweatpants/configuration"
+
+module Sweatpants
+  class << self
+    attr_writer :configuration
+  end
+
+  def self.configuration
+    @configuration ||= Configuration.new
+  end
+
+  def self.configure
+    yield(configuration)
+  end
+
+  def self.reset
+    @configuration = Configuration.new
+  end
+end

data/lib/sweatpants/client.rb

@@ -0,0 +1,45 @@
+require 'elasticsearch'
+
+module Sweatpants
+  class Client
+
+    attr_reader :queue, :client, :actions_to_trap, :flush_frequency
+
+    def initialize es_params=nil
+      @client = es_params.nil? ? Sweatpants.configuration.client : Elasticsearch::Client.new(es_params)
+      @queue = Sweatpants.configuration.queue
+      @flush_frequency = Sweatpants.configuration.flush_frequency
+      @actions_to_trap = Sweatpants.configuration.actions_to_trap
+      @timer = Sweatpants::Timer.new(@flush_frequency)
+      @timer.on_tick { flush }
+    end
+
+    def flush
+      begin
+        @client.bulk @queue.dequeue
+      rescue Exception => e
+        $stderr.puts e # use a Logger, maybe @client's?
+      end
+    end
+
+    def method_missing(method_name, *args, &block)
+      if trap_request?(method_name, *args)
+        delay(method_name, *args)
+      else
+        @client.send(method_name, args[0])
+      end
+    end
+
+    private
+
+    def delay method_name, *args
+      request = Sweatpants::QueuedRequest.new method_name, args.first
+      @queue.enqueue request.to_bulk
+    end
+
+    def trap_request? action, *args
+      sweatpants_arguments = args[1] || {}
+      !sweatpants_arguments[:immediate] && @actions_to_trap.include?(action) 
+    end
+  end
+end

data/lib/sweatpants/configuration.rb

@@ -0,0 +1,29 @@
+# mostly taken from http://brandonhilkert.com/blog/ruby-gem-configuration-patterns
+module Sweatpants
+  class Configuration
+    attr_accessor :flush_frequency, :queue, :actions_to_trap, :client
+
+    def initialize
+      @flush_frequency = 1
+      @queue_type = :redis
+      @actions_to_trap = [:index]
+      @client = Elasticsearch::Client.new
+      @redis_config = {
+        host: 'localhost',
+        port: 6379,
+        list: 'sweatpants_queue',
+        database: 3
+      }
+    end
+
+    def queue
+      case @queue_type
+      when :redis
+        Sweatpants::RedisQueue.new(@redis_config)
+      else
+        Sweatpants::SimpleQueue.new
+      end
+    end
+
+  end
+end

data/lib/sweatpants/queue.rb

@@ -0,0 +1,60 @@
+require 'redis'
+
+module Sweatpants
+  class SimpleQueue < Array
+    def initialize
+      super
+    end
+
+    def enqueue request
+      true if self << request
+    end
+
+    def dequeue count=nil
+      count.nil? ? pop(self.size) : pop(count)
+    end
+
+    def peek
+      self
+    end
+  end
+end
+
+module Sweatpants
+  class RedisQueue
+    attr_reader :redis
+    
+    def initialize params = {}
+      @list_name = params[:list]
+      @redis = Redis.new(host: params[:host], port: params[:port])
+      @redis.select params[:database]
+    end
+
+    def enqueue request
+      @redis.rpush @list_name, request.to_json
+    end
+
+    def dequeue count=nil
+      count = @redis.llen(@list_name) unless count
+      multi_pop count
+    end
+
+    def multi_pop count
+      items = @redis.multi {
+        peek count
+        @redis.ltrim(@list_name, count, -1)
+      }[0]
+      items
+    end
+
+    def flushall
+      @redis.flushall
+    end
+
+    def peek count = nil
+      count = @redis.llen(@list_name) unless count
+      @redis.lrange(@list_name, 0, count - 1)
+    end
+  end
+end
+

data/lib/sweatpants/queued_request.rb

@@ -0,0 +1,23 @@
+
+module Sweatpants
+  class QueuedRequest
+
+    def initialize method, params
+      @method = method
+      @type = params[:type] || nil
+      @index = params[:index] || nil
+      @id = params[:id] || nil
+      @body = params[:body] || nil
+    end
+
+    def to_bulk
+      [bulk_header, @body].map(&:to_json).join("\n")
+    end
+
+    def bulk_header
+      header = { @method.to_sym => { _index: @index, _type: @type } }
+      header[:id] = @id if @id
+      header
+    end
+  end
+end

data/lib/sweatpants/timer.rb

@@ -0,0 +1,28 @@
+module Sweatpants
+  class Timer
+    attr_reader :blocks
+    def initialize frequency
+      @blocks = []
+      @frequency = frequency
+      spawn_tick_thread
+    end
+
+    def spawn_tick_thread
+      Thread.new do 
+        while true do
+          sleep @frequency
+          call_blocks
+        end
+      end
+    end
+
+    def call_blocks
+      @blocks.each &:call
+    end
+
+    def on_tick &block
+      raise 'Timer#register requires a block' unless block
+      @blocks << block 
+    end
+  end
+end

data/spec/client_spec.rb

@@ -0,0 +1,81 @@
+require 'spec_helper'
+require 'redis'
+
+module Sweatpants
+  describe Client do
+    let(:request_1){ {index: "matches", type: 'MyIndex', body: {stuff: 'some stuff'} } }
+
+    describe '#new' do
+      it "instantiates with a client and queue" do
+        sweatpants = Sweatpants::Client.new
+        sweatpants.instance_variable_get(:@client).should_not be_nil
+        sweatpants.instance_variable_get(:@queue).should be_a Sweatpants::RedisQueue
+      end
+    end
+
+    describe '#configure' do
+      before do
+        Sweatpants.configure do |config|
+          config.flush_frequency = 5
+        end
+      end
+
+      it "has a flush frequency of 5 seconds" do
+        client = Sweatpants::Client.new
+        expect(client.flush_frequency).to eq(5) 
+      end
+    end
+
+    describe '#flush' do
+      before :each do
+        Sweatpants.reset
+        @es_client = double('es_client')
+        allow(@es_client).to receive(:bulk)
+        Sweatpants.configure do |config|
+          config.client = @es_client
+          config.flush_frequency = 10000
+        end
+        @client = Sweatpants::Client.new
+        @jobs = [{here: 'are'}, {some: 'jobs'}]
+        @jobs.map{|job| @client.queue.enqueue job}
+        @client.flush
+      end
+
+      it 'sends the requests with client.bulk' do
+        expect(@es_client).to have_received(:bulk).with(@jobs.map(&:to_json))
+      end
+
+    end
+
+    describe 'traps requests' do
+      
+      before :each do
+        Sweatpants.reset
+        fake_client = double(search: nil, index: nil)
+        Sweatpants.configure do |config|
+          config.client = fake_client
+          config.flush_frequency = 10000
+        end
+        @sweatpants = Sweatpants::Client.new
+        @sweatpants.queue.flushall
+      end
+
+      it "traps an index request" do
+        @sweatpants.index(request_1)
+        expect(@sweatpants.queue.peek).to have(1).item
+      end
+
+      it "doesn't trap a search request" do
+        @sweatpants.search(request_1)
+        expect(@sweatpants.client).to have_received(:search).with(request_1)
+        expect(@sweatpants.queue.peek).to have(0).items
+      end
+
+      it "doesn't trap a request marked as immediate" do
+        @sweatpants.index(request_1, {immediate: true})
+        expect(@sweatpants.queue.peek).to have(0).items
+        expect(@sweatpants.client).to have_received(:index).with(request_1)
+      end
+    end
+  end
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,52 @@
+# spec/mega_lotto/configuration_spec.rb
+
+require "spec_helper"
+
+# client, queue, flush_frequency, actions_to_trap, timer
+
+module Sweatpants
+  describe Configuration do
+    describe "#flush_frequency" do
+      it "default value is 1 second" do
+        Configuration.new.flush_frequency = 1
+      end
+    end
+
+    describe "#actions_to_trap" do
+      it "default value is [:index]" do
+        Configuration.new.actions_to_trap = [:index]
+      end
+    end
+
+    describe "#flush_frequency=" do
+      it "can set value" do
+        config = Configuration.new
+        config.flush_frequency = 7
+        expect(config.flush_frequency).to eq(7)
+      end
+    end
+
+    describe "#actions_to_trap=" do
+      it "can set value" do
+        config = Configuration.new
+        config.actions_to_trap = [:index, :test_action]
+        expect(config.actions_to_trap).to eq([:index, :test_action])
+      end
+    end
+
+    describe ".reset" do
+      before :each do
+        Sweatpants.configure do |config|
+          config.flush_frequency = 5
+        end
+      end
+
+      it "resets the configuration" do
+        Sweatpants.reset
+        config = Sweatpants.configuration
+        expect(config.flush_frequency).to eq(1)
+        expect(config.actions_to_trap).to eq([:index])
+      end
+    end
+  end
+end

data/spec/queue_spec.rb

@@ -0,0 +1,54 @@
+require 'spec_helper'
+require 'fakeredis'
+
+describe Sweatpants::RedisQueue do
+
+  let(:request_1) { ({foo: "bar", baz: "buzz"})}
+  let(:request_2) { ({some: "stuff"})}
+  let(:request_3) { ({other: "things"})}
+  let(:request_4) { ({fizz: 'bang'})}
+
+  describe '#initialize' do
+    it "works" do
+      Sweatpants::RedisQueue.new
+    end
+  end
+  
+  describe '#enqueue' do
+    
+    before :each do
+      Redis.new.flushall
+      @queue = Sweatpants::RedisQueue.new
+    end
+
+    it "can enqueue requests (json strings)" do
+      @queue.enqueue(request_1)
+      @queue.enqueue(request_2)
+      expect(@queue.peek(2)).to eq([request_1.to_json, request_2.to_json])
+    end
+  end
+
+  describe '#dequeue' do
+
+    before :each do
+      Redis.new.flushall
+      @queue = Sweatpants::RedisQueue.new
+    end
+    
+    it "dequeues all requests by default" do
+      @queue.enqueue(request_1)
+      @queue.enqueue(request_2)
+      expect(@queue.dequeue).to eq([request_1.to_json, request_2.to_json])
+    end
+
+    it "dequeues the specified number of requests" do
+      @queue.enqueue(request_1)
+      @queue.enqueue(request_2)
+      @queue.enqueue(request_3)
+      @queue.enqueue(request_4)
+      expect(@queue.dequeue(1)).to eq([request_1.to_json])
+      expect(@queue.dequeue(2)).to eq([request_2.to_json, request_3.to_json])
+    end
+  end
+
+end

data/spec/request_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+
+describe Sweatpants::QueuedRequest do
+
+  let(:params_1) { { type: "MyType", index: "users", id: 123, body: {some: 'stuff'} } }
+  let(:request_1) { Sweatpants::QueuedRequest.new :index, params_1 }
+
+  describe '#initialize' do
+    it 'initializes with proper params' do
+      Sweatpants::QueuedRequest.new :index, params_1
+    end
+
+    it 'raises when initialized without proper params' do
+      expect{Sweatpants::QueuedRequest.new}.to raise_error
+      expect{Sweatpants::QueuedRequest.new :index }.to raise_error
+    end
+  end
+
+  describe '#to_bulk' do
+    it 'properly formats a bulk request' do
+      expect(request_1.to_bulk).to eq("{\"index\":{\"_index\":\"users\",\"_type\":\"MyType\"},\"id\":123}\n{\"some\":\"stuff\"}")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+require 'coveralls'
+require 'sweatpants'
+Coveralls.wear!

data/spec/sweatpants_spec.rb

@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe Sweatpants do
+  describe '::configuration' do
+    it 'has a configuration member' do
+      Sweatpants.configuration.should be_a Sweatpants::Configuration
+    end
+  end
+end

data/spec/timer_spec.rb

@@ -0,0 +1,59 @@
+require 'spec_helper'
+
+describe Sweatpants::Timer do
+
+  before :each do
+    @timer = Sweatpants::Timer.new 1
+  end
+
+  describe '#on_tick' do
+
+    context 'block given' do
+
+      before :each do
+        3.times{ @timer.on_tick {} }
+      end
+
+      it 'stores the given blocks' do
+        expect(@timer).to have(3).blocks
+      end
+    end
+
+    context 'block not given' do
+      it 'throws an exception when no block is given' do
+        expect{ @timer.on_tick }.to raise_error
+      end
+    end
+  end
+
+  describe '#call_blocks' do
+
+    context 'with blocks present' do
+     
+      before :each do
+        @block_1_called = false
+        @block_2_called = false
+        
+        @timer.on_tick do
+          @block_1_called = true
+        end
+        
+        @timer.on_tick do
+          @block_2_called = true
+        end
+      end
+
+      it 'calls all blocks present' do  
+        @timer.call_blocks
+        expect(@block_1_called).to be_true
+        expect(@block_2_called).to be_true
+      end
+    end
+
+    context 'with no blocks present' do
+      it 'should not raise when no blocks are present' do
+        @timer.call_blocks
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,66 @@
+--- !ruby/object:Gem::Specification
+name: sweatpants
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Benton Anderson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-05-05 00:00:00.000000000 Z
+dependencies: []
+description: Sponsored by Originate(http://originate.com)
+email: benton.anderson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sweatpants.rb
+- lib/sweatpants/client.rb
+- lib/sweatpants/configuration.rb
+- lib/sweatpants/queue.rb
+- lib/sweatpants/queued_request.rb
+- lib/sweatpants/timer.rb
+- spec/client_spec.rb
+- spec/configuration_spec.rb
+- spec/queue_spec.rb
+- spec/request_spec.rb
+- spec/spec_helper.rb
+- spec/sweatpants_spec.rb
+- spec/timer_spec.rb
+homepage: http://github.com/bentona/sweatpants
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Wrapper for elasticsearch-ruby that bulks-up requests
+test_files:
+- spec/client_spec.rb
+- spec/configuration_spec.rb
+- spec/queue_spec.rb
+- spec/request_spec.rb
+- spec/spec_helper.rb
+- spec/sweatpants_spec.rb
+- spec/timer_spec.rb