prefetcher 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bda803d355067711c1c81142eaaf80928a4bd2ac
+  data.tar.gz: 55f9e91d9f0f4f7d2d50d47d6454d799a031bbad
+SHA512:
+  metadata.gz: 11f7a3809331c5895df81bbc9882712f573bca07c47da1af20e96b06ad511dbde9155a9cfc70d3609aaa698be3ed87135f50ed61ea27438b76f18b8ac0406e59
+  data.tar.gz: 0d4081babac5f0ed943fed7944ff4c02ba436030ce6d49b3b3cd53b5f112ba733b6a5297d0bd3e074d2182a450c7f08291c8a79944834252f35471c55eab0ba8

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.travis.yml

@@ -0,0 +1,18 @@
+language: ruby
+
+addons:
+  code_climate:
+    repo_token: c17041d1834d6f6ff02afdcc14e2537fb8255218240f72de22db6951c4efdd82
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.1
+  - 2.1.2
+  - rbx-2.2.6
+  - jruby
+
+matrix:
+  allow_failures: 
+    - rvm: rbx-2.2.6
+    - rvm: jruby

data/Gemfile

@@ -0,0 +1,15 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fetcher.gemspec
+gemspec
+
+gem 'mock_redis'
+
+gem 'rspec'
+
+gem 'ffaker'
+gem 'fakeweb'
+gem 'pry'
+gem 'pry-nav'
+
+gem "codeclimate-test-reporter", group: :test, require: nil

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Alex Rozumey
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,62 @@
+# Prefetcher
+[![Build Status](https://travis-ci.org/brain-geek/prefetcher.svg?branch=master)](https://travis-ci.org/brain-geek/prefetcher)
+[![Code Climate](https://codeclimate.com/github/brain-geek/prefetcher/badges/gpa.svg)](https://codeclimate.com/github/brain-geek/prefetcher)
+[![Test Coverage](https://codeclimate.com/github/brain-geek/prefetcher/badges/coverage.svg)](https://codeclimate.com/github/brain-geek/prefetcher)
+
+This gem provides a simple-to-use interface to work with frequently requested http requests from your api. It gets request response from memory, if possible. But also this means you have to update this cache from time to time (using [whenever](https://github.com/javan/whenever), for example). Any kind of non-200 responses will not be memoized, so you can be always sure that you don't use broken data. Redis is used to store data. [RDoc](http://rdoc.info/github/brain-geek/prefetcher/master/frames)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'prefetcher'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install prefetcher
+
+You can also override redis connection details (if not using default localhost:6379 ):
+
+    Prefetcher.redis_connection = Redis.new(:host => "10.0.1.1", :port => 6380, :db => 15)
+
+See [redis gem documentation](https://github.com/redis/redis-rb#getting-started) for more options when creating redis connection.
+    
+## Usage
+
+### Using cached requests
+
+After installing project you can request any URL:
+    
+    Prefetcher::HttpPrefetcher.new('http://www.reddit.com/r/ruby').get
+
+Calling #get any number of times will return data from cache.
+
+### Force get
+
+If you want to force request (and save the response), you can call #fetch:
+
+    Prefetcher::HttpPrefetcher.new('http://www.reddit.com/r/ruby').fetch
+
+This will cause actual http request.
+
+### Updating cache
+
+Calling manualy. You can call *Prefetcher.update_all* to fetch all URLs right now.
+
+You can also automate this call using [whenever](https://github.com/javan/whenever). Just add this code to your schedule.rb .
+
+    every 30.minutes do
+      runner "Prefetcher.update_all"
+    end%
+
+## Contributing
+
+1. Fork it ( https://github.com/brain-geek/prefetcher/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new
+
+task :default => :spec

data/lib/prefetcher.rb

@@ -0,0 +1,26 @@
+require "redis"
+require "active_support/core_ext/string/output_safety"
+require "active_support/core_ext/hash/except"
+
+require "prefetcher/http_fetcher"
+require "prefetcher/http_memoizer"
+
+require "prefetcher/version"
+
+module Prefetcher
+
+  # Updates all memoized requests
+  def self.update_all(options = {})
+    HttpMemoizer.new(options).get_list.each do |url|
+      HttpFetcher.new(options.merge(url: url)).fetch
+    end
+  end
+
+  def self.redis_connection
+    @redis_connection ||= Redis.new
+  end
+
+  def self.redis_connection=(conn)
+    @redis_connection = conn
+  end
+end

data/lib/prefetcher/http_fetcher.rb

@@ -0,0 +1,47 @@
+module Prefetcher
+  class HttpFetcher
+    attr_reader :url, :redis_connection, :memoizer
+
+    def initialize(params = {})
+      @url = params.fetch(:url)
+      @redis_connection = params.fetch(:redis_connection, Prefetcher.redis_connection)
+      @memoizer = params.fetch(:memoizer, HttpMemoizer.new(redis_connection: @redis_connection))
+    end
+
+    # Makes request to given URL
+    def fetch
+      uri = URI(URI.encode(self.url))
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      request = Net::HTTP::Get.new(uri.request_uri)
+
+      response = http.request(request)
+
+      if response.code == "200"
+        memoize(response.body)
+        response.body
+      else
+        ''
+      end
+    end
+
+    # Returns cached version if availible. If not cached - makes request using #fetch .
+    def get
+      (get_from_memory || fetch).html_safe.force_encoding('utf-8')
+    end
+
+    protected
+    def cache_key
+      "cached-url-#{url}"
+    end
+
+    def get_from_memory
+      @redis_connection.get(cache_key)
+    end
+
+    def memoize(response)
+      memoizer.push(url)
+      @redis_connection.set(cache_key, response)
+    end
+  end
+end

data/lib/prefetcher/http_memoizer.rb

@@ -0,0 +1,28 @@
+module Prefetcher
+  class HttpMemoizer
+    attr_reader :redis_connection
+
+    def initialize(params = {})
+      @redis_connection = params.fetch(:redis_connection, Prefetcher.redis_connection)
+    end
+
+    # Add URL to memoized list
+    def push(url)
+      redis.sadd(cache_key, url)
+    end
+
+    # Get all memoized URLs
+    def get_list
+      redis.smembers cache_key
+    end
+
+    protected 
+    def cache_key
+      "urls-list"
+    end
+
+    def redis
+      Prefetcher.redis_connection
+    end
+  end
+end

data/lib/prefetcher/version.rb

@@ -0,0 +1,3 @@
+module Prefetcher
+  VERSION = "0.0.2"
+end

data/prefetcher.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'prefetcher/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "prefetcher"
+  spec.version       = Prefetcher::VERSION
+  spec.authors       = ["Alex Rozumiy"]
+  spec.email         = ["brain-geek@yandex.ua"]
+  spec.summary       = %q{Prefetching/caching tool for external requests}
+  spec.description   = %q{This gem provides possibility to have 'fresh' prefetched result of external http request all the time}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "redis"
+  spec.add_dependency "activesupport"
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+end

data/spec/prefetcher/http_fetcher_spec.rb

@@ -0,0 +1,135 @@
+require 'spec_helper'
+
+describe Prefetcher::HttpFetcher do
+  let(:redis_connection) { MockRedis.new }
+
+  let(:default_params) { Hash[url: url, redis_connection: redis_connection] }
+  let(:params) { default_params }
+  let(:object) { described_class.new(params) }
+
+  let(:url) { Faker::Internet.http_url }
+  let(:request_body) { Faker::HTMLIpsum.ul_short }
+
+  describe "#initialize" do
+    it "fails if no url given" do
+      expect { described_class.new(default_params.except(:url)).to raise_error }
+    end
+
+    it "uses default connection if not set explicitly" do
+      Prefetcher.redis_connection = (connection = double('Redis'))
+      object = described_class.new(default_params.except(:redis_connection))
+
+      expect(object.redis_connection).to be connection
+      expect(object.memoizer.redis_connection).to be connection
+    end
+  end
+
+  describe "#fetch" do
+    subject { object.fetch }
+
+    describe "200 response" do
+      before { FakeWeb.register_uri(:get, url, :body => request_body) }
+
+      it "gets data from real world http query" do
+        expect(subject).to eq request_body
+      end
+
+      it "makes http requests the same number of times as called" do
+        FakeWeb.register_uri(:get, url,
+                     [{:body => "1", :status => ["200", "OK"]},
+                      {:body => "2", :status => ["200", "OK"]},
+                      {:body => "3", :status => ["200", "OK"]}])
+
+        expect(object.fetch).to eq "1"
+        expect(object.fetch).to eq "2"
+        expect(object.fetch).to eq "3"
+        expect(object.fetch).to eq "3" # fakeweb feature - when no more responces, it uses last
+      end
+
+      describe "saves output to redis" do
+        it "to corresponding key" do
+          expect(redis_connection.get("cached-url-#{url}")).to be_nil
+
+          subject
+
+          expect(redis_connection.get("cached-url-#{url}")).to eq request_body
+        end
+      end
+
+      it "also pushes given url to url_memoizer" do
+        expect(object.memoizer).to receive(:push).with(url)
+        subject
+      end
+    end
+
+    describe "500 response" do
+      before do
+        FakeWeb.register_uri(:get, url, body: request_body, status: ["500", "Internal Server Error"])
+      end
+
+      it "returns empty string" do
+        expect(subject).to be_empty
+      end
+
+      it "does not write this to cache" do
+        subject
+        expect(redis_connection.get("cached-url-#{url}")).to be_nil
+      end
+    end
+
+    describe "404 response" do
+      before do
+        FakeWeb.register_uri(:get, url, body: request_body, status: ["404", "Not Found"])
+      end
+
+      it "returns empty string" do
+        expect(subject).to be_empty
+      end
+
+      it "does not write this to cache" do
+        subject
+        expect(redis_connection.get("cached-url-#{url}")).to be_nil
+      end
+    end
+  end
+
+  describe "#get" do
+    let(:request_body) { Faker::HTMLIpsum.ul_short.force_encoding('US-ASCII') } # real-world case with encoding
+    subject { object.get }
+
+    describe "when no data already present in cache" do
+      before { expect(object).to receive(:fetch).and_return(request_body) }
+
+      it "calls #fetch" do
+        expect(subject).to eq request_body
+      end
+
+      it "returns html safe value" do
+        expect(subject).to be_html_safe
+      end
+
+      it "returns utf-8 encoded string" do
+        expect(subject.encoding.to_s).to eq "UTF-8"
+      end
+    end
+
+    describe "when there is data in cache" do
+      before do
+        expect(object).to_not receive(:fetch)
+        redis_connection.set("cached-url-#{url}", request_body)
+      end
+
+      it "returns data only from cache" do
+        expect(subject).to eq request_body
+      end
+
+      it "returns html safe value" do
+        expect(subject).to be_html_safe
+      end
+
+      it "returns utf-8 encoded string" do
+        expect(subject.encoding.to_s).to eq "UTF-8"
+      end
+    end
+  end
+end

data/spec/prefetcher/http_memoizer_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe Prefetcher::HttpMemoizer do
+  describe "#get_list" do
+    let(:memoizer) { described_class.new }
+    subject { memoizer.get_list }
+
+    it "returns empty array by default" do
+      expect(subject).to be_empty
+    end
+
+    it "returns the url once, even if it was memorized multiple times" do
+      url = Faker::Internet.http_url
+
+      memoizer.push url
+      memoizer.push url
+
+      expect(subject).to eq [url]
+    end
+
+    it "returns all given urls" do
+      urls = 3.times.map { Faker::Internet.http_url }
+
+      urls.each do |url|
+        memoizer.push url
+      end
+
+      expect(subject.sort).to eq urls.sort
+    end
+  end
+end

data/spec/prefetcher_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe Prefetcher do
+  describe "redis connection accessors" do
+    it "returns previously set connection" do
+      described_class.redis_connection = (conn = double("Redis"))
+
+      expect(described_class.redis_connection).to eq conn
+    end
+
+    it "uses Redis.new without arguments by default" do
+      expect(Redis).to receive(:new).with(no_args).and_return(conn = double("Redis"))
+
+      expect(described_class.redis_connection).to_not eq conn
+
+      described_class.redis_connection = nil
+
+      expect(described_class.redis_connection).to eq conn
+    end
+  end
+
+  describe "::update_all" do
+    let(:url) { Faker::Internet.http_url }
+
+    it "should update data in HttpFetcher fetched URLs" do
+        FakeWeb.register_uri(:get, url,
+                     [{:body => "1", :status => ["200", "OK"]},
+                      {:body => "2", :status => ["200", "OK"]}])
+
+        obj = Prefetcher::HttpFetcher.new(url: url)
+
+        expect(obj.get).to eq "1"
+        expect(obj.get).to eq "1"
+
+        described_class.update_all
+
+        expect(obj.get).to eq "2"
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require "codeclimate-test-reporter"
+CodeClimate::TestReporter.start
+
+require 'prefetcher'
+
+Bundler.require
+
+# Disabling old rspec should syntax
+RSpec.configure do |config|
+  config.raise_errors_for_deprecations!
+  
+  config.before(:each) do
+    Prefetcher.redis_connection = MockRedis.new
+  end
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: prefetcher
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Alex Rozumiy
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem provides possibility to have 'fresh' prefetched result of external
+  http request all the time
+email:
+- brain-geek@yandex.ua
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/prefetcher.rb
+- lib/prefetcher/http_fetcher.rb
+- lib/prefetcher/http_memoizer.rb
+- lib/prefetcher/version.rb
+- prefetcher.gemspec
+- spec/prefetcher/http_fetcher_spec.rb
+- spec/prefetcher/http_memoizer_spec.rb
+- spec/prefetcher_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.6
+signing_key: 
+specification_version: 4
+summary: Prefetching/caching tool for external requests
+test_files:
+- spec/prefetcher/http_fetcher_spec.rb
+- spec/prefetcher/http_memoizer_spec.rb
+- spec/prefetcher_spec.rb
+- spec/spec_helper.rb