polipus-elasticsearch 0.0.1

data/.gitignore

@@ -0,0 +1,35 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Stefano Fontanelli
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Polipus: addons for ElasticSearch
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'polipus-elasticsearch'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install polipus-elasticsearch
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/polipus-elasticsearch/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+# coding: utf-8
+require 'bundler/gem_tasks'

data/lib/polipus-elasticsearch/index/page.rb

@@ -0,0 +1,168 @@
+require 'elasticsearch/model'
+
+ENV['POLIPUS_ELASTICSEACH_INDEX_SHARDS']    ||= '1'
+ENV['POLIPUS_ELASTICSEACH_INDEX_REPLICAS']  ||= '0'
+
+module Polipus
+  module ElasticSearch
+    class Page
+      include Elasticsearch::Model
+
+      DEFAULT_INDEX_NAME = 'polipus-pages'
+      document_type 'polipus_page'
+      index_name DEFAULT_INDEX_NAME
+
+      settings(
+        index: {
+          number_of_shards: ENV['POLIPUS_ELASTICSEACH_INDEX_SHARDS'].to_i,
+          number_of_replicas: ENV['POLIPUS_ELASTICSEACH_INDEX_REPLICAS'].to_i
+        }
+      )
+      mapping(_all: { enabled: false }) do
+        indexes(
+          :id,
+          index: :not_analyzed
+        )
+        indexes(
+          :body,
+          type: :string
+        )
+        indexes(
+          :code,
+          type: :integer
+        )
+        indexes(
+          :depth,
+          type: :integer
+        )
+        indexes(
+          :error,
+          type: :string
+        )
+        indexes(
+          :fetched,
+          type: :boolean
+        )
+        indexes(
+          :fetched_at,
+          type: :integer
+        )
+        indexes(
+          :headers,
+          type: :string
+        )
+        indexes(
+          :links,
+          type: :string
+        )
+        indexes(
+          :redirect_to,
+          type: :string
+        )
+        indexes(
+          :referer,
+          type: :string
+        )
+        indexes(
+          :response_time,
+          type: :integer
+        )
+        indexes(
+          :url,
+          type: :string
+        )
+        indexes(
+          :user_data,
+          type: :string
+        )
+      end
+
+      def self.client
+        __elasticsearch__.client
+      end
+
+      def self.count
+        client.count(index: index_name, type: document_type)['count'].to_i
+      end
+
+      def self.create_index!(name)
+        index_name(name) unless name.nil?
+        __elasticsearch__.create_index!(index: index_name)
+      end
+
+      def self.clear_index!
+        client.delete_by_query(
+          index: index_name,
+           body: { query: { match_all: {} } }
+        )
+      end
+
+      def self.delete_index!
+        client.indices.delete(index: index_name)
+      end
+
+      def self.exists?(id)
+        client.exists?(
+          index: index_name,
+          type: document_type,
+          id: id
+        )
+      end
+
+      def self.get(id)
+        return unless exists?(id)
+        client.get_source(
+          index: index_name,
+          type: document_type,
+          id: id
+        )
+      end
+
+      def self.index_exists?
+        client.indices.exists?(index: index_name)
+      end
+
+      def self.process_document(obj)
+        doc = { '_type' => document_type }
+        properties.each do |p|
+          doc[p.to_s] = obj.respond_to?(p.to_s) ? obj.send(p.to_s) : obj[p.to_s]
+        end
+        doc.reject { |_, value| value.nil? }
+      end
+
+      def self.properties
+        mapping.to_hash[document_type.to_sym][:properties].keys.map { |k| k.to_s }
+      end
+
+      def self.remove(id, refresh = false)
+        return unless exists?(id)
+        client.delete(
+          index: index_name,
+          type: document_type,
+          id: id,
+          refresh: refresh,
+          version: Time.now.to_i,
+          version_type: :external
+        )
+      end
+
+      def self.setup(client_)
+        __elasticsearch__.client = client_
+      end
+
+      def self.store(document, refresh = false)
+        document = process_document(document)
+        client.index(
+          index: index_name,
+          type: document_type,
+          id: document['id'],
+          body: document,
+          refresh: refresh,
+          version: document['fetched_at'].to_i,
+          version_type: :external
+        )
+        document['id']
+      end
+    end
+  end
+end

data/lib/polipus-elasticsearch/storage/elasticsearch_store.rb

@@ -0,0 +1,100 @@
+# encoding: UTF-8
+require 'base64'
+require 'multi_json'
+require 'polipus'
+require 'polipus-elasticsearch'
+
+module Polipus
+  module Storage
+    class ElasticSearchStore < Base
+      BINARY_FIELDS = %w(body headers user_data)
+      DEFAULT_INDEX = Polipus::ElasticSearch::Page
+
+      attr_accessor :index, :index_name, :except, :compress, :semaphore, :refresh
+
+      def initialize(client, options = {})
+        @index = options[:index] || options['index'] || DEFAULT_INDEX
+        @index_name = options[:index_name] || options['index_name']
+        @except = options[:except] || options['except'] || []
+        @compress = options[:compress] || options['compress']
+        @semaphore = Mutex.new
+        @refresh = options[:refresh] || options['refresh'] || true
+        index.setup(client)
+        index.create_index!(index_name) unless index.index_exists?
+      end
+
+      def add(page)
+        semaphore.synchronize do
+          obj = page.to_hash
+          Array(except).each { |field| obj.delete(field.to_s) }
+          BINARY_FIELDS.each do |field|
+            next if obj[field.to_s].nil? || obj[field.to_s].empty?
+            obj[field.to_s] = MultiJson.encode(obj[field.to_s]) if field.to_s == 'user_data'
+            obj[field.to_s] = Base64.encode64(obj[field.to_s])
+          end
+          obj['id'] = uuid(page)
+          obj['fetched_at'] = obj['fetched_at'].to_i
+          index.store(obj, refresh)
+        end
+      end
+
+      def clear
+        index.clear_index! if index.index_exists?
+      end
+
+      def count
+        index.count
+      end
+
+      def drop
+        index.delete_index! if index.index_exists?
+      end
+
+      def each
+        # This method is implemented only for testing purposes
+        response = index.client.search(
+          index: index_name,
+          body: {
+            query: { match_all: {} },
+            from: 0,
+            size: 25
+          }
+        )
+        response['hits']['hits'].each do |data|
+          page = load_page(data['_source'])
+          yield uuid(page), page
+        end
+      end
+
+      def exists?(page)
+        @semaphore.synchronize do
+          index.exists?(uuid(page))
+        end
+      end
+
+      def get(page)
+        @semaphore.synchronize do
+          load_page(index.get(uuid(page)))
+        end
+      end
+
+      def remove(page)
+        @semaphore.synchronize do
+          index.remove(uuid(page), refresh)
+        end
+      end
+
+      def load_page(data)
+        return nil if data.nil?
+        BINARY_FIELDS.each do |field|
+          next if data[field.to_s].nil? || data[field.to_s].empty?
+          data[field.to_s] = Base64.decode64(data[field.to_s])
+          data[field.to_s] = MultiJson.decode(data[field.to_s]) if field.to_s == 'user_data'
+        end
+        page = Page.from_hash(data)
+        page.fetched_at ||= 0
+        page
+      end
+    end
+  end
+end

data/lib/polipus-elasticsearch.rb

@@ -0,0 +1,3 @@
+# encoding: UTF-8
+require 'polipus-elasticsearch/index/page'
+require 'polipus-elasticsearch/storage/elasticsearch_store'

data/polipus-elasticsearch.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'polipus-elasticsearch'
+  spec.version       = '0.0.1'
+  spec.authors       = ['Stefano Fontanelli']
+  spec.email         = ['s.fontanelli@gmail.com']
+  spec.summary       = 'Add support for ElasticSearch in Polipus crawler'
+  spec.description   = 'Add support for ElasticSearch in Polipus crawler'
+  spec.homepage      = 'https://github.com/stefanofontanelli/polipus-elasticsearch'
+  spec.license       = 'MIT'
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(/^bin\//) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(/^(test|spec|features)\//)
+  spec.require_paths = ['lib']
+  spec.add_runtime_dependency 'elasticsearch', '~> 1.0.4'
+  spec.add_runtime_dependency 'elasticsearch-model', '~> 0.1.4'
+  spec.add_runtime_dependency 'polipus', '~> 0.3', '>= 0.3.0'
+  spec.add_development_dependency 'rake', '~> 10.3'
+  spec.add_development_dependency 'rspec', '~> 3.1.0'
+  spec.add_development_dependency 'flexmock', '~> 1.3'
+  spec.add_development_dependency 'vcr', '~> 2.9.0'
+  spec.add_development_dependency 'webmock', '~> 1.20.0'
+  spec.add_development_dependency 'coveralls'
+end

data/spec/polipus-elasticsearch/storage/elasticsearch_store_spec.rb

@@ -0,0 +1,175 @@
+# encoding: UTF-8
+require 'logger'
+require 'polipus-elasticsearch'
+require 'spec_helper'
+
+describe Polipus::Storage::ElasticSearchStore do
+  before(:each)do
+    @logger = Logger.new(STDOUT)
+    @client = Elasticsearch::Client.new(host: '127.0.0.1', logger: @logger)
+    @client.transport.logger.level = Logger::INFO
+    @index_name = 'polipus_elasticsearch_test'
+    @storage = Polipus::Storage::ElasticSearchStore.new(
+      @client,
+      index_name: @index_name,
+      refresh: true
+    )
+    @storage_without_code_and_body = Polipus::Storage::ElasticSearchStore.new(
+      @client,
+      index_name: @index_name,
+      except: ['code', 'body'],
+      refresh: true
+    )
+  end
+
+  after(:each) do
+    @storage.drop
+  end
+
+  it 'should store a page' do
+    p = page_factory 'http://www.google.com'
+    uuid = @storage.add(p)
+    expect(uuid).to eq('ed646a3334ca891fd3467db131372140')
+    p = @storage.get(p)
+    expect(p).not_to be_nil
+    expect(p.url.to_s).to eq('http://www.google.com')
+    expect(p.body).to eq('<html></html>')
+    @storage.remove(p)
+    p = @storage.get(p)
+    expect(p).to be_nil
+  end
+
+  it 'should store all the relevant data from the page' do
+    url = "http://www.duckduckgo.com"
+    referer = "http://www.actually.nowhere.com"
+    redirectto = "#{url}/your_super_awesome_results?page=42"
+    now = Time.now.to_i
+    p = page_factory(
+      url,
+      {
+        referer: referer,
+        redirect_to: redirectto,
+        fetched_at: now
+      })
+    uuid = @storage.add p
+    expect(uuid).to eq('3cd657f53c74f22c1a21b420ce3863fd')
+    p = @storage.get p
+
+    expect(p.url.to_s).to eq(url)
+    expect(p.referer.to_s).to eq(referer)
+    expect(p.redirect_to.to_s).to eq(redirectto)
+    expect(p.fetched_at).to eq(now)
+    expect(p.body).to eq('<html></html>')
+
+    # for the sake of the other tests...
+    expect(@storage.remove(p)).to be_truthy
+  end
+
+  it 'should update a page' do
+    p = page_factory 'http://www.google.com', code: 301
+    @storage.add p
+    p = @storage.get p
+    expect(p.code).to eq(301)
+  end
+
+  it 'should iterate over stored pages' do
+    p = page_factory('http://www.google.com')
+    @storage.add(p)
+    @storage.each do |k, page|
+      expect(k).to eq('ed646a3334ca891fd3467db131372140')
+      expect(page.url.to_s).to eq('http://www.google.com')
+    end
+  end
+
+  it 'should delete a page' do
+    p = page_factory 'http://www.google.com', code: 301
+    @storage.remove p
+    expect(@storage.get(p)).to be_nil
+  end
+
+  it 'should store a page removing a query string from the uuid generation' do
+    p = page_factory 'http://www.asd.com/?asd=lol'
+    p_no_query = page_factory 'http://www.asd.com/?asdas=dasda&adsda=1'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove p
+  end
+
+  it 'should store a page removing a query string from the uuid generation no ending slash' do
+    p = page_factory 'http://www.asd.com?asd=lol'
+    p_no_query = page_factory 'http://www.asd.com'
+    @storage.include_query_string_in_uuid = false
+    @storage.add p
+    expect(@storage.exists?(p_no_query)).to be_truthy
+    @storage.remove p
+  end
+
+  it 'should store a page with user data associated' do
+    p = page_factory 'http://www.user.com'
+    p.user_data.name = 'Test User Data'
+    @storage.add p
+    expect(@storage.exists?(p)).to be_truthy
+    p = @storage.get(p)
+    expect(p.user_data.name).to eq('Test User Data')
+    @storage.remove p
+  end
+
+  it 'should honor the except parameters' do
+    pag = page_factory 'http://www.user-doo.com'
+    expect(pag.code).to eq(200)
+    expect(pag.body).to eq('<html></html>')
+
+    @storage_without_code_and_body.add(pag)
+    pag = @storage_without_code_and_body.get(pag)
+
+    expect(pag.body).to be_nil
+    expect(pag.code).to eq(0)
+    @storage_without_code_and_body.remove(pag)
+  end
+
+  it 'should return false if a doc not exists' do
+    @storage.include_query_string_in_uuid = false
+    p_other  = page_factory 'http://www.asdrrrr.com'
+    expect(@storage.exists?(p_other)).to be_falsey
+    @storage.add p_other
+    expect(@storage.exists?(p_other)).to be_truthy
+    p_other  = page_factory 'http://www.asdrrrr.com?trk=asd-lol'
+    expect(@storage.exists?(p_other)).to be_truthy
+    @storage.include_query_string_in_uuid = true
+    expect(@storage.exists?(p_other)).to be_falsey
+    @storage.include_query_string_in_uuid = false
+    @storage.remove p_other
+  end
+
+  it 'should set page.fetched_at based on the id creation' do
+    p = page_factory 'http://www.user-doojo.com'
+    @storage.add p
+    expect(p.fetched_at).not_to be_nil
+    p = @storage.get p
+    expect(p.fetched_at).not_to be_nil
+    @storage.remove p
+  end
+
+  it 'should NOT set page.fetched_at if already present' do
+    p = page_factory 'http://www.user-doojooo.com'
+    p.fetched_at = 10
+    @storage.add p
+    p = @storage.get p
+    expect(p.fetched_at).to be 10
+    @storage.remove p
+  end
+
+  it 'should store two pages and the count will be two' do
+    pages = ['http://www.google.com', 'http://www.duckduckgo.com'].map do |url|
+      page_factory(url).tap do |page|
+        @storage.add(page)
+      end
+    end
+    expect(@storage.count).to be 2
+    pages.each do |page|
+      @storage.remove(page)
+    end
+    expect(@storage.count).to be 0
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,51 @@
+# Require this file using `require "spec_helper"`
+# to ensure that it is only loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+require 'digest/md5'
+require 'coveralls'
+require 'vcr'
+require 'webmock/rspec'
+
+Coveralls.wear!
+
+require 'polipus'
+
+VCR.configure do |c|
+  c.cassette_library_dir = "#{File.dirname(__FILE__)}/cassettes"
+  c.hook_into :webmock
+  c.allow_http_connections_when_no_cassette = true
+  c.ignore_localhost = true
+end
+
+RSpec.configure do |config|
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+  config.mock_with :flexmock
+  config.around(:each) do |example|
+    t = Time.now
+    print example.metadata[:full_description]
+    VCR.use_cassette(
+      Digest::MD5.hexdigest(example.metadata[:full_description]),
+      record: :all
+    ) do
+      example.run
+    end
+    puts " [#{Time.now - t}s]"
+  end
+  config.before(:each) { Polipus::SignalHandler.disable }
+end
+
+def page_factory(url, params = {})
+  params[:code] ||= 200 unless params.has_key?(:code)
+  params[:body] = '<html></html>' unless params.has_key?(:body)
+  params[:fetched_at] = Time.now.to_i
+  sleep(1)
+  Polipus::Page.new(url, params)
+end

metadata

@@ -0,0 +1,215 @@
+--- !ruby/object:Gem::Specification
+name: polipus-elasticsearch
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Stefano Fontanelli
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-07-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: elasticsearch
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.4
+- !ruby/object:Gem::Dependency
+  name: elasticsearch-model
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.4
+- !ruby/object:Gem::Dependency
+  name: polipus
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: flexmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.20.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.20.0
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Add support for ElasticSearch in Polipus crawler
+email:
+- s.fontanelli@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/polipus-elasticsearch.rb
+- lib/polipus-elasticsearch/index/page.rb
+- lib/polipus-elasticsearch/storage/elasticsearch_store.rb
+- polipus-elasticsearch.gemspec
+- spec/polipus-elasticsearch/storage/elasticsearch_store_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/stefanofontanelli/polipus-elasticsearch
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -3053519168912849089
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -3053519168912849089
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23.2
+signing_key: 
+specification_version: 3
+summary: Add support for ElasticSearch in Polipus crawler
+test_files:
+- spec/polipus-elasticsearch/storage/elasticsearch_store_spec.rb
+- spec/spec_helper.rb