sledgehammer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8391efc4413b6922054b8531950a5412f3d95429
+  data.tar.gz: 1b8b0b2301cfc4ee217d92c208bc4cd88ecde5f2
+SHA512:
+  metadata.gz: 398055d333de9da4f07d110dd602c9ab66d78fd4ddbfdcd4e5ca0bdea92954da343d96f7ba5fb5c3e83b511179a0e6ddf8449ef5011cb885fa4ac4366be3a61e
+  data.tar.gz: 055c2d8e2852fa8dd605c2236b8e962373be94f2022d359f021dbf09d4e84a6aea393c65b95f1ef3b7d31f49084bde26a45bb40ceccc6a4d30b3a6d290d347fa

data/.gitignore

@@ -0,0 +1,4 @@
+.idea
+tmp/
+.DS_Store
+*/.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org"
+gemspec

data/Gemfile.lock

@@ -0,0 +1,79 @@
+PATH
+  remote: .
+  specs:
+    sledgehammer (0.1.0)
+      activerecord (~> 4.1)
+      sidekiq (~> 3.1)
+      typhoeus (~> 0.6)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.1.4)
+      activesupport (= 4.1.4)
+      builder (~> 3.1)
+    activerecord (4.1.4)
+      activemodel (= 4.1.4)
+      activesupport (= 4.1.4)
+      arel (~> 5.0.0)
+    activesupport (4.1.4)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    arel (5.0.1.20140414130214)
+    builder (3.2.2)
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    connection_pool (2.0.0)
+    diff-lcs (1.2.5)
+    ethon (0.7.1)
+      ffi (>= 1.3.0)
+    ffi (1.9.3)
+    i18n (0.6.11)
+    json (1.8.1)
+    minitest (5.4.0)
+    rake (10.3.2)
+    redis (3.1.0)
+    redis-namespace (1.5.0)
+      redis (~> 3.0, >= 3.0.4)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-sidekiq (1.0.0)
+      rspec (>= 2.0.0)
+      sidekiq (>= 2.4.0)
+    rspec-support (3.0.2)
+    sidekiq (3.2.1)
+      celluloid (>= 0.15.2)
+      connection_pool (>= 2.0.0)
+      json
+      redis (>= 3.0.6)
+      redis-namespace (>= 1.3.1)
+    sqlite3 (1.3.9)
+    thread_safe (0.3.4)
+    timers (1.1.0)
+    typhoeus (0.6.9)
+      ethon (>= 0.7.1)
+    tzinfo (1.2.1)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.5)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+  rspec-sidekiq (~> 1.0.0)
+  sledgehammer!
+  sqlite3

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Growth Republic Ltd.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,61 @@
+# Sledgehammer
+
+Sledgehammer is a gem which allows to crawl websites in search of email addresses.
+It uses Typhoeus and Sidekiq to spawn ultra-fast workers which gathers data in no-time.
+
+## Installation
+
+Include the gem in your Gemfile
+
+```ruby
+gem "sledgehammer"
+```
+
+Bundle the Gemfile
+
+```ruby
+bundle install
+```
+
+Run the install script which will create a migration file and a config file.
+
+```ruby
+bundle exec rails generate sledgehammer:install
+```
+
+Migrate your database
+
+```ruby
+bundle exec rake db:migrate
+```
+
+## Setup
+
+You should be aware of using this gem with application with sqlite3 database.
+Due to multi threaded nature of gem you will be greeted with "SQLite3::BusyException: database is locked" errors.
+PostgreSQL, MySQL or MongoDB should be just fine.
+
+## Usage
+
+Run sidekiq worker form your code:
+
+
+```ruby
+Sledgehammer::CrawlWorker.perform_async ARRAY_OF_URLS, [OPTIONS]
+```
+
+Here is sample usage:
+
+```ruby
+Sledgehammer::CrawlWorker.perform_async ['http://example.com'], { depth_limit: 3 }
+```
+
+## Contributors
+
+[d4rky-pl] (https://github.com/d4rky-pl)
+
+[rabsztok](https://github.com/rabsztok)
+
+## License
+
+Sledgehammer is Copyright © 2014 Growth Republic. It is free software, and may be redistributed under the terms specified in the LICENSE file.

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
+
+# Default directory to look in is `/specs`
+# Run with `rake spec`
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.rspec_opts = ['--color']
+end
+
+task :default => :spec

data/coverage/.last_run.json

@@ -0,0 +1,5 @@
+{
+  "result": {
+    "covered_percent": 99.31
+  }
+}

data/db/migrate/20140626075744_create_pages.rb

@@ -0,0 +1,12 @@
+class CreatePages < ActiveRecord::Migration
+  def change
+    create_table :pages do |t|
+      t.references :website, index: true
+      t.string :url
+      t.integer :depth
+      t.boolean :completed
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20140626080142_create_contacts.rb

@@ -0,0 +1,9 @@
+class CreateContacts < ActiveRecord::Migration
+  def change
+    create_table :contacts do |t|
+      t.string :email, unique: true
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20140626105612_create_websites.rb

@@ -0,0 +1,9 @@
+class CreateWebsites < ActiveRecord::Migration
+  def change
+    create_table :websites do |t|
+      t.string :hostname
+
+      t.timestamps
+    end
+  end
+end

data/db/migrate/20140704070249_create_page_contacts.rb

@@ -0,0 +1,8 @@
+class CreatePageContacts < ActiveRecord::Migration
+  def change
+    create_table :page_contacts do |t|
+      t.references :page, index: true
+      t.references :contact, index: true
+    end
+  end
+end

data/lib/generators/sledgehammer/USAGE

@@ -0,0 +1,11 @@
+Description:
+    Generates ActiveRecord migrations needed to use this gem in Rails application
+
+Example:
+    rails generate sledgehammer:install
+
+    This will create:
+        db/migrate/20140626075744_create_pages.rb
+        db/migrate/20140626080142_create_contacts.rb
+        db/migrate/20140626105612_create_websites.rb
+        db/migrate/20140704070249_create_page_contacts.rb

data/lib/generators/sledgehammer/install_generator.rb

@@ -0,0 +1,16 @@
+require 'rails/generators/base'
+
+module Sledgehammer
+  module Generators
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path('../../../../', __FILE__)
+
+      def generate_migrations
+        copy_file "db/migrate/20140626075744_create_pages.rb"
+        copy_file "db/migrate/20140626080142_create_contacts.rb"
+        copy_file "db/migrate/20140626105612_create_websites.rb"
+        copy_file "db/migrate/20140704070249_create_page_contacts.rb"
+      end
+    end
+  end
+end

data/lib/sledgehammer.rb

@@ -0,0 +1,11 @@
+require 'sidekiq'
+require 'typhoeus'
+require 'sledgehammer/version'
+require 'sledgehammer/models/contact'
+require 'sledgehammer/models/page'
+require 'sledgehammer/models/page_contact'
+require 'sledgehammer/models/website'
+require 'sledgehammer/workers/crawl_worker'
+
+module Sledgehammer
+end

data/lib/sledgehammer/models/contact.rb

@@ -0,0 +1,6 @@
+class Sledgehammer::Contact < ActiveRecord::Base
+  has_many :page_contacts
+  has_many :pages, through: :page_contacts
+
+  validates :email, uniqueness: true
+end

data/lib/sledgehammer/models/page.rb

@@ -0,0 +1,12 @@
+class Sledgehammer::Page < ActiveRecord::Base
+  belongs_to :website
+  has_many :page_contacts
+  has_many :contacts, through: :page_contacts
+  before_create :create_website!
+
+  protected
+  def create_website!
+    hostname = URI.parse(url).host
+    self.website = Sledgehammer::Website.find_or_create_by(hostname: hostname)
+  end
+end

data/lib/sledgehammer/models/page_contact.rb

@@ -0,0 +1,4 @@
+class Sledgehammer::PageContact < ActiveRecord::Base
+  belongs_to :page
+  belongs_to :contact
+end

data/lib/sledgehammer/models/website.rb

@@ -0,0 +1,4 @@
+class Sledgehammer::Website < ActiveRecord::Base
+  has_many :pages
+  has_many :contacts, through: :pages
+end

data/lib/sledgehammer/version.rb

@@ -0,0 +1,3 @@
+module Sledgehammer
+  VERSION = '0.1.0'
+end

data/lib/sledgehammer/workers/crawl_worker.rb

@@ -0,0 +1,113 @@
+class Sledgehammer::CrawlWorker
+  include ::Sidekiq::Worker
+  MAIL_REGEX = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.(?!jpg|gif|png)[A-Z0-9]+/i
+  URL_REGEX  = /<a\s+(?:[^>]*?\s+)?href="((?:http|\/)[^"]+)"/
+  DEFAULT_OPTIONS = { depth: 0, depth_limit: 1, queue: 'default' }
+
+  #
+  # Callbacks to overload in application
+  #
+  def before_queue(urls)
+    # stub
+  end
+
+  #
+  # Stops element from being added to queue if returns false
+  #
+  def on_queue(url)
+    true
+  end
+
+  def after_queue(urls)
+    # stub
+  end
+
+  def on_complete(response)
+    page = self.find_or_create_page!(response.request.url)
+    unless page.completed?
+      self.parse_emails(response, page)
+      self.parse_urls(response)
+      page.update_attributes completed: true
+    end
+  end
+
+  #
+  # There shouldn't be any need to overload methods below
+  #
+
+  def perform(urls, opts = {})
+    @options = HashWithIndifferentAccess.new(DEFAULT_OPTIONS)
+    @options.merge!(opts)
+
+    return if @options[:depth] == @options[:depth_limit]
+
+    before_queue(urls)
+    urls.each { |site| self.queue(site) }
+    run_queue
+    after_queue(urls)
+  end
+
+  def queue(url)
+    return unless self.on_queue(url) && valid_url?(url)
+
+    request = Typhoeus::Request.new(url)
+    request.on_complete { |response| self.on_complete(response) }
+
+    Typhoeus::Hydra.hydra.queue(request)
+  end
+
+  def run_queue
+    Typhoeus::Hydra.hydra.run
+  end
+
+  protected
+  def find_or_create_page!(request_url)
+    page = Sledgehammer::Page.find_by(url: request_url)
+
+    if page.blank?
+      hostname = URI.parse(request_url).host
+      website  = Sledgehammer::Website.find_or_create_by(hostname: hostname)
+      page     = Sledgehammer::Page.create!(url: request_url, depth: @options[:depth], website: website)
+    elsif page.depth < @options[:depth]
+      page.update_attributes completed: false
+    end
+    page
+  end
+
+  def parse_emails(response, page)
+    mail_list = response.body.scan MAIL_REGEX
+    mail_list.each do |email|
+      contact = Sledgehammer::Contact.find_or_create_by(email: email)
+      Sledgehammer::PageContact.find_or_create_by page: page, contact: contact
+    end
+  end
+
+  # TODO: remove url == '/' because we not always start at root page
+  def parse_urls(response)
+    request_url = response.request.url
+    request_url = "http://#{request_url}" unless request_url.match /^http/
+
+    url_list = response.body.scan(URL_REGEX).flatten.map do |url|
+      if url == request_url || !valid_url?(url)
+        nil
+      elsif url.starts_with?('/')
+        URI.join(request_url, url).to_s
+      else
+        url
+      end
+    end.compact
+
+    opts         = @options.dup
+    opts[:depth] += 1
+
+    unless opts[:depth] >= opts[:depth_limit] || url_list.empty?
+      Sidekiq::Client.push('queue' => opts[:queue],
+        'class' => self.class,
+        'args' => [url_list, opts])
+    end
+  end
+
+  def valid_url?(url)
+    !!URI.parse(url) rescue false
+  end
+end

data/sledgehammer.gemspec

@@ -0,0 +1,29 @@
+$:.push File.expand_path("../lib", __FILE__)
+require 'sledgehammer/version'
+
+Gem::Specification.new do |spec|
+  spec.name    = 'sledgehammer'
+  spec.version = Sledgehammer::VERSION
+
+  spec.authors     = ['Michał Matyas', 'Maciej Walusiak']
+  spec.email       = ['michal@higher.lv', 'rabsztok@gmail.com']
+  spec.summary     = 'Crawls websites and harvests e-mails'
+  spec.description = 'Website crawler harvesting e-mails. Uses Sidekiq and Typhoeus.'
+  spec.homepage    = 'https://github.com/growthrepublic/sledgehammer'
+  spec.license     = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec-sidekiq', '~> 1.0.0'
+  spec.add_development_dependency 'sqlite3'
+
+  spec.add_runtime_dependency 'activerecord', '~> 4.1'
+  spec.add_runtime_dependency 'typhoeus', '~> 0.6'
+  spec.add_runtime_dependency 'sidekiq', '~> 3.1'
+
+end

data/spec/fixtures/example2_com.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Example website 2: Tests Attack</title>
+</head>
+<body>
+<div class="container">This is a second example site to test following links.</div>
+<div>
+  test4@example.com
+</div>
+</body>
+</html>

data/spec/fixtures/example_com.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Example website</title>
+</head>
+<body>
+<div class="container">This is an example site for the testing purposes. It contains two external links to different test pages and three e-mails.</div>
+<div>
+  <a class="abc" href="#">This link should not be found</a>
+  <a href="/testing">This is relative link, it should be properly translated and followed</a>
+  <a href="http://www.example2.com">This is external link, it should be followed</a>
+  <a href="http:/www(broken).\com">This is corrupted external link, it should not be followed</a>
+</div>
+<div>
+  <a href="mailto:test1@example.com">This is first e-mail, inside link</a>
+  test2@example.com This is second e-mail, outside antyhing and surrounded only by whitespaces
+  john.doe@especially.long.tld.xn--clchc0ea0b2g2a9gcd This is third e-mail, with the longest possible TLD
+</div>
+</body>
+</html>

data/spec/fixtures/example_com_testing.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="utf-8">
+  <title>Example website / Testing</title>
+</head>
+<body>
+<div class="container">This is a subpage of Example website.</div>
+<div>
+  test3@example.com
+</div>
+</body>
+</html>

data/spec/models/contact_spec.rb

@@ -0,0 +1,3 @@
+RSpec.describe Sledgehammer::Contact, :type => :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

data/spec/models/page_contact.rb

@@ -0,0 +1,3 @@
+RSpec.describe Sledgehammer::PageContact, :type => :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

data/spec/models/page_spec.rb

@@ -0,0 +1,3 @@
+RSpec.describe Sledgehammer::Page, :type => :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

data/spec/models/website_spec.rb

@@ -0,0 +1,3 @@
+RSpec.describe Sledgehammer::Website, :type => :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require File.expand_path('../support/active_record_helper', __FILE__)
+require File.expand_path('../../lib/sledgehammer.rb', __FILE__)
+
+RSpec.configure do |config|
+  config.before :each do
+    Typhoeus::Expectation.clear
+  end
+end

data/spec/support/active_record_helper.rb

@@ -0,0 +1,13 @@
+require 'active_record'
+
+ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:"
+ActiveRecord::Migrator.up "db/migrate"
+
+RSpec.configure do |config|
+  config.around do |example|
+    ActiveRecord::Base.transaction do
+      example.run
+      raise ActiveRecord::Rollback
+    end
+  end
+end

data/spec/workers/crawl_worker_spec.rb

@@ -0,0 +1,33 @@
+require 'sidekiq/testing'
+
+RSpec.describe Sledgehammer::CrawlWorker, sidekiq: :fake do
+
+  before(:example) do
+    fixture_directory = File.expand_path('../../fixtures', __FILE__)
+    Typhoeus.stub('http://www.example.com').and_return Typhoeus::Response.new(code: 200, body: File.read(File.join(fixture_directory, 'example_com.html')))
+    Typhoeus.stub('http://www.example.com/testing').and_return Typhoeus::Response.new(code: 200, body: File.read(File.join(fixture_directory, 'example_com_testing.html')))
+    Typhoeus.stub('http://www.example2.com').and_return Typhoeus::Response.new(code: 200, body: File.read(File.join(fixture_directory, 'example2_com.html')))
+  end
+
+  let(:worker) { Sledgehammer::CrawlWorker.new }
+
+  describe "#perform" do
+    it "finds all e-mail addresses on the first site" do
+      worker.perform(['http://www.example.com'])
+      expect(Sledgehammer::Contact.count).to eq(3)
+    end
+
+    it "doesn't work when the depth limit is hit" do
+      worker.perform(['http://www.example.com'], { 'depth' => 2, 'depth_limit' => 2 })
+      expect(Sledgehammer::Contact.count).to eq(0)
+    end
+
+    it "crawls all pages and finds e-mails on them" do
+      Sidekiq::Testing.inline! do
+        worker.perform(['http://www.example.com'], { 'depth_limit' => 3 })
+        expect(Sledgehammer::Contact.count).to eq(5)
+        expect(Sledgehammer::Page.count).to eq(3)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,201 @@
+--- !ruby/object:Gem::Specification
+name: sledgehammer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Michał Matyas
+- Maciej Walusiak
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: typhoeus
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+description: Website crawler harvesting e-mails. Uses Sidekiq and Typhoeus.
+email:
+- michal@higher.lv
+- rabsztok@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- coverage/.last_run.json
+- db/migrate/20140626075744_create_pages.rb
+- db/migrate/20140626080142_create_contacts.rb
+- db/migrate/20140626105612_create_websites.rb
+- db/migrate/20140704070249_create_page_contacts.rb
+- lib/generators/sledgehammer/USAGE
+- lib/generators/sledgehammer/install_generator.rb
+- lib/sledgehammer.rb
+- lib/sledgehammer/models/contact.rb
+- lib/sledgehammer/models/page.rb
+- lib/sledgehammer/models/page_contact.rb
+- lib/sledgehammer/models/website.rb
+- lib/sledgehammer/version.rb
+- lib/sledgehammer/workers/crawl_worker.rb
+- sledgehammer.gemspec
+- spec/fixtures/example2_com.html
+- spec/fixtures/example_com.html
+- spec/fixtures/example_com_testing.html
+- spec/models/contact_spec.rb
+- spec/models/page_contact.rb
+- spec/models/page_spec.rb
+- spec/models/website_spec.rb
+- spec/spec_helper.rb
+- spec/support/active_record_helper.rb
+- spec/workers/crawl_worker_spec.rb
+homepage: https://github.com/growthrepublic/sledgehammer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Crawls websites and harvests e-mails
+test_files:
+- spec/fixtures/example2_com.html
+- spec/fixtures/example_com.html
+- spec/fixtures/example_com_testing.html
+- spec/models/contact_spec.rb
+- spec/models/page_contact.rb
+- spec/models/page_spec.rb
+- spec/models/website_spec.rb
+- spec/spec_helper.rb
+- spec/support/active_record_helper.rb
+- spec/workers/crawl_worker_spec.rb
+has_rdoc: