iron-crawler 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a2dc8e70c7883e03bbc90ec200e0f80ef59d059
-  data.tar.gz: a348eb768509de52f3406d1fa23d52ddea636c62
+  metadata.gz: f6357b1e7e083df930cf9e58bee810de58908c88
+  data.tar.gz: 7f402f09862bca26320f8af26c11b97225e00f7f
 SHA512:
-  metadata.gz: b5f54a41ccbdadccecf7d522dcd81c0a13e424e3f3064f9aa9e45775e1f597b0b073d0ae6212d7cc163db3db4bd238031f13e4a86801d35eb84c6cc113f8440c
-  data.tar.gz: 522d5666aa95be4477ffb6d4922606fe6c02fe4f5afec4adacd5338ae36ea641f5bd852c68fe04188a6e590537c10e2b9b16a1ac3ee05390b24a1fde1a1516f3
+  metadata.gz: 07c7fcda49f067c91a85bfc8982592d30c53af316d4415c8551b38523915ca165576a2cdd23e1923be0ad985361e5a839a360fbacc12eb0dd005b26ad68595a9
+  data.tar.gz: 8e497245dde2de5f4e46e60362531f260808072f6b586775dfc476984a3790d2e39159b044a9c946135b5b2b0011464ad2d744af227c298f9511c5db6ac46a3c

data/README.md

@@ -14,10 +14,18 @@ From a starting URL, crawl all links on that URL and print a list of URLs visite
 
 It's easy to get started!
 
+## Install
+
+```
+gem install iron-crawler
+```
+
+## Run
+
 ```
 iron-crawler <url>
 ```
 
-will crawl any site for you.
+The above command will crawl any site for you.
 
 

data/VERSION

@@ -1 +1 @@
-1.0.0
+1.1.0

data/bin/iron-crawler

@@ -12,6 +12,5 @@ url = ARGV.first
 
 Announce.info "Crawling #{url}..."
 agent = Crawler.new
-agent.spiderize(url)
+puts agent.spiderize(url)
 
-puts agent.history

data/iron-crawler.gemspec

@@ -0,0 +1,96 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: iron-crawler 1.1.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "iron-crawler"
+  s.version = "1.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
+  s.authors = ["Ben Visser"]
+  s.date = "2016-02-08"
+  s.description = "A generic web crawler that doesn't crawl outside URLs."
+  s.email = "theodore.r.visser@gmail.com"
+  s.executables = ["iron-crawler"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.md"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "bin/iron-crawler",
+    "iron-crawler.gemspec",
+    "lib/iron-crawler.rb",
+    "lib/iron-crawler/crawler.rb",
+    "tasks/jeweler.rake",
+    "tasks/qa.rake"
+  ]
+  s.homepage = "http://github.com/noqcks/iron-crawler"
+  s.licenses = ["MIT"]
+  s.rubygems_version = "2.4.6"
+  s.summary = "A generic web crawler."
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<mechanize>, [">= 0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 2.0.1"])
+      s.add_development_dependency(%q<simplecov>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["~> 3.4.0"])
+      s.add_development_dependency(%q<reek>, ["~> 3.10.0"])
+      s.add_development_dependency(%q<roodi>, ["~> 5.0.0"])
+      s.add_development_dependency(%q<yard>, ["~> 0.8.0"])
+      s.add_development_dependency(%q<churn>, ["~> 1.0.2"])
+      s.add_development_dependency(%q<flay>, ["~> 2.7.0"])
+      s.add_development_dependency(%q<flog>, ["~> 4.3.2"])
+      s.add_development_dependency(%q<inch>, ["~> 0.7.0"])
+      s.add_development_dependency(%q<rubocop>, ["~> 0.37.0"])
+    else
+      s.add_dependency(%q<mechanize>, [">= 0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<rdoc>, ["~> 3.12"])
+      s.add_dependency(%q<bundler>, ["~> 1.0"])
+      s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+      s.add_dependency(%q<simplecov>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["~> 3.4.0"])
+      s.add_dependency(%q<reek>, ["~> 3.10.0"])
+      s.add_dependency(%q<roodi>, ["~> 5.0.0"])
+      s.add_dependency(%q<yard>, ["~> 0.8.0"])
+      s.add_dependency(%q<churn>, ["~> 1.0.2"])
+      s.add_dependency(%q<flay>, ["~> 2.7.0"])
+      s.add_dependency(%q<flog>, ["~> 4.3.2"])
+      s.add_dependency(%q<inch>, ["~> 0.7.0"])
+      s.add_dependency(%q<rubocop>, ["~> 0.37.0"])
+    end
+  else
+    s.add_dependency(%q<mechanize>, [">= 0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<rdoc>, ["~> 3.12"])
+    s.add_dependency(%q<bundler>, ["~> 1.0"])
+    s.add_dependency(%q<jeweler>, ["~> 2.0.1"])
+    s.add_dependency(%q<simplecov>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["~> 3.4.0"])
+    s.add_dependency(%q<reek>, ["~> 3.10.0"])
+    s.add_dependency(%q<roodi>, ["~> 5.0.0"])
+    s.add_dependency(%q<yard>, ["~> 0.8.0"])
+    s.add_dependency(%q<churn>, ["~> 1.0.2"])
+    s.add_dependency(%q<flay>, ["~> 2.7.0"])
+    s.add_dependency(%q<flog>, ["~> 4.3.2"])
+    s.add_dependency(%q<inch>, ["~> 0.7.0"])
+    s.add_dependency(%q<rubocop>, ["~> 0.37.0"])
+  end
+end
+

data/lib/iron-crawler/crawler.rb

@@ -14,17 +14,41 @@ class Crawler < Mechanize
   #
   def spiderize(url)
     @mech.max_history = nil
-    stack = @mech.get(url).links
+    page = @mech.get(url)
+    stack = page.links
+    stack.push(*src_links(page))
+
     while link = stack.pop
       next if reject(link)
       puts "crawling #{link.uri}"
       begin
         page = link.click
         next unless Mechanize::Page === page
+        stack.push(*src_links(page))
         stack.push(*page.links)
       rescue Mechanize::ResponseCodeError
       end
     end
+    return @mech.history
+  end
+
+
+  # Since mechanize doesn't treat src elements as links, this will
+  # return all src links from a page.
+  # @param [Mechanize::Page] A mechanize page object.
+  # @return [Array] An array of created Mechanize::Page::Link objects.
+  #
+  def src_links(page)
+    links = []
+    page.search("script").each do |element|
+      next if element.attributes['src'].nil?
+      doc = Nokogiri::HTML::Document.new
+      node = Nokogiri::XML::Node.new('foo', doc)
+      node['href'] = element.attributes['src'].value
+      link = Mechanize::Page::Link.new(node, @mech, page)
+      links.push(link)
+    end
+    return links
   end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: iron-crawler
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Ben Visser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-07 00:00:00.000000000 Z
+date: 2016-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
@@ -237,6 +237,7 @@ files:
 - Rakefile
 - VERSION
 - bin/iron-crawler
+- iron-crawler.gemspec
 - lib/iron-crawler.rb
 - lib/iron-crawler/crawler.rb
 - tasks/jeweler.rake