iron-crawler 1.1.1 → 1.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/iron-crawler.gemspec +2 -2
- data/lib/iron-crawler/crawler.rb +2 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4ed6f5a0ea861fb73e6844672c16e95011ba6a30
|
|
4
|
+
data.tar.gz: 02d9807f73d11234283713eb84689d5d5fd821a2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: eecbbbee14e7bfa894fa1c31087dec6a45ec898b6ad2e05953562ed1adae65ef01432d90af5edcb79268bc36c702854bd3c9fc66cb8293c9f7620e9583b1218f
|
|
7
|
+
data.tar.gz: d20ceec13925f3c3e12ed2c0dcea3424a6d5d8cafe5d258974779e1e178928c7b258067ad3196d29d151da158a8ebe83501aea39cf6966a3eae03f60723e47db
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
1.1.
|
|
1
|
+
1.1.2
|
data/iron-crawler.gemspec
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
|
5
|
-
# stub: iron-crawler 1.1.
|
|
5
|
+
# stub: iron-crawler 1.1.2 ruby lib
|
|
6
6
|
|
|
7
7
|
Gem::Specification.new do |s|
|
|
8
8
|
s.name = "iron-crawler"
|
|
9
|
-
s.version = "1.1.
|
|
9
|
+
s.version = "1.1.2"
|
|
10
10
|
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
12
12
|
s.require_paths = ["lib"]
|
data/lib/iron-crawler/crawler.rb
CHANGED
|
@@ -5,6 +5,7 @@ class Crawler < Mechanize
|
|
|
5
5
|
|
|
6
6
|
def initialize
|
|
7
7
|
@mech = Mechanize.new
|
|
8
|
+
@mech.max_history = nil
|
|
8
9
|
end
|
|
9
10
|
|
|
10
11
|
|
|
@@ -13,8 +14,8 @@ class Crawler < Mechanize
|
|
|
13
14
|
# @return [Hash] A hash of URls crawled.
|
|
14
15
|
#
|
|
15
16
|
def spiderize(url)
|
|
16
|
-
@mech.max_history = nil
|
|
17
17
|
page = @mech.get(url)
|
|
18
|
+
|
|
18
19
|
stack = page.links
|
|
19
20
|
stack.push(*src_links(page))
|
|
20
21
|
|