iron-crawler 1.2.0 → 1.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/VERSION +1 -1
- data/iron-crawler.gemspec +2 -2
- data/lib/iron-crawler/crawler.rb +2 -2
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d53f93401d5b8804705c9cb101318ec4a6b7706d
|
|
4
|
+
data.tar.gz: 455c40d29eb07d10f3e8be38c119732a3e83d153
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d956e2a1e250432b903a2d2696cc968431eaa1f97bb4b3be07888d68b915025a09b42b9c5098f20a0e85b41d081d11d26cde68a1e371e52104d18f89a2dc73f5
|
|
7
|
+
data.tar.gz: b850df21e7c249035ae0847af58a3e4d7fa14f097423d1fee496a0d1b7e82dfa335f93721427c12ed9a6775e6c8de1a0e7025006bf596e4679279d4553d8b1e9
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
1.2.
|
|
1
|
+
1.2.1
|
data/iron-crawler.gemspec
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
|
5
|
-
# stub: iron-crawler 1.2.
|
|
5
|
+
# stub: iron-crawler 1.2.1 ruby lib
|
|
6
6
|
|
|
7
7
|
Gem::Specification.new do |s|
|
|
8
8
|
s.name = "iron-crawler"
|
|
9
|
-
s.version = "1.2.
|
|
9
|
+
s.version = "1.2.1"
|
|
10
10
|
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
|
12
12
|
s.require_paths = ["lib"]
|
data/lib/iron-crawler/crawler.rb
CHANGED
|
@@ -29,7 +29,8 @@ class Crawler < Mechanize
|
|
|
29
29
|
rescue Mechanize::ResponseCodeError
|
|
30
30
|
end
|
|
31
31
|
end
|
|
32
|
-
|
|
32
|
+
|
|
33
|
+
@mech.history.inspect
|
|
33
34
|
end
|
|
34
35
|
|
|
35
36
|
|
|
@@ -57,7 +58,6 @@ class Crawler < Mechanize
|
|
|
57
58
|
# @return [Boolean] true if we should reject URL.
|
|
58
59
|
#
|
|
59
60
|
def reject(link)
|
|
60
|
-
# TODO: are we accounting for subdomains?
|
|
61
61
|
if not_valid_uri?(link) || not_same_domain?(link) || already_spidered?(link)
|
|
62
62
|
return true
|
|
63
63
|
else
|