krawler 1.0.4 → 1.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. data/lib/krawler/version.rb +1 -1
  2. data/lib/krawler.rb +4 -1
  3. metadata +4 -4
@@ -1,3 +1,3 @@
1
1
  module Krawler
2
- VERSION = '1.0.4'
2
+ VERSION = '1.0.5'
3
3
  end
data/lib/krawler.rb CHANGED
@@ -3,7 +3,7 @@ require 'mechanize'
3
3
  require 'timeout'
4
4
  require 'uri'
5
5
  require 'thread'
6
-
6
+ require 'pry'
7
7
  module Krawler
8
8
 
9
9
  class Base
@@ -67,6 +67,9 @@ module Krawler
67
67
  end
68
68
 
69
69
  def crawl_page(link, agent)
70
+ link = link.to_s.gsub(/per_page=36/, '')
71
+ link = link =~ /(.*)(page=\d)/ ? "#{$1}per_page=36&#{$2}" : "#{link}?per_page=36&page=1"
72
+
70
73
  @crawled_links << link
71
74
 
72
75
  begin
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: krawler
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.4
4
+ version: 1.0.5
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,11 +9,11 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-07-23 00:00:00.000000000 Z
12
+ date: 2012-07-27 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: mechanize
16
- requirement: &70288588270600 !ruby/object:Gem::Requirement
16
+ requirement: &70180957485360 !ruby/object:Gem::Requirement
17
17
  none: false
18
18
  requirements:
19
19
  - - ~>
@@ -21,7 +21,7 @@ dependencies:
21
21
  version: 2.5.0
22
22
  type: :runtime
23
23
  prerelease: false
24
- version_requirements: *70288588270600
24
+ version_requirements: *70180957485360
25
25
  description: Simple little website crawler.
26
26
  email:
27
27
  - mike@urlgonomics.com