macaron 2.0.1 → 2.0.2
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +4 -4
- data/bin/macaron +5 -5
- data/lib/macaron.rb +6 -6
- data/lib/macaron/page.rb +1 -3
- data/lib/macaron/spawner.rb +8 -4
- data/lib/macaron/version.rb +1 -1
- metadata +51 -3
data/README.md
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
# Macaron
|
2
2
|
Macaron is a simple web scraper implemented in ruby. It's used for service alive testing.
|
3
3
|
|
4
|
+
## Support
|
5
|
+
Ruby 1.9.x
|
6
|
+
|
4
7
|
## Install
|
5
8
|
gem install macaron
|
6
9
|
|
7
10
|
## Example
|
8
11
|
```ruby
|
9
12
|
require 'macaron'
|
10
|
-
|
11
|
-
|
12
|
-
# 1st argument is for start url, 2nd is for the depth to dig (optional)
|
13
|
-
spawner.dig("http://www.google.com/", 2)
|
13
|
+
Macaron::Spawner.new("http://www.google.com")
|
14
14
|
```
|
15
15
|
|
16
16
|
## CLI (Command Line Interface)
|
data/bin/macaron
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
#!/usr/bin/ruby
|
2
|
-
|
3
|
-
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$:.unshift File.dirname(__FILE__) + "/../lib"
|
3
|
+
|
4
4
|
require 'optparse'
|
5
|
-
require 'macaron
|
5
|
+
require 'macaron'
|
6
6
|
|
7
7
|
options = {:depth => 2}
|
8
8
|
options_parser = OptionParser.new do |opts|
|
@@ -40,4 +40,4 @@ end
|
|
40
40
|
url = ARGV.first
|
41
41
|
puts "Started"
|
42
42
|
|
43
|
-
Spawner.new(url, options)
|
43
|
+
Macaron::Spawner.new(url, options)
|
data/lib/macaron.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
1
|
+
module Macaron
|
2
|
+
autoload :Version, 'macaron/version'
|
3
|
+
autoload :Spawner, 'macaron/spawner'
|
4
|
+
autoload :Crawler, 'macaron/crawler'
|
5
|
+
autoload :Page , 'macaron/page'
|
6
|
+
end
|
data/lib/macaron/page.rb
CHANGED
@@ -1,13 +1,11 @@
|
|
1
1
|
require 'open-uri'
|
2
2
|
require 'nokogiri'
|
3
|
-
require 'thread'
|
4
3
|
|
5
4
|
module Macaron
|
6
5
|
class Page
|
7
6
|
def initialize(url, bot=nil)
|
8
7
|
@url = url
|
9
8
|
@bot = bot
|
10
|
-
@@bot_lock = Mutex.new
|
11
9
|
end
|
12
10
|
|
13
11
|
def fetch
|
@@ -41,7 +39,7 @@ module Macaron
|
|
41
39
|
def content
|
42
40
|
if @bot
|
43
41
|
# only activate one browser, needs to be thread safe.
|
44
|
-
|
42
|
+
@bot.synchronize {
|
45
43
|
@bot.goto(@url)
|
46
44
|
@bot.html
|
47
45
|
}
|
data/lib/macaron/spawner.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
require 'timeout'
|
2
2
|
require 'observer'
|
3
|
+
require 'threadpool'
|
3
4
|
require 'watir-webdriver'
|
4
5
|
|
5
6
|
module Macaron
|
6
7
|
class Spawner
|
7
|
-
def initialize(url, options)
|
8
|
+
def initialize(url, options={})
|
8
9
|
@options = options
|
9
10
|
|
10
11
|
# threadpool(init workers, max workers, job timeout)
|
@@ -21,7 +22,10 @@ module Macaron
|
|
21
22
|
@awaiting_counter = 1
|
22
23
|
|
23
24
|
# bot is a webdriver
|
24
|
-
|
25
|
+
if @options[:with_watir]
|
26
|
+
bot = Watir::Browser.new
|
27
|
+
bot.extend(MonitorMixin)
|
28
|
+
end
|
25
29
|
|
26
30
|
loop do
|
27
31
|
break if @awaiting_counter == 0
|
@@ -32,7 +36,7 @@ module Macaron
|
|
32
36
|
next
|
33
37
|
end
|
34
38
|
|
35
|
-
job =
|
39
|
+
job = Crawler.new(url, bot)
|
36
40
|
job.add_observer(self)
|
37
41
|
|
38
42
|
threadpool.load(job)
|
@@ -58,7 +62,7 @@ module Macaron
|
|
58
62
|
if @options[:with_watir]
|
59
63
|
10
|
60
64
|
else
|
61
|
-
|
65
|
+
4
|
62
66
|
end
|
63
67
|
end
|
64
68
|
|
data/lib/macaron/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: macaron
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,55 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
date: 2012-12-11 00:00:00.000000000 Z
|
13
|
-
dependencies:
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: e-threadpool
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 1.3.0
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 1.3.0
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: nokogiri
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 1.5.5
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 1.5.5
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: watir-webdriver
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 0.6.2
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.6.2
|
14
62
|
description:
|
15
63
|
email: dalema22@gmail.com
|
16
64
|
executables:
|
@@ -26,7 +74,7 @@ files:
|
|
26
74
|
- lib/macaron.rb
|
27
75
|
- LICENSE
|
28
76
|
- README.md
|
29
|
-
homepage:
|
77
|
+
homepage: https://github.com/eguitarz/macaron
|
30
78
|
licenses: []
|
31
79
|
post_install_message:
|
32
80
|
rdoc_options: []
|