macaron 2.0.1 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +4 -4
- data/bin/macaron +5 -5
- data/lib/macaron.rb +6 -6
- data/lib/macaron/page.rb +1 -3
- data/lib/macaron/spawner.rb +8 -4
- data/lib/macaron/version.rb +1 -1
- metadata +51 -3
data/README.md
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
# Macaron
|
2
2
|
Macaron is a simple web scraper implemented in ruby. It's used for service alive testing.
|
3
3
|
|
4
|
+
## Support
|
5
|
+
Ruby 1.9.x
|
6
|
+
|
4
7
|
## Install
|
5
8
|
gem install macaron
|
6
9
|
|
7
10
|
## Example
|
8
11
|
```ruby
|
9
12
|
require 'macaron'
|
10
|
-
|
11
|
-
|
12
|
-
# 1st argument is for start url, 2nd is for the depth to dig (optional)
|
13
|
-
spawner.dig("http://www.google.com/", 2)
|
13
|
+
Macaron::Spawner.new("http://www.google.com")
|
14
14
|
```
|
15
15
|
|
16
16
|
## CLI (Command Line Interface)
|
data/bin/macaron
CHANGED
@@ -1,8 +1,8 @@
|
|
1
|
-
#!/usr/bin/ruby
|
2
|
-
|
3
|
-
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
$:.unshift File.dirname(__FILE__) + "/../lib"
|
3
|
+
|
4
4
|
require 'optparse'
|
5
|
-
require 'macaron
|
5
|
+
require 'macaron'
|
6
6
|
|
7
7
|
options = {:depth => 2}
|
8
8
|
options_parser = OptionParser.new do |opts|
|
@@ -40,4 +40,4 @@ end
|
|
40
40
|
url = ARGV.first
|
41
41
|
puts "Started"
|
42
42
|
|
43
|
-
Spawner.new(url, options)
|
43
|
+
Macaron::Spawner.new(url, options)
|
data/lib/macaron.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
1
|
+
module Macaron
|
2
|
+
autoload :Version, 'macaron/version'
|
3
|
+
autoload :Spawner, 'macaron/spawner'
|
4
|
+
autoload :Crawler, 'macaron/crawler'
|
5
|
+
autoload :Page , 'macaron/page'
|
6
|
+
end
|
data/lib/macaron/page.rb
CHANGED
@@ -1,13 +1,11 @@
|
|
1
1
|
require 'open-uri'
|
2
2
|
require 'nokogiri'
|
3
|
-
require 'thread'
|
4
3
|
|
5
4
|
module Macaron
|
6
5
|
class Page
|
7
6
|
def initialize(url, bot=nil)
|
8
7
|
@url = url
|
9
8
|
@bot = bot
|
10
|
-
@@bot_lock = Mutex.new
|
11
9
|
end
|
12
10
|
|
13
11
|
def fetch
|
@@ -41,7 +39,7 @@ module Macaron
|
|
41
39
|
def content
|
42
40
|
if @bot
|
43
41
|
# only activate one browser, needs to be thread safe.
|
44
|
-
|
42
|
+
@bot.synchronize {
|
45
43
|
@bot.goto(@url)
|
46
44
|
@bot.html
|
47
45
|
}
|
data/lib/macaron/spawner.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
require 'timeout'
|
2
2
|
require 'observer'
|
3
|
+
require 'threadpool'
|
3
4
|
require 'watir-webdriver'
|
4
5
|
|
5
6
|
module Macaron
|
6
7
|
class Spawner
|
7
|
-
def initialize(url, options)
|
8
|
+
def initialize(url, options={})
|
8
9
|
@options = options
|
9
10
|
|
10
11
|
# threadpool(init workers, max workers, job timeout)
|
@@ -21,7 +22,10 @@ module Macaron
|
|
21
22
|
@awaiting_counter = 1
|
22
23
|
|
23
24
|
# bot is a webdriver
|
24
|
-
|
25
|
+
if @options[:with_watir]
|
26
|
+
bot = Watir::Browser.new
|
27
|
+
bot.extend(MonitorMixin)
|
28
|
+
end
|
25
29
|
|
26
30
|
loop do
|
27
31
|
break if @awaiting_counter == 0
|
@@ -32,7 +36,7 @@ module Macaron
|
|
32
36
|
next
|
33
37
|
end
|
34
38
|
|
35
|
-
job =
|
39
|
+
job = Crawler.new(url, bot)
|
36
40
|
job.add_observer(self)
|
37
41
|
|
38
42
|
threadpool.load(job)
|
@@ -58,7 +62,7 @@ module Macaron
|
|
58
62
|
if @options[:with_watir]
|
59
63
|
10
|
60
64
|
else
|
61
|
-
|
65
|
+
4
|
62
66
|
end
|
63
67
|
end
|
64
68
|
|
data/lib/macaron/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: macaron
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,7 +10,55 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
date: 2012-12-11 00:00:00.000000000 Z
|
13
|
-
dependencies:
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: e-threadpool
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ~>
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 1.3.0
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ~>
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 1.3.0
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: nokogiri
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ~>
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: 1.5.5
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ~>
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: 1.5.5
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: watir-webdriver
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ~>
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: 0.6.2
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ~>
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 0.6.2
|
14
62
|
description:
|
15
63
|
email: dalema22@gmail.com
|
16
64
|
executables:
|
@@ -26,7 +74,7 @@ files:
|
|
26
74
|
- lib/macaron.rb
|
27
75
|
- LICENSE
|
28
76
|
- README.md
|
29
|
-
homepage:
|
77
|
+
homepage: https://github.com/eguitarz/macaron
|
30
78
|
licenses: []
|
31
79
|
post_install_message:
|
32
80
|
rdoc_options: []
|