maxwell 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/maxwell/converter.rb +5 -1
- data/lib/maxwell/version.rb +1 -1
- data/lib/maxwell.rb +61 -11
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4dee2d7fc3af9b8e4dc64b1b6588be1735aa9f35
|
4
|
+
data.tar.gz: cc1b31de45500795d3630bad15a2b4c0dc0b6117
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ca1d52b57421703c92f3e2697f56dca22765baca6180b4e28da0669474b74d4ad580ff709c8934b3c173f44694d991e760c579f8c2469f1ad0770b839b7caf47
|
7
|
+
data.tar.gz: 7ad8eee5272c711110a5111a23b7ce450f4f5ceb5a1fe7b5943be57980d5ed1504d09f02707c9474b0c39df269db901e0e86124343ee6bf043339d6f639591b6
|
data/lib/maxwell/converter.rb
CHANGED
@@ -4,7 +4,11 @@ require 'httpclient'
|
|
4
4
|
class Maxwell
|
5
5
|
module Converter
|
6
6
|
def self.execute(url)
|
7
|
-
client = HTTPClient.new(
|
7
|
+
client = HTTPClient.new(
|
8
|
+
default_header: {
|
9
|
+
"User-Agent" => "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
|
10
|
+
}
|
11
|
+
)
|
8
12
|
|
9
13
|
html = begin
|
10
14
|
client.get_content(url)
|
data/lib/maxwell/version.rb
CHANGED
data/lib/maxwell.rb
CHANGED
@@ -1,19 +1,69 @@
|
|
1
1
|
require "maxwell/converter"
|
2
|
-
require 'csv'
|
3
2
|
|
4
3
|
class Maxwell
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
4
|
+
class Base
|
5
|
+
class << self
|
6
|
+
def attr_scrape(*attr_scrapes)
|
7
|
+
@acquirer_class = Class.new do
|
8
|
+
attr_accessor *attr_scrapes
|
9
|
+
@@attributes = attr_scrapes
|
10
|
+
|
11
|
+
def initialize(nokogiri_obj)
|
12
|
+
@html = nokogiri_obj
|
13
|
+
end
|
14
|
+
|
15
|
+
def result
|
16
|
+
@@attributes.map { |k| [k, send(k)] }.to_h
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def regist_strategy(link_selectore=nil, &strategy_blk)
|
22
|
+
@link_selectore = link_selectore
|
23
|
+
@strategy_blk = strategy_blk
|
24
|
+
end
|
25
|
+
|
26
|
+
def regist_handler(&handler_blk)
|
27
|
+
@handler_blk = handler_blk
|
15
28
|
end
|
16
29
|
end
|
30
|
+
|
31
|
+
def execute(root_url)
|
32
|
+
if self.link_selectore
|
33
|
+
html = Maxwell::Converter.execute(root_url)
|
34
|
+
html.css(self.link_selectore).each do |a|
|
35
|
+
execute_for_result a[:href]
|
36
|
+
end
|
37
|
+
else
|
38
|
+
execute_for_result root_url
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def link_selectore
|
43
|
+
self.class.instance_eval("@link_selectore")
|
44
|
+
end
|
45
|
+
|
46
|
+
def strategy_blk
|
47
|
+
self.class.instance_eval("@strategy_blk")
|
48
|
+
end
|
49
|
+
|
50
|
+
def handler_blk
|
51
|
+
self.class.instance_eval("@handler_blk")
|
52
|
+
end
|
53
|
+
|
54
|
+
def acquirer_class
|
55
|
+
self.class.instance_eval("@acquirer_class")
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
def execute_for_result(tip_url)
|
60
|
+
acquirer = acquirer_class.new(Maxwell::Converter.execute(tip_url))
|
61
|
+
acquirer.instance_eval &self.strategy_blk
|
62
|
+
|
63
|
+
acquirer.result.tap do |result|
|
64
|
+
self.handler_blk.call(result) if self.handler_blk
|
65
|
+
end
|
66
|
+
end
|
17
67
|
end
|
18
68
|
end
|
19
69
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: maxwell
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- gogotanaka
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|