points-scraper 0.0.3 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/bin/points +59 -0
- data/lib/points-scraper/anamileage.rb +45 -0
- data/lib/points-scraper/default.rb +24 -0
- data/lib/points-scraper/rakuten.rb +41 -0
- data/lib/points-scraper/tpoint.rb +38 -0
- data/lib/points-scraper/version.rb +1 -1
- data/misc/_points +4 -0
- data/points-scraper.gemspec +1 -0
- metadata +24 -12
- data/bin/points-anamileage +0 -47
- data/bin/points-rakuten +0 -43
- data/bin/points-tpoint +0 -40
data/bin/points
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
#! /usr/bin/env ruby
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
#
|
4
|
+
# points.rb: getting mileages/points from shopping web sites.
|
5
|
+
#
|
6
|
+
# Copyright (C) 2012 by TADA Tadashi <t@tdtds.jp>
|
7
|
+
# Distributed under GPL.
|
8
|
+
#
|
9
|
+
|
10
|
+
require 'rubygems'
|
11
|
+
require 'points-scraper'
|
12
|
+
require 'thor'
|
13
|
+
require 'pit'
|
14
|
+
|
15
|
+
module Points::Scraper
|
16
|
+
class App < Thor
|
17
|
+
desc 'tpoint', 'getting current T-POINT from T-SITE.'
|
18
|
+
def tpoint
|
19
|
+
require 'points-scraper/tpoint'
|
20
|
+
puts TPoint.new( *auth( 'tsite' ) ).start
|
21
|
+
end
|
22
|
+
|
23
|
+
desc 'ana', 'getting current ANA Mileage.'
|
24
|
+
def ana
|
25
|
+
require 'points-scraper/anamileage'
|
26
|
+
puts AnaMileage.new( *auth( 'ana' ) ).start
|
27
|
+
end
|
28
|
+
|
29
|
+
desc 'rakuten', 'getting current Rakuten Points.'
|
30
|
+
def rakuten
|
31
|
+
require 'points-scraper/rakuten'
|
32
|
+
puts Rakuten.new( *auth( 'rakuten' ) ).start
|
33
|
+
end
|
34
|
+
|
35
|
+
desc 'help', 'show command help.'
|
36
|
+
def help
|
37
|
+
super
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
def auth( key )
|
42
|
+
login = Pit::get( key, :require => {
|
43
|
+
'user' => "your ID of #{key}.",
|
44
|
+
'pass' => "your Password of #{key}."
|
45
|
+
} )
|
46
|
+
[login['user'], login['pass']]
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
Points::Scraper::App.start
|
52
|
+
|
53
|
+
# Local Variables:
|
54
|
+
# mode: ruby
|
55
|
+
# indent-tabs-mode: t
|
56
|
+
# tab-width: 3
|
57
|
+
# ruby-indent-level: 3
|
58
|
+
# End:
|
59
|
+
# vim: ts=3
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# amileage.rb: mileage by scraping www.ana.co.jp. for PointsScraper
|
4
|
+
#
|
5
|
+
# Copyright (C) 2012 by MATSUI Shinsuke <poppen.jp@gmail.com>
|
6
|
+
# Distributed under GPL.
|
7
|
+
#
|
8
|
+
|
9
|
+
require 'points-scraper/default'
|
10
|
+
|
11
|
+
module Points::Scraper
|
12
|
+
class AnaMileage < Default
|
13
|
+
URL = 'https://www.ana.co.jp/'
|
14
|
+
|
15
|
+
def start
|
16
|
+
start_scrape do |agent|
|
17
|
+
agent = Mechanize::new
|
18
|
+
agent.set_proxy( *ENV['HTTP_PROXY'].split( /:/ ) ) if ENV['HTTP_PROXY']
|
19
|
+
|
20
|
+
agent.get(URL)
|
21
|
+
|
22
|
+
agent.page.form_with(:name => 'loginForm') do|form|
|
23
|
+
form.custno = @user
|
24
|
+
form.password = @pass
|
25
|
+
form.click_button
|
26
|
+
end
|
27
|
+
|
28
|
+
agent.page.forms[0].tap do|form|
|
29
|
+
form.encoding = 'CP932'
|
30
|
+
form.click_button
|
31
|
+
end
|
32
|
+
|
33
|
+
agent.page.at('//div[@class="mileInfo_a2"]//span[1]').text
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
# Local Variables:
|
40
|
+
# mode: ruby
|
41
|
+
# indent-tabs-mode: t
|
42
|
+
# tab-width: 3
|
43
|
+
# ruby-indent-level: 3
|
44
|
+
# End:
|
45
|
+
# vim: ts=3
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# default.rb: PointsScraper commons
|
4
|
+
#
|
5
|
+
# Copyright (C) 2012 by TADA Tadashi <t@tdtds.jp>
|
6
|
+
# Distributed under GPL.
|
7
|
+
#
|
8
|
+
|
9
|
+
module Points::Scraper
|
10
|
+
class Default
|
11
|
+
require 'mechanize'
|
12
|
+
|
13
|
+
def initialize( user, pass )
|
14
|
+
@user, @pass = user, pass
|
15
|
+
end
|
16
|
+
|
17
|
+
private
|
18
|
+
def start_scrape
|
19
|
+
agent = Mechanize::new
|
20
|
+
agent.set_proxy( *ENV['HTTP_PROXY'].split( /:/ ) ) if ENV['HTTP_PROXY']
|
21
|
+
yield agent
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# rakuten.rb: Rakuten Points for PointsScraper
|
4
|
+
# by scraping www.rakuten.co.jp.
|
5
|
+
#
|
6
|
+
# Copyright (C) 2012 by MATSUI Shinsuke <poppen.jp@gmail.com>
|
7
|
+
# Distributed under GPL.
|
8
|
+
#
|
9
|
+
|
10
|
+
require 'points-scraper/default'
|
11
|
+
|
12
|
+
module Points::Scraper
|
13
|
+
class Rakuten < Default
|
14
|
+
URL = 'https://point.rakuten.co.jp'
|
15
|
+
|
16
|
+
def start
|
17
|
+
start_scrape do |agent|
|
18
|
+
agent = Mechanize::new
|
19
|
+
agent.set_proxy( *ENV['HTTP_PROXY'].split( /:/ ) ) if ENV['HTTP_PROXY']
|
20
|
+
|
21
|
+
page = agent.get(URL)
|
22
|
+
|
23
|
+
login_page = agent.click( page.link_with(:href => /login/i) )
|
24
|
+
point_page = login_page.form_with(:name => 'LoginForm') do|form|
|
25
|
+
form.u = @user
|
26
|
+
form.p = @pass
|
27
|
+
end.click_button
|
28
|
+
|
29
|
+
point_page.at('//div[@id="pointAccount"]//dl[@class="total"]/dd').text
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
# Local Variables:
|
36
|
+
# mode: ruby
|
37
|
+
# indent-tabs-mode: t
|
38
|
+
# tab-width: 3
|
39
|
+
# ruby-indent-level: 3
|
40
|
+
# End:
|
41
|
+
# vim: ts=3
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# -*- coding: utf-8 -*-
|
2
|
+
#
|
3
|
+
# tpoint.rb: T-POINT for PointsScraper
|
4
|
+
#
|
5
|
+
# Copyright (C) 2012 by TADA Tadashi <t@tdtds.jp>
|
6
|
+
# Distributed under GPL.
|
7
|
+
#
|
8
|
+
|
9
|
+
require 'points-scraper/default'
|
10
|
+
|
11
|
+
module Points::Scraper
|
12
|
+
class TPoint < Default
|
13
|
+
URL = 'https://tsite.jp'
|
14
|
+
|
15
|
+
def start
|
16
|
+
start_scrape do |agent|
|
17
|
+
agent.get( URL + '/tm/pc/login/STKIp0001001.do' )
|
18
|
+
|
19
|
+
agent.page.form_with( :name => 'form1' ) do |form|
|
20
|
+
form.action = URL + '/tm/pc/login/STKIp0001010.do'
|
21
|
+
form['LOGIN_ID'] = @user
|
22
|
+
form['PASSWORD'] = @pass
|
23
|
+
form.click_button
|
24
|
+
end
|
25
|
+
|
26
|
+
agent.page.at( 'p.point > span.number' ).text
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# Local Variables:
|
33
|
+
# mode: ruby
|
34
|
+
# indent-tabs-mode: t
|
35
|
+
# tab-width: 3
|
36
|
+
# ruby-indent-level: 3
|
37
|
+
# End:
|
38
|
+
# vim: ts=3
|
data/misc/_points
ADDED
data/points-scraper.gemspec
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: points-scraper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -10,11 +10,22 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2012-
|
13
|
+
date: 2012-04-15 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: thor
|
17
|
+
requirement: &85964140 !ruby/object:Gem::Requirement
|
18
|
+
none: false
|
19
|
+
requirements:
|
20
|
+
- - ! '>='
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: '0'
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: *85964140
|
15
26
|
- !ruby/object:Gem::Dependency
|
16
27
|
name: mechanize
|
17
|
-
requirement: &
|
28
|
+
requirement: &85963930 !ruby/object:Gem::Requirement
|
18
29
|
none: false
|
19
30
|
requirements:
|
20
31
|
- - ! '>='
|
@@ -22,10 +33,10 @@ dependencies:
|
|
22
33
|
version: '0'
|
23
34
|
type: :runtime
|
24
35
|
prerelease: false
|
25
|
-
version_requirements: *
|
36
|
+
version_requirements: *85963930
|
26
37
|
- !ruby/object:Gem::Dependency
|
27
38
|
name: pit
|
28
|
-
requirement: &
|
39
|
+
requirement: &85963720 !ruby/object:Gem::Requirement
|
29
40
|
none: false
|
30
41
|
requirements:
|
31
42
|
- - ! '>='
|
@@ -33,27 +44,28 @@ dependencies:
|
|
33
44
|
version: '0'
|
34
45
|
type: :runtime
|
35
46
|
prerelease: false
|
36
|
-
version_requirements: *
|
47
|
+
version_requirements: *85963720
|
37
48
|
description: ! 'getting mileages/points by web scraping shopping sites: T-SITE, ANA
|
38
49
|
and rakuten.'
|
39
50
|
email:
|
40
51
|
- t@tdtds.jp
|
41
52
|
- poppen.jp@gmal.com
|
42
53
|
executables:
|
43
|
-
- points
|
44
|
-
- points-rakuten
|
45
|
-
- points-tpoint
|
54
|
+
- points
|
46
55
|
extensions: []
|
47
56
|
extra_rdoc_files: []
|
48
57
|
files:
|
49
58
|
- .gitignore
|
50
59
|
- Gemfile
|
51
60
|
- Rakefile
|
52
|
-
- bin/points
|
53
|
-
- bin/points-rakuten
|
54
|
-
- bin/points-tpoint
|
61
|
+
- bin/points
|
55
62
|
- lib/points-scraper.rb
|
63
|
+
- lib/points-scraper/anamileage.rb
|
64
|
+
- lib/points-scraper/default.rb
|
65
|
+
- lib/points-scraper/rakuten.rb
|
66
|
+
- lib/points-scraper/tpoint.rb
|
56
67
|
- lib/points-scraper/version.rb
|
68
|
+
- misc/_points
|
57
69
|
- points-scraper.gemspec
|
58
70
|
homepage: https://github.com/tdtds/points-scraper
|
59
71
|
licenses: []
|
data/bin/points-anamileage
DELETED
@@ -1,47 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
#
|
3
|
-
# points-anamileage.rb: getting current mileage
|
4
|
-
# by scraping www.ana.co.jp.
|
5
|
-
#
|
6
|
-
# Copyright (C) 2012 by MATSUI Shinsuke <poppen.jp@gmail.com>
|
7
|
-
# Distributed under GPL.
|
8
|
-
#
|
9
|
-
|
10
|
-
require 'rubygems'
|
11
|
-
require 'mechanize'
|
12
|
-
require 'pit'
|
13
|
-
|
14
|
-
URL = 'https://www.ana.co.jp/'
|
15
|
-
|
16
|
-
@login = Pit::get( 'ana', :require => {
|
17
|
-
'user' => 'your ID of ana.',
|
18
|
-
'pass' => 'your Password of ana.'
|
19
|
-
} )
|
20
|
-
|
21
|
-
agent = Mechanize::new
|
22
|
-
agent.set_proxy( *ENV['HTTP_PROXY'].split( /:/ ) ) if ENV['HTTP_PROXY']
|
23
|
-
|
24
|
-
agent.get(URL)
|
25
|
-
|
26
|
-
agent.page.form_with(:name => 'loginForm') do|form|
|
27
|
-
form.custno = @login['user']
|
28
|
-
form.password = @login['pass']
|
29
|
-
form.click_button
|
30
|
-
end
|
31
|
-
|
32
|
-
agent.page.forms[0].tap do|form|
|
33
|
-
form.encoding = 'CP932'
|
34
|
-
form.click_button
|
35
|
-
end
|
36
|
-
|
37
|
-
point = agent.page.at('//div[@class="mileInfo_a2"]//span[1]').text
|
38
|
-
|
39
|
-
puts point
|
40
|
-
|
41
|
-
# Local Variables:
|
42
|
-
# mode: ruby
|
43
|
-
# indent-tabs-mode: t
|
44
|
-
# tab-width: 3
|
45
|
-
# ruby-indent-level: 3
|
46
|
-
# End:
|
47
|
-
# vim: ts=3
|
data/bin/points-rakuten
DELETED
@@ -1,43 +0,0 @@
|
|
1
|
-
#! /usr/bin/env ruby
|
2
|
-
#
|
3
|
-
# points-rakuten.rb: getting current Rakuten Point
|
4
|
-
# by scraping www.rakuten.co.jp.
|
5
|
-
#
|
6
|
-
# Copyright (C) 2012 by MATSUI Shinsuke <poppen.jp@gmail.com>
|
7
|
-
# Distributed under GPL.
|
8
|
-
#
|
9
|
-
|
10
|
-
require 'rubygems'
|
11
|
-
require 'mechanize'
|
12
|
-
require 'pit'
|
13
|
-
|
14
|
-
URL = 'https://point.rakuten.co.jp'
|
15
|
-
|
16
|
-
@login = Pit::get( 'rakuten', :require => {
|
17
|
-
'user' => 'your ID of rakuten.',
|
18
|
-
'pass' => 'your Password of rakuten.'
|
19
|
-
} )
|
20
|
-
|
21
|
-
agent = Mechanize::new
|
22
|
-
agent.set_proxy( *ENV['HTTP_PROXY'].split( /:/ ) ) if ENV['HTTP_PROXY']
|
23
|
-
|
24
|
-
page = agent.get(URL)
|
25
|
-
|
26
|
-
login_page = agent.click( page.link_with(:href => /login/i) )
|
27
|
-
point_page = login_page.form_with(:name => 'LoginForm') do|form|
|
28
|
-
form.u = @login['user']
|
29
|
-
form.p = @login['pass']
|
30
|
-
end.click_button
|
31
|
-
|
32
|
-
point = point_page.at(
|
33
|
-
'//div[@id="pointAccount"]//dl[@class="total"]/dd').text
|
34
|
-
|
35
|
-
puts point
|
36
|
-
|
37
|
-
# Local Variables:
|
38
|
-
# mode: ruby
|
39
|
-
# indent-tabs-mode: t
|
40
|
-
# tab-width: 3
|
41
|
-
# ruby-indent-level: 3
|
42
|
-
# End:
|
43
|
-
# vim: ts=3
|
data/bin/points-tpoint
DELETED
@@ -1,40 +0,0 @@
|
|
1
|
-
#! /usr/bin/env ruby
|
2
|
-
#
|
3
|
-
# tsite.rb: getting current T-Point by scraping tsite.jp.
|
4
|
-
#
|
5
|
-
# Copyright (C) 2011 by TADA Tadashi <t@tdtds.jp>
|
6
|
-
# Distributed under GPL.
|
7
|
-
#
|
8
|
-
|
9
|
-
require 'rubygems'
|
10
|
-
require 'mechanize'
|
11
|
-
require 'pit'
|
12
|
-
|
13
|
-
URL = 'https://tsite.jp'
|
14
|
-
|
15
|
-
@login = Pit::get( 'tsite', :require => {
|
16
|
-
'user' => 'your ID of tsite.jp.',
|
17
|
-
'pass' => 'your Password of tsite.jp.'
|
18
|
-
} )
|
19
|
-
|
20
|
-
agent = Mechanize::new
|
21
|
-
agent.set_proxy( *ENV['HTTP_PROXY'].split( /:/ ) ) if ENV['HTTP_PROXY']
|
22
|
-
|
23
|
-
agent.get( URL + '/tm/pc/login/STKIp0001001.do' )
|
24
|
-
|
25
|
-
agent.page.form_with( :name => 'form1' ) do |form|
|
26
|
-
form.action = URL + '/tm/pc/login/STKIp0001010.do'
|
27
|
-
form['LOGIN_ID'] = @login['user']
|
28
|
-
form['PASSWORD'] = @login['pass']
|
29
|
-
form.click_button
|
30
|
-
end
|
31
|
-
|
32
|
-
puts agent.page.at( 'p.point > span.number' ).text
|
33
|
-
|
34
|
-
# Local Variables:
|
35
|
-
# mode: ruby
|
36
|
-
# indent-tabs-mode: t
|
37
|
-
# tab-width: 3
|
38
|
-
# ruby-indent-level: 3
|
39
|
-
# End:
|
40
|
-
# vim: ts=3
|