autility 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/Readme.md +5 -1
- data/bin/autility +20 -0
- data/lib/autility.rb +1 -0
- data/lib/autility/movistar_spain.rb +122 -0
- data/lib/autility/version.rb +1 -1
- data/test/autility/movistar_spain_test.rb +32 -0
- metadata +17 -14
data/Readme.md
CHANGED
@@ -5,7 +5,11 @@ A scraper that automatically downloads PDF invoices from utility vendors. That's
|
|
5
5
|
It will focus mainly on spanish vendors, but pull requests will be accepted
|
6
6
|
to support as many vendors as possible.
|
7
7
|
|
8
|
-
For now it supports downloading invoices from
|
8
|
+
For now it supports downloading invoices from:
|
9
|
+
|
10
|
+
* Endesa
|
11
|
+
* Vodafone (Spain)
|
12
|
+
* Movistar (Spain)
|
9
13
|
|
10
14
|
## Install
|
11
15
|
|
data/bin/autility
CHANGED
@@ -47,3 +47,23 @@ command :vodafone do |c|
|
|
47
47
|
end
|
48
48
|
end
|
49
49
|
|
50
|
+
command :movistar do |c|
|
51
|
+
c.syntax = 'autility movistar [options]'
|
52
|
+
c.summary = 'Downloads invoices from Movistar.'
|
53
|
+
c.description = 'Downloads invoices from Movistar.'
|
54
|
+
c.example 'Download the invoice for the current month of user "foo" with password "bar"', 'utilities movistar --user foo --password bar'
|
55
|
+
c.option '--user USER', String, 'The username for Movistar Online'
|
56
|
+
c.option '--password PASSWORD', String, 'The password for Movistar Online'
|
57
|
+
c.option '--month MONTH_NUMBER', Integer, 'The month (e.g. 5, 9, 12...). By default it\'s the current month'
|
58
|
+
c.option '--output-folder FOLDER_NAME', String, 'The output folder to store the PDF in. Defaults to ~/utilities'
|
59
|
+
c.action do |args, options|
|
60
|
+
options.default :month => Time.now.month
|
61
|
+
options.default :output_folder => File.expand_path("~/utilities")
|
62
|
+
|
63
|
+
raise "Both --user and --password are required options." unless options.user && options.password
|
64
|
+
|
65
|
+
result = Autility::MovistarSpain.scrape(options.user, options.password, options.month, options.output_folder)
|
66
|
+
puts "Path: #{result}"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
data/lib/autility.rb
CHANGED
@@ -0,0 +1,122 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'capybara'
|
3
|
+
require 'capybara/dsl'
|
4
|
+
require 'show_me_the_cookies'
|
5
|
+
require 'fileutils'
|
6
|
+
|
7
|
+
Capybara.register_driver :selenium do |app|
|
8
|
+
Capybara::Selenium::Driver.new(app, :browser => :firefox)
|
9
|
+
end
|
10
|
+
|
11
|
+
module Autility
|
12
|
+
# Public: A scraper for all the utility invoices in MovistarSpain.
|
13
|
+
#
|
14
|
+
# Examples
|
15
|
+
#
|
16
|
+
# # Download the invoice from September this year and store it in /tmp.
|
17
|
+
# MovistarSpain.scrape("user", "password", 9, "/tmp") # Download all invoices from September this year
|
18
|
+
#
|
19
|
+
class MovistarSpain
|
20
|
+
include Capybara::DSL
|
21
|
+
include ShowMeTheCookies
|
22
|
+
|
23
|
+
# Public: Instantiates a new scraper and fires it to download the utility
|
24
|
+
# invoices from MovistarSpain.
|
25
|
+
#
|
26
|
+
# Returns nothing.
|
27
|
+
def self.scrape(*args)
|
28
|
+
new(*args).scrape
|
29
|
+
end
|
30
|
+
|
31
|
+
def initialize(user, password, month, folder)
|
32
|
+
@user = user
|
33
|
+
@password = password
|
34
|
+
@month = month
|
35
|
+
@folder = folder
|
36
|
+
@year = Time.now.year
|
37
|
+
end
|
38
|
+
|
39
|
+
# Public: Scrapes the vodafone website and gets the invoice for the current
|
40
|
+
# month, saving it to @folder.
|
41
|
+
#
|
42
|
+
# Returns the String path of the saved document.
|
43
|
+
def scrape
|
44
|
+
setup_capybara
|
45
|
+
log_in
|
46
|
+
|
47
|
+
FileUtils.mkdir_p(@folder)
|
48
|
+
filename = "#{@folder}/movistar_#{month}_#{@year}.pdf"
|
49
|
+
document.save(filename)
|
50
|
+
filename
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
# Internal: Logs in to the MovistarSpain website.
|
56
|
+
#
|
57
|
+
# Returns nothing.
|
58
|
+
def log_in
|
59
|
+
visit "/on/pub/ServNav?servicio=home&home=personal&v_segmento=EMPR"
|
60
|
+
fill_in "usuario", :with => @user
|
61
|
+
fill_in "contrasenia", :with => @password
|
62
|
+
first("#enviar").click
|
63
|
+
end
|
64
|
+
|
65
|
+
# Internal: Gets the latest invoice and returns it as a Document (not
|
66
|
+
# fetched yet).
|
67
|
+
#
|
68
|
+
# Returns the Document to be fetched.
|
69
|
+
def document
|
70
|
+
@document ||= begin
|
71
|
+
invoice = nil
|
72
|
+
|
73
|
+
cookie = Cookie.new("wacsessionid", get_me_the_cookie("wacsessionid")[:value])
|
74
|
+
|
75
|
+
visit "https://www.movistar.es/id/priv/seg00/jsp/UFE/es/IDAA00SCUFE_1_facturas_STB.jsp?SOLDUP="
|
76
|
+
|
77
|
+
combo = all(".combos_numerados_facturacion1")[1]
|
78
|
+
select = combo.find(".combos_numeros")
|
79
|
+
id = select[:id]
|
80
|
+
options = select.all('option')
|
81
|
+
|
82
|
+
found = options.detect do |date|
|
83
|
+
date.text =~ /#{month}-#{@year}/
|
84
|
+
end
|
85
|
+
|
86
|
+
url = "https://www.movistar.es"
|
87
|
+
|
88
|
+
if found
|
89
|
+
select(found.text, from: id)
|
90
|
+
find("#aceptar").click
|
91
|
+
|
92
|
+
within_window(page.driver.browser.window_handles.last) do
|
93
|
+
within_frame(find('frame')[:id]) do
|
94
|
+
within_frame(find('iframe')[:id]) do
|
95
|
+
url += find('#descargar a')[:href].gsub("javascript:CargarGuardar('","").gsub("')","")
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
else
|
100
|
+
raise "MovistarSpain invoice for month #{month} is not available yet."
|
101
|
+
end
|
102
|
+
|
103
|
+
Document.new(url, :get, cookie)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
# Internal: Returns the String current month padded with zeros to the left.
|
108
|
+
def month
|
109
|
+
@month.to_s.rjust(2, '0')
|
110
|
+
end
|
111
|
+
|
112
|
+
# Internal: Sets the configuration for capybara to work with the MovistarSpain
|
113
|
+
# website.
|
114
|
+
#
|
115
|
+
# Returns nothing.
|
116
|
+
def setup_capybara
|
117
|
+
Capybara.run_server = false
|
118
|
+
Capybara.current_driver = :selenium
|
119
|
+
Capybara.app_host = 'https://www.movistar.es'
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
data/lib/autility/version.rb
CHANGED
@@ -0,0 +1,32 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative '../test_helper'
|
3
|
+
|
4
|
+
module Autility
|
5
|
+
describe MovistarSpain do
|
6
|
+
subject { MovistarSpain.new("foo", "bar", 11, "/tmp/utilities") }
|
7
|
+
|
8
|
+
describe ".scrape" do
|
9
|
+
it 'delegates to an instance' do
|
10
|
+
MovistarSpain.stubs(:new).with("foo", "bar", 11, "/tmp/utilities").returns subject
|
11
|
+
subject.expects(:scrape)
|
12
|
+
|
13
|
+
MovistarSpain.scrape("foo", "bar", 11, "/tmp/utilities")
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#scrape' do
|
18
|
+
it 'gets the document in' do
|
19
|
+
subject.expects(:setup_capybara)
|
20
|
+
subject.expects(:log_in)
|
21
|
+
document = Document.new(url = stub, cookie = stub, {})
|
22
|
+
subject.expects(:document).returns(document)
|
23
|
+
document.expects(:save)
|
24
|
+
|
25
|
+
month = 11
|
26
|
+
year = Time.now.year
|
27
|
+
|
28
|
+
subject.scrape.must_equal "/tmp/utilities/movistar_#{month}_#{year}.pdf"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: autility
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2011-12-
|
12
|
+
date: 2011-12-15 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: capybara
|
16
|
-
requirement: &
|
16
|
+
requirement: &70096387494420 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ! '>='
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: '0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *70096387494420
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: capybara-webkit
|
27
|
-
requirement: &
|
27
|
+
requirement: &70096387493420 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ! '>='
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: '0'
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *70096387493420
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: show_me_the_cookies
|
38
|
-
requirement: &
|
38
|
+
requirement: &70096387492300 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ! '>='
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: '0'
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *70096387492300
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: commander
|
49
|
-
requirement: &
|
49
|
+
requirement: &70096387489320 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ! '>='
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: '0'
|
55
55
|
type: :runtime
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *70096387489320
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: mocha
|
60
|
-
requirement: &
|
60
|
+
requirement: &70096387488100 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ! '>='
|
@@ -65,10 +65,10 @@ dependencies:
|
|
65
65
|
version: '0'
|
66
66
|
type: :development
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *70096387488100
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: minitest
|
71
|
-
requirement: &
|
71
|
+
requirement: &70096387487580 !ruby/object:Gem::Requirement
|
72
72
|
none: false
|
73
73
|
requirements:
|
74
74
|
- - ! '>='
|
@@ -76,7 +76,7 @@ dependencies:
|
|
76
76
|
version: '0'
|
77
77
|
type: :development
|
78
78
|
prerelease: false
|
79
|
-
version_requirements: *
|
79
|
+
version_requirements: *70096387487580
|
80
80
|
description: Downloads utility invoices from common spanish firms such as Endesa or
|
81
81
|
Vodafone.
|
82
82
|
email:
|
@@ -99,12 +99,14 @@ files:
|
|
99
99
|
- lib/autility/data.rb
|
100
100
|
- lib/autility/document.rb
|
101
101
|
- lib/autility/endesa.rb
|
102
|
+
- lib/autility/movistar_spain.rb
|
102
103
|
- lib/autility/version.rb
|
103
104
|
- lib/autility/vodafone_spain.rb
|
104
105
|
- test/autility/command_test.rb
|
105
106
|
- test/autility/cookie_test.rb
|
106
107
|
- test/autility/document_test.rb
|
107
108
|
- test/autility/endesa_test.rb
|
109
|
+
- test/autility/movistar_spain_test.rb
|
108
110
|
- test/autility/vodafone_spain_test.rb
|
109
111
|
- test/test_helper.rb
|
110
112
|
homepage: http://github.com/codegram/autility
|
@@ -136,5 +138,6 @@ test_files:
|
|
136
138
|
- test/autility/cookie_test.rb
|
137
139
|
- test/autility/document_test.rb
|
138
140
|
- test/autility/endesa_test.rb
|
141
|
+
- test/autility/movistar_spain_test.rb
|
139
142
|
- test/autility/vodafone_spain_test.rb
|
140
143
|
- test/test_helper.rb
|