autility 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +4 -0
- data/.rvmrc +47 -0
- data/Gemfile +5 -0
- data/Rakefile +11 -0
- data/Readme.md +45 -0
- data/autility.gemspec +27 -0
- data/bin/autility +49 -0
- data/lib/autility.rb +11 -0
- data/lib/autility/command.rb +40 -0
- data/lib/autility/cookie.rb +18 -0
- data/lib/autility/data.rb +21 -0
- data/lib/autility/document.rb +35 -0
- data/lib/autility/endesa.rb +110 -0
- data/lib/autility/version.rb +3 -0
- data/lib/autility/vodafone_spain.rb +102 -0
- data/test/autility/command_test.rb +21 -0
- data/test/autility/cookie_test.rb +13 -0
- data/test/autility/document_test.rb +19 -0
- data/test/autility/endesa_test.rb +32 -0
- data/test/autility/vodafone_spain_test.rb +32 -0
- data/test/test_helper.rb +5 -0
- metadata +140 -0
data/.gitignore
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
# This is an RVM Project .rvmrc file, used to automatically load the ruby
|
4
|
+
# development environment upon cd'ing into the directory
|
5
|
+
|
6
|
+
# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional.
|
7
|
+
environment_id="1.9.3@utilities"
|
8
|
+
|
9
|
+
#
|
10
|
+
# First we attempt to load the desired environment directly from the environment
|
11
|
+
# file. This is very fast and efficicent compared to running through the entire
|
12
|
+
# CLI and selector. If you want feedback on which environment was used then
|
13
|
+
# insert the word 'use' after --create as this triggers verbose mode.
|
14
|
+
#
|
15
|
+
if [[ -d "${rvm_path:-$HOME/.rvm}/environments" \
|
16
|
+
&& -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
|
17
|
+
then
|
18
|
+
\. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
|
19
|
+
|
20
|
+
if [[ -s ".rvm/hooks/after_use" ]]
|
21
|
+
then
|
22
|
+
. ".rvm/hooks/after_use"
|
23
|
+
fi
|
24
|
+
else
|
25
|
+
# If the environment file has not yet been created, use the RVM CLI to select.
|
26
|
+
if ! rvm --create "$environment_id"
|
27
|
+
then
|
28
|
+
echo "Failed to create RVM environment ''."
|
29
|
+
fi
|
30
|
+
fi
|
31
|
+
|
32
|
+
#
|
33
|
+
# If you use an RVM gemset file to install a list of gems (*.gems), you can have
|
34
|
+
# it be automatically loaded. Uncomment the following and adjust the filename if
|
35
|
+
# necessary.
|
36
|
+
#
|
37
|
+
# filename=".gems"
|
38
|
+
# if [[ -s "$filename" ]] ; then
|
39
|
+
# rvm gemset import "$filename" | grep -v already | grep -v listed | grep -v complete | sed '/^$/d'
|
40
|
+
# fi
|
41
|
+
|
42
|
+
#
|
43
|
+
# If you use bundler and would like to run bundle each time you enter the
|
44
|
+
# directory, you can uncomment the following code.
|
45
|
+
#
|
46
|
+
export PATH="./bin:$PATH"
|
47
|
+
#
|
data/Gemfile
ADDED
data/Rakefile
ADDED
data/Readme.md
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# autility
|
2
|
+
|
3
|
+
A scraper that automatically downloads PDF invoices from utility vendors. That's it.
|
4
|
+
|
5
|
+
It will focus mainly on spanish vendors, but pull requests will be accepted
|
6
|
+
to support as many vendors as possible.
|
7
|
+
|
8
|
+
For now it supports downloading invoices from Endesa and Vodafone Spain.
|
9
|
+
|
10
|
+
## Install
|
11
|
+
|
12
|
+
$ gem install autility
|
13
|
+
|
14
|
+
## Usage
|
15
|
+
|
16
|
+
$ autility VENDOR --user USER --password PASSWORD [--month MONTH] [--output-folder OUTPUT_FOLDER]
|
17
|
+
$ autility --help
|
18
|
+
$ autility --help endesa
|
19
|
+
$ autility --help vodafone
|
20
|
+
|
21
|
+
## Examples
|
22
|
+
|
23
|
+
Downloading the Endesa invoice for the current month:
|
24
|
+
|
25
|
+
$ autility endesa --user john --password doe
|
26
|
+
|
27
|
+
Downloading the Vodafone invoice for the past October and save it to ~/billing:
|
28
|
+
|
29
|
+
$ autility vodafone --user john --password doe --month 10 --output-folder ~/billing
|
30
|
+
|
31
|
+
## Parsing the output
|
32
|
+
|
33
|
+
The output is simple, thus parseable by UNIX tools such as grep or awk. It
|
34
|
+
looks like this:
|
35
|
+
|
36
|
+
Path: /Users/john/billing/vodafone_10_2011.pdf
|
37
|
+
|
38
|
+
For example, to open the file right after downloading it:
|
39
|
+
|
40
|
+
$ open `autility endesa --user john --password doe | grep Path: | awk '{print $2}'`
|
41
|
+
|
42
|
+
## License
|
43
|
+
|
44
|
+
Released under the MIT License.
|
45
|
+
Copyright 2011 [Codegram Technologies](http://codegram.com)
|
data/autility.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "autility/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "autility"
|
7
|
+
s.version = Autility::VERSION
|
8
|
+
s.authors = ["Josep M. Bach"]
|
9
|
+
s.email = ["josep.m.bach@gmail.com"]
|
10
|
+
s.homepage = "http://github.com/codegram/utilities"
|
11
|
+
s.summary = %q{Downloads utility invoices from common spanish firms such as Endesa or Vodafone.}
|
12
|
+
s.description = %q{Downloads utility invoices from common spanish firms such as Endesa or Vodafone.}
|
13
|
+
|
14
|
+
s.rubyforge_project = "utilities"
|
15
|
+
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
|
21
|
+
s.add_runtime_dependency "capybara"
|
22
|
+
s.add_runtime_dependency "capybara-webkit"
|
23
|
+
s.add_runtime_dependency "show_me_the_cookies"
|
24
|
+
s.add_runtime_dependency "commander"
|
25
|
+
s.add_development_dependency "mocha"
|
26
|
+
s.add_development_dependency "minitest"
|
27
|
+
end
|
data/bin/autility
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'commander/import'
|
5
|
+
require_relative '../lib/autility'
|
6
|
+
|
7
|
+
program :version, Autility::VERSION
|
8
|
+
program :description, 'Autility is a CLI interface to download invoices from various utility vendors, such as Vodafone or Endesa.'
|
9
|
+
|
10
|
+
command :endesa do |c|
|
11
|
+
c.syntax = 'autility endesa [options]'
|
12
|
+
c.summary = 'Downloads invoices from Endesa.'
|
13
|
+
c.description = 'Downloads invoices from Endesa.'
|
14
|
+
c.example 'Download the invoice for the current month of user "foo" with password "bar"', 'autility endesa --user foo --password bar'
|
15
|
+
c.option '--user USER', String, 'The username for Endesa Online'
|
16
|
+
c.option '--password PASSWORD', String, 'The password for Endesa Online'
|
17
|
+
c.option '--month MONTH_NUMBER', Integer, 'The month (e.g. 5, 9, 12...). By default it\'s the current month'
|
18
|
+
c.option '--output-folder FOLDER_NAME', String, 'The output folder to store the PDF in. Defaults to ~/utilities'
|
19
|
+
c.action do |args, options|
|
20
|
+
options.default :month => Time.now.month
|
21
|
+
options.default :output_folder => File.expand_path("~/utilities")
|
22
|
+
|
23
|
+
raise "Both --user and --password are required options." unless options.user && options.password
|
24
|
+
|
25
|
+
result = Autility::Endesa.scrape(options.user, options.password, options.month, options.output_folder)
|
26
|
+
puts "Path: #{result}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
command :vodafone do |c|
|
31
|
+
c.syntax = 'autility vodafone [options]'
|
32
|
+
c.summary = 'Downloads invoices from Vodafone.'
|
33
|
+
c.description = 'Downloads invoices from Vodafone.'
|
34
|
+
c.example 'Download the invoice for the current month of user "foo" with password "bar"', 'utilities vodafone --user foo --password bar'
|
35
|
+
c.option '--user USER', String, 'The username for Vodafone Online'
|
36
|
+
c.option '--password PASSWORD', String, 'The password for Vodafone Online'
|
37
|
+
c.option '--month MONTH_NUMBER', Integer, 'The month (e.g. 5, 9, 12...). By default it\'s the current month'
|
38
|
+
c.option '--output-folder FOLDER_NAME', String, 'The output folder to store the PDF in. Defaults to ~/utilities'
|
39
|
+
c.action do |args, options|
|
40
|
+
options.default :month => Time.now.month
|
41
|
+
options.default :output_folder => File.expand_path("~/utilities")
|
42
|
+
|
43
|
+
raise "Both --user and --password are required options." unless options.user && options.password
|
44
|
+
|
45
|
+
result = Autility::VodafoneSpain.scrape(options.user, options.password, options.month, options.output_folder)
|
46
|
+
puts "Path: #{result}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
data/lib/autility.rb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
require_relative 'autility/version'
|
2
|
+
require_relative 'autility/document'
|
3
|
+
require_relative 'autility/command'
|
4
|
+
require_relative 'autility/cookie'
|
5
|
+
|
6
|
+
require_relative 'autility/endesa'
|
7
|
+
require_relative 'autility/vodafone_spain'
|
8
|
+
|
9
|
+
module Autility
|
10
|
+
end
|
11
|
+
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Autility
|
2
|
+
# Public: A Command is a system cURL command builder.
|
3
|
+
#
|
4
|
+
# Examples
|
5
|
+
#
|
6
|
+
# command = Command.build(url, params, cookie, "foo.pdf")
|
7
|
+
# # => "curl --data ..."
|
8
|
+
#
|
9
|
+
module Command
|
10
|
+
# Public: Builds a command to fetch and save a remote document via cURL.
|
11
|
+
#
|
12
|
+
# url - The String url where we can get the document
|
13
|
+
# method - the String HTTP method to use.
|
14
|
+
# cookie - the session Cookie needed to access the url
|
15
|
+
# params - The Array of POST params needed to fetch it
|
16
|
+
# path - the String path to save the document to.
|
17
|
+
#
|
18
|
+
# Returns the String command ready to execute.
|
19
|
+
def build(url, method, cookie, params, path)
|
20
|
+
out = "curl"
|
21
|
+
|
22
|
+
if method == :post
|
23
|
+
out = "curl --data \""
|
24
|
+
|
25
|
+
out << params.to_a.map do |name, value|
|
26
|
+
"#{name}=#{value}"
|
27
|
+
end.join("&")
|
28
|
+
|
29
|
+
out << "\""
|
30
|
+
end
|
31
|
+
|
32
|
+
out << " #{cookie.to_command}" if cookie
|
33
|
+
out << " \"#{url}\""
|
34
|
+
out << " -o "
|
35
|
+
out << path
|
36
|
+
out
|
37
|
+
end
|
38
|
+
module_function :build
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Autility
|
2
|
+
# Public: Represents a browser cookie.
|
3
|
+
#
|
4
|
+
# Examples
|
5
|
+
#
|
6
|
+
# cookie = Cookie.new("JSESSIONID", "123")
|
7
|
+
# cookie.to_command
|
8
|
+
# # => "--cookie \"JSESSIONID=123\""
|
9
|
+
#
|
10
|
+
class Cookie < Struct.new(:name, :value)
|
11
|
+
# Public: Converts the cookie to a cURL option.
|
12
|
+
#
|
13
|
+
# Returns the String cURL option.
|
14
|
+
def to_command
|
15
|
+
%Q(--cookie "#{name}=#{value}")
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Autility
|
2
|
+
# Public: Represents a remote document we are interested in.
|
3
|
+
#
|
4
|
+
# Examples
|
5
|
+
#
|
6
|
+
# document = Document.new(url, params, cookie)
|
7
|
+
# document.save("~/utilities/endesa_11_2012.pdf")
|
8
|
+
#
|
9
|
+
class Document
|
10
|
+
# Public: Initializes a new document.
|
11
|
+
#
|
12
|
+
# url - The String url where we can get the document
|
13
|
+
# params - The Array of POST params needed to fetch it
|
14
|
+
# cookie - the session Cookie needed to access the url
|
15
|
+
def initialize(url, params, cookie)
|
16
|
+
@url = url
|
17
|
+
@params = params
|
18
|
+
@cookie = cookie
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Autility
|
2
|
+
# Public: Represents a remote document we are interested in.
|
3
|
+
#
|
4
|
+
# Examples
|
5
|
+
#
|
6
|
+
# document = Document.new(url, :post, cookie, params)
|
7
|
+
# document.save("~/utilities/endesa_11_2012.pdf")
|
8
|
+
#
|
9
|
+
class Document
|
10
|
+
attr_reader :url, :params, :cookie
|
11
|
+
|
12
|
+
# Public: Initializes a new document.
|
13
|
+
#
|
14
|
+
# url - The String url where we can get the document
|
15
|
+
# method - The String HTTP method to use.
|
16
|
+
# cookie - the session Cookie needed to access the url
|
17
|
+
# params - The Array of POST params needed to fetch it
|
18
|
+
def initialize(url, method=:get, cookie=nil, params={})
|
19
|
+
@url = url
|
20
|
+
@method = method
|
21
|
+
@params = params
|
22
|
+
@cookie = cookie
|
23
|
+
end
|
24
|
+
|
25
|
+
# Public: Fetches the document and saves it to a particular path.
|
26
|
+
#
|
27
|
+
# path - The String path where we will save the document.
|
28
|
+
#
|
29
|
+
# Returns nothing.
|
30
|
+
def save(path)
|
31
|
+
command = Command.build(@url, @method, @cookie, @params, path)
|
32
|
+
system(command)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'capybara'
|
3
|
+
require 'capybara/dsl'
|
4
|
+
require 'show_me_the_cookies'
|
5
|
+
require 'fileutils'
|
6
|
+
|
7
|
+
module Autility
|
8
|
+
# Public: A scraper for all the utility invoices i Endesa.
|
9
|
+
#
|
10
|
+
# Examples
|
11
|
+
#
|
12
|
+
# # Download the invoice from September this year and store it in /tmp.
|
13
|
+
# Endesa.scrape("user", "password", 9, "/tmp") # Download all invoices from September this year
|
14
|
+
#
|
15
|
+
class Endesa
|
16
|
+
include Capybara::DSL
|
17
|
+
include ShowMeTheCookies
|
18
|
+
|
19
|
+
# Public: Instantiates a new scraper and fires it to download the utility
|
20
|
+
# invoices from Endesa.
|
21
|
+
#
|
22
|
+
# Returns nothing.
|
23
|
+
def self.scrape(*args)
|
24
|
+
new(*args).scrape
|
25
|
+
end
|
26
|
+
|
27
|
+
def initialize(user, password, month, folder)
|
28
|
+
@user = user
|
29
|
+
@password = password
|
30
|
+
@month = month
|
31
|
+
@folder = folder
|
32
|
+
@year = Time.now.year
|
33
|
+
end
|
34
|
+
|
35
|
+
# Public: Scrapes the endesa website and gets the invoice for the current
|
36
|
+
# month, saving it to @folder.
|
37
|
+
#
|
38
|
+
# Returns the String path of the saved document.
|
39
|
+
def scrape
|
40
|
+
setup_capybara
|
41
|
+
log_in
|
42
|
+
|
43
|
+
FileUtils.mkdir_p(@folder)
|
44
|
+
filename = "#{@folder}/endesa_#{month}_#{@year}.pdf"
|
45
|
+
document.save(filename)
|
46
|
+
filename
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
# Internal: Logs in to the Endesa website.
|
52
|
+
#
|
53
|
+
# Returns nothing.
|
54
|
+
def log_in
|
55
|
+
visit "/gp/login.do"
|
56
|
+
fill_in "id", :with => @user
|
57
|
+
fill_in "clave", :with => @password
|
58
|
+
first("#entrar").click
|
59
|
+
end
|
60
|
+
|
61
|
+
# Internal: Gets the latest invoice and returns it as a Document (not
|
62
|
+
# fetched yet).
|
63
|
+
#
|
64
|
+
# Returns the Document to be fetched.
|
65
|
+
def document
|
66
|
+
@document ||= begin
|
67
|
+
visit "/gp/GenericForward.do?TO=ListadoFacturas"
|
68
|
+
links = all("#capa_contenido form table table td > a")
|
69
|
+
invoices = lambda { |link| link.text =~ %r{\d{2}/\d{2}/\d{2}} }
|
70
|
+
|
71
|
+
invoice = links.select(&invoices).detect do |link|
|
72
|
+
link.text =~ %r{/#{month}/}
|
73
|
+
# link.text =~ %r{/11/}
|
74
|
+
end
|
75
|
+
|
76
|
+
raise "Endesa invoice for month #{month} is not available yet." unless invoice
|
77
|
+
|
78
|
+
cfactura, cd_contrext = invoice[:onclick].scan(/'(\w+)','(\d+)'/).flatten
|
79
|
+
url = "https://www.gp.endesaonline.com/gp/obtenerFacturaPDF.do?cfactura=#{cfactura}&cd_contrext=#{cd_contrext}"
|
80
|
+
cookie = Cookie.new("JSESSIONID", get_me_the_cookie("JSESSIONID")[:value])
|
81
|
+
Document.new(url, :post, cookie)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Internal: Hack for ShowMeTheCookies.
|
86
|
+
def driver
|
87
|
+
Capybara.current_driver
|
88
|
+
end
|
89
|
+
|
90
|
+
# Internal: Hack for ShowMeTheCookies.
|
91
|
+
def drivers
|
92
|
+
send :adapters
|
93
|
+
end
|
94
|
+
|
95
|
+
# Internal: Returns the String current month padded with zeros to the left.
|
96
|
+
def month
|
97
|
+
@month.to_s.rjust(2, '0')
|
98
|
+
end
|
99
|
+
|
100
|
+
# Internal: Sets the configuration for capybara to work with the Endesa
|
101
|
+
# website.
|
102
|
+
#
|
103
|
+
# Returns nothing.
|
104
|
+
def setup_capybara
|
105
|
+
Capybara.run_server = false
|
106
|
+
Capybara.current_driver = :selenium
|
107
|
+
Capybara.app_host = 'https://www.gp.endesaonline.com'
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'capybara'
|
3
|
+
require 'capybara/dsl'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
module Autility
|
7
|
+
# Public: A scraper for all the utility invoices in VodafoneSpain.
|
8
|
+
#
|
9
|
+
# Examples
|
10
|
+
#
|
11
|
+
# # Download the invoice from September this year and store it in /tmp.
|
12
|
+
# VodafoneSpain.scrape("user", "password", 9, "/tmp") # Download all invoices from September this year
|
13
|
+
#
|
14
|
+
class VodafoneSpain
|
15
|
+
include Capybara::DSL
|
16
|
+
|
17
|
+
# Public: Instantiates a new scraper and fires it to download the utility
|
18
|
+
# invoices from VodafoneSpain.
|
19
|
+
#
|
20
|
+
# Returns nothing.
|
21
|
+
def self.scrape(*args)
|
22
|
+
new(*args).scrape
|
23
|
+
end
|
24
|
+
|
25
|
+
def initialize(user, password, month, folder)
|
26
|
+
@user = user
|
27
|
+
@password = password
|
28
|
+
@month = month
|
29
|
+
@folder = folder
|
30
|
+
@year = Time.now.year
|
31
|
+
end
|
32
|
+
|
33
|
+
# Public: Scrapes the vodafone website and gets the invoice for the current
|
34
|
+
# month, saving it to @folder.
|
35
|
+
#
|
36
|
+
# Returns the String path of the saved document.
|
37
|
+
def scrape
|
38
|
+
setup_capybara
|
39
|
+
log_in
|
40
|
+
|
41
|
+
FileUtils.mkdir_p(@folder)
|
42
|
+
filename = "#{@folder}/vodafone_#{month}_#{@year}.pdf"
|
43
|
+
document.save(filename)
|
44
|
+
filename
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
# Internal: Logs in to the VodafoneSpain website.
|
50
|
+
#
|
51
|
+
# Returns nothing.
|
52
|
+
def log_in
|
53
|
+
visit "/cpar/do/home/loginVodafone?type=empresas"
|
54
|
+
fill_in "user", :with => @user
|
55
|
+
fill_in "pass", :with => @password
|
56
|
+
first(".botonera_entrar #login").click
|
57
|
+
end
|
58
|
+
|
59
|
+
# Internal: Gets the latest invoice and returns it as a Document (not
|
60
|
+
# fetched yet).
|
61
|
+
#
|
62
|
+
# Returns the Document to be fetched.
|
63
|
+
def document
|
64
|
+
@document ||= begin
|
65
|
+
invoice = nil
|
66
|
+
|
67
|
+
visit "https://areaclientes.vodafone.es/cwgc/do/ebilling/get?ebplink=/tbmb/main/dashboard/invoices.do"
|
68
|
+
|
69
|
+
within "table#recent_invoices_tab1" do
|
70
|
+
row = all("tr").detect do |tr|
|
71
|
+
tr.text =~ /-#{month}-#{@year}/
|
72
|
+
end
|
73
|
+
invoice = row.find('#scriptOnly a')
|
74
|
+
end
|
75
|
+
|
76
|
+
raise "VodafoneSpain invoice for month #{month} is not available yet." unless invoice
|
77
|
+
|
78
|
+
url = "https://b2b.ebilling.vodafone.es/tbmb"
|
79
|
+
url += invoice[:onclick].gsub("downloadDocument('", '')[0..-2]
|
80
|
+
|
81
|
+
p url
|
82
|
+
|
83
|
+
Document.new(url)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
# Internal: Returns the String current month padded with zeros to the left.
|
88
|
+
def month
|
89
|
+
@month.to_s.rjust(2, '0')
|
90
|
+
end
|
91
|
+
|
92
|
+
# Internal: Sets the configuration for capybara to work with the VodafoneSpain
|
93
|
+
# website.
|
94
|
+
#
|
95
|
+
# Returns nothing.
|
96
|
+
def setup_capybara
|
97
|
+
Capybara.run_server = false
|
98
|
+
Capybara.current_driver = :selenium
|
99
|
+
Capybara.app_host = 'https://canalonline.vodafone.es'
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module Autility
|
4
|
+
describe Command do
|
5
|
+
let(:url) { "http://example.com/some_document.pdf" }
|
6
|
+
let(:params) do
|
7
|
+
{ foo: "bar", baz: "yeah" }
|
8
|
+
end
|
9
|
+
let(:cookie) { stub(to_command: "--cookie \"JSESSIONID=123\"") }
|
10
|
+
|
11
|
+
describe ".build" do
|
12
|
+
it 'builds a system command' do
|
13
|
+
command = Command.build(url, :post, cookie, params, "foo.pdf")
|
14
|
+
|
15
|
+
command.must_equal(
|
16
|
+
%Q{curl --data "foo=bar&baz=yeah" --cookie "JSESSIONID=123" "http://example.com/some_document.pdf" -o foo.pdf}
|
17
|
+
)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module Autility
|
4
|
+
describe Cookie do
|
5
|
+
subject { Cookie.new("JSESSIONID", "123") }
|
6
|
+
|
7
|
+
describe "#to_command" do
|
8
|
+
it 'converts the cookie to a cURL option' do
|
9
|
+
subject.to_command.must_equal "--cookie \"JSESSIONID=123\""
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module Autility
|
4
|
+
describe Document do
|
5
|
+
describe "#save" do
|
6
|
+
let(:url) { stub }
|
7
|
+
let(:params) { stub }
|
8
|
+
let(:cookie) { stub }
|
9
|
+
|
10
|
+
subject { Document.new(url, :get, cookie, params) }
|
11
|
+
|
12
|
+
it "fetches the document and saves it to a path" do
|
13
|
+
Command.stubs(:build).with(url, :get, cookie, params, "foo.pdf").returns command = stub
|
14
|
+
subject.expects(:system).with command
|
15
|
+
subject.save("foo.pdf")
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative '../test_helper'
|
3
|
+
|
4
|
+
module Autility
|
5
|
+
describe Endesa do
|
6
|
+
subject { Endesa.new("foo", "bar", 11, "/tmp/utilities") }
|
7
|
+
|
8
|
+
describe ".scrape" do
|
9
|
+
it 'delegates to an instance' do
|
10
|
+
Endesa.stubs(:new).with("foo", "bar", 11, "/tmp/utilities").returns subject
|
11
|
+
subject.expects(:scrape)
|
12
|
+
|
13
|
+
Endesa.scrape("foo", "bar", 11, "/tmp/utilities")
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#scrape' do
|
18
|
+
it 'gets the document in' do
|
19
|
+
subject.expects(:setup_capybara)
|
20
|
+
subject.expects(:log_in)
|
21
|
+
document = Document.new(url = stub, cookie = stub, {})
|
22
|
+
subject.expects(:document).returns(document)
|
23
|
+
document.expects(:save)
|
24
|
+
|
25
|
+
month = 11
|
26
|
+
year = Time.now.year
|
27
|
+
|
28
|
+
subject.scrape.must_equal "/tmp/utilities/endesa_#{month}_#{year}.pdf"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative '../test_helper'
|
3
|
+
|
4
|
+
module Autility
|
5
|
+
describe VodafoneSpain do
|
6
|
+
subject { VodafoneSpain.new("foo", "bar", 11, "/tmp/utilities") }
|
7
|
+
|
8
|
+
describe ".scrape" do
|
9
|
+
it 'delegates to an instance' do
|
10
|
+
VodafoneSpain.stubs(:new).with("foo", "bar", 11, "/tmp/utilities").returns subject
|
11
|
+
subject.expects(:scrape)
|
12
|
+
|
13
|
+
VodafoneSpain.scrape("foo", "bar", 11, "/tmp/utilities")
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#scrape' do
|
18
|
+
it 'gets the document in' do
|
19
|
+
subject.expects(:setup_capybara)
|
20
|
+
subject.expects(:log_in)
|
21
|
+
document = Document.new(url = stub, cookie = stub, {})
|
22
|
+
subject.expects(:document).returns(document)
|
23
|
+
document.expects(:save)
|
24
|
+
|
25
|
+
month = 11
|
26
|
+
year = Time.now.year
|
27
|
+
|
28
|
+
subject.scrape.must_equal "/tmp/utilities/vodafone_#{month}_#{year}.pdf"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
data/test/test_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,140 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: autility
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Josep M. Bach
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2011-12-14 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: capybara
|
16
|
+
requirement: &70363203567720 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *70363203567720
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: capybara-webkit
|
27
|
+
requirement: &70363203567120 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *70363203567120
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: show_me_the_cookies
|
38
|
+
requirement: &70363203566080 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
type: :runtime
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *70363203566080
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: commander
|
49
|
+
requirement: &70363203565080 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :runtime
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *70363203565080
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: mocha
|
60
|
+
requirement: &70363203564520 !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - ! '>='
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: '0'
|
66
|
+
type: :development
|
67
|
+
prerelease: false
|
68
|
+
version_requirements: *70363203564520
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: minitest
|
71
|
+
requirement: &70363203564040 !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: *70363203564040
|
80
|
+
description: Downloads utility invoices from common spanish firms such as Endesa or
|
81
|
+
Vodafone.
|
82
|
+
email:
|
83
|
+
- josep.m.bach@gmail.com
|
84
|
+
executables:
|
85
|
+
- autility
|
86
|
+
extensions: []
|
87
|
+
extra_rdoc_files: []
|
88
|
+
files:
|
89
|
+
- .gitignore
|
90
|
+
- .rvmrc
|
91
|
+
- Gemfile
|
92
|
+
- Rakefile
|
93
|
+
- Readme.md
|
94
|
+
- autility.gemspec
|
95
|
+
- bin/autility
|
96
|
+
- lib/autility.rb
|
97
|
+
- lib/autility/command.rb
|
98
|
+
- lib/autility/cookie.rb
|
99
|
+
- lib/autility/data.rb
|
100
|
+
- lib/autility/document.rb
|
101
|
+
- lib/autility/endesa.rb
|
102
|
+
- lib/autility/version.rb
|
103
|
+
- lib/autility/vodafone_spain.rb
|
104
|
+
- test/autility/command_test.rb
|
105
|
+
- test/autility/cookie_test.rb
|
106
|
+
- test/autility/document_test.rb
|
107
|
+
- test/autility/endesa_test.rb
|
108
|
+
- test/autility/vodafone_spain_test.rb
|
109
|
+
- test/test_helper.rb
|
110
|
+
homepage: http://github.com/codegram/utilities
|
111
|
+
licenses: []
|
112
|
+
post_install_message:
|
113
|
+
rdoc_options: []
|
114
|
+
require_paths:
|
115
|
+
- lib
|
116
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
117
|
+
none: false
|
118
|
+
requirements:
|
119
|
+
- - ! '>='
|
120
|
+
- !ruby/object:Gem::Version
|
121
|
+
version: '0'
|
122
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
123
|
+
none: false
|
124
|
+
requirements:
|
125
|
+
- - ! '>='
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '0'
|
128
|
+
requirements: []
|
129
|
+
rubyforge_project: utilities
|
130
|
+
rubygems_version: 1.8.10
|
131
|
+
signing_key:
|
132
|
+
specification_version: 3
|
133
|
+
summary: Downloads utility invoices from common spanish firms such as Endesa or Vodafone.
|
134
|
+
test_files:
|
135
|
+
- test/autility/command_test.rb
|
136
|
+
- test/autility/cookie_test.rb
|
137
|
+
- test/autility/document_test.rb
|
138
|
+
- test/autility/endesa_test.rb
|
139
|
+
- test/autility/vodafone_spain_test.rb
|
140
|
+
- test/test_helper.rb
|