autility 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/.rvmrc +47 -0
- data/Gemfile +5 -0
- data/Rakefile +11 -0
- data/Readme.md +45 -0
- data/autility.gemspec +27 -0
- data/bin/autility +49 -0
- data/lib/autility.rb +11 -0
- data/lib/autility/command.rb +40 -0
- data/lib/autility/cookie.rb +18 -0
- data/lib/autility/data.rb +21 -0
- data/lib/autility/document.rb +35 -0
- data/lib/autility/endesa.rb +110 -0
- data/lib/autility/version.rb +3 -0
- data/lib/autility/vodafone_spain.rb +102 -0
- data/test/autility/command_test.rb +21 -0
- data/test/autility/cookie_test.rb +13 -0
- data/test/autility/document_test.rb +19 -0
- data/test/autility/endesa_test.rb +32 -0
- data/test/autility/vodafone_spain_test.rb +32 -0
- data/test/test_helper.rb +5 -0
- metadata +140 -0
data/.gitignore
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1,47 @@
|
|
1
|
+
#!/usr/bin/env bash
|
2
|
+
|
3
|
+
# This is an RVM Project .rvmrc file, used to automatically load the ruby
|
4
|
+
# development environment upon cd'ing into the directory
|
5
|
+
|
6
|
+
# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional.
|
7
|
+
environment_id="1.9.3@utilities"
|
8
|
+
|
9
|
+
#
|
10
|
+
# First we attempt to load the desired environment directly from the environment
|
11
|
+
# file. This is very fast and efficicent compared to running through the entire
|
12
|
+
# CLI and selector. If you want feedback on which environment was used then
|
13
|
+
# insert the word 'use' after --create as this triggers verbose mode.
|
14
|
+
#
|
15
|
+
if [[ -d "${rvm_path:-$HOME/.rvm}/environments" \
|
16
|
+
&& -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
|
17
|
+
then
|
18
|
+
\. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
|
19
|
+
|
20
|
+
if [[ -s ".rvm/hooks/after_use" ]]
|
21
|
+
then
|
22
|
+
. ".rvm/hooks/after_use"
|
23
|
+
fi
|
24
|
+
else
|
25
|
+
# If the environment file has not yet been created, use the RVM CLI to select.
|
26
|
+
if ! rvm --create "$environment_id"
|
27
|
+
then
|
28
|
+
echo "Failed to create RVM environment ''."
|
29
|
+
fi
|
30
|
+
fi
|
31
|
+
|
32
|
+
#
|
33
|
+
# If you use an RVM gemset file to install a list of gems (*.gems), you can have
|
34
|
+
# it be automatically loaded. Uncomment the following and adjust the filename if
|
35
|
+
# necessary.
|
36
|
+
#
|
37
|
+
# filename=".gems"
|
38
|
+
# if [[ -s "$filename" ]] ; then
|
39
|
+
# rvm gemset import "$filename" | grep -v already | grep -v listed | grep -v complete | sed '/^$/d'
|
40
|
+
# fi
|
41
|
+
|
42
|
+
#
|
43
|
+
# If you use bundler and would like to run bundle each time you enter the
|
44
|
+
# directory, you can uncomment the following code.
|
45
|
+
#
|
46
|
+
export PATH="./bin:$PATH"
|
47
|
+
#
|
data/Gemfile
ADDED
data/Rakefile
ADDED
data/Readme.md
ADDED
@@ -0,0 +1,45 @@
|
|
1
|
+
# autility
|
2
|
+
|
3
|
+
A scraper that automatically downloads PDF invoices from utility vendors. That's it.
|
4
|
+
|
5
|
+
It will focus mainly on spanish vendors, but pull requests will be accepted
|
6
|
+
to support as many vendors as possible.
|
7
|
+
|
8
|
+
For now it supports downloading invoices from Endesa and Vodafone Spain.
|
9
|
+
|
10
|
+
## Install
|
11
|
+
|
12
|
+
$ gem install autility
|
13
|
+
|
14
|
+
## Usage
|
15
|
+
|
16
|
+
$ autility VENDOR --user USER --password PASSWORD [--month MONTH] [--output-folder OUTPUT_FOLDER]
|
17
|
+
$ autility --help
|
18
|
+
$ autility --help endesa
|
19
|
+
$ autility --help vodafone
|
20
|
+
|
21
|
+
## Examples
|
22
|
+
|
23
|
+
Downloading the Endesa invoice for the current month:
|
24
|
+
|
25
|
+
$ autility endesa --user john --password doe
|
26
|
+
|
27
|
+
Downloading the Vodafone invoice for the past October and save it to ~/billing:
|
28
|
+
|
29
|
+
$ autility vodafone --user john --password doe --month 10 --output-folder ~/billing
|
30
|
+
|
31
|
+
## Parsing the output
|
32
|
+
|
33
|
+
The output is simple, thus parseable by UNIX tools such as grep or awk. It
|
34
|
+
looks like this:
|
35
|
+
|
36
|
+
Path: /Users/john/billing/vodafone_10_2011.pdf
|
37
|
+
|
38
|
+
For example, to open the file right after downloading it:
|
39
|
+
|
40
|
+
$ open `autility endesa --user john --password doe | grep Path: | awk '{print $2}'`
|
41
|
+
|
42
|
+
## License
|
43
|
+
|
44
|
+
Released under the MIT License.
|
45
|
+
Copyright 2011 [Codegram Technologies](http://codegram.com)
|
data/autility.gemspec
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "autility/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "autility"
|
7
|
+
s.version = Autility::VERSION
|
8
|
+
s.authors = ["Josep M. Bach"]
|
9
|
+
s.email = ["josep.m.bach@gmail.com"]
|
10
|
+
s.homepage = "http://github.com/codegram/utilities"
|
11
|
+
s.summary = %q{Downloads utility invoices from common spanish firms such as Endesa or Vodafone.}
|
12
|
+
s.description = %q{Downloads utility invoices from common spanish firms such as Endesa or Vodafone.}
|
13
|
+
|
14
|
+
s.rubyforge_project = "utilities"
|
15
|
+
|
16
|
+
s.files = `git ls-files`.split("\n")
|
17
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
18
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
|
21
|
+
s.add_runtime_dependency "capybara"
|
22
|
+
s.add_runtime_dependency "capybara-webkit"
|
23
|
+
s.add_runtime_dependency "show_me_the_cookies"
|
24
|
+
s.add_runtime_dependency "commander"
|
25
|
+
s.add_development_dependency "mocha"
|
26
|
+
s.add_development_dependency "minitest"
|
27
|
+
end
|
data/bin/autility
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rubygems'
|
4
|
+
require 'commander/import'
|
5
|
+
require_relative '../lib/autility'
|
6
|
+
|
7
|
+
program :version, Autility::VERSION
|
8
|
+
program :description, 'Autility is a CLI interface to download invoices from various utility vendors, such as Vodafone or Endesa.'
|
9
|
+
|
10
|
+
command :endesa do |c|
|
11
|
+
c.syntax = 'autility endesa [options]'
|
12
|
+
c.summary = 'Downloads invoices from Endesa.'
|
13
|
+
c.description = 'Downloads invoices from Endesa.'
|
14
|
+
c.example 'Download the invoice for the current month of user "foo" with password "bar"', 'autility endesa --user foo --password bar'
|
15
|
+
c.option '--user USER', String, 'The username for Endesa Online'
|
16
|
+
c.option '--password PASSWORD', String, 'The password for Endesa Online'
|
17
|
+
c.option '--month MONTH_NUMBER', Integer, 'The month (e.g. 5, 9, 12...). By default it\'s the current month'
|
18
|
+
c.option '--output-folder FOLDER_NAME', String, 'The output folder to store the PDF in. Defaults to ~/utilities'
|
19
|
+
c.action do |args, options|
|
20
|
+
options.default :month => Time.now.month
|
21
|
+
options.default :output_folder => File.expand_path("~/utilities")
|
22
|
+
|
23
|
+
raise "Both --user and --password are required options." unless options.user && options.password
|
24
|
+
|
25
|
+
result = Autility::Endesa.scrape(options.user, options.password, options.month, options.output_folder)
|
26
|
+
puts "Path: #{result}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
command :vodafone do |c|
|
31
|
+
c.syntax = 'autility vodafone [options]'
|
32
|
+
c.summary = 'Downloads invoices from Vodafone.'
|
33
|
+
c.description = 'Downloads invoices from Vodafone.'
|
34
|
+
c.example 'Download the invoice for the current month of user "foo" with password "bar"', 'utilities vodafone --user foo --password bar'
|
35
|
+
c.option '--user USER', String, 'The username for Vodafone Online'
|
36
|
+
c.option '--password PASSWORD', String, 'The password for Vodafone Online'
|
37
|
+
c.option '--month MONTH_NUMBER', Integer, 'The month (e.g. 5, 9, 12...). By default it\'s the current month'
|
38
|
+
c.option '--output-folder FOLDER_NAME', String, 'The output folder to store the PDF in. Defaults to ~/utilities'
|
39
|
+
c.action do |args, options|
|
40
|
+
options.default :month => Time.now.month
|
41
|
+
options.default :output_folder => File.expand_path("~/utilities")
|
42
|
+
|
43
|
+
raise "Both --user and --password are required options." unless options.user && options.password
|
44
|
+
|
45
|
+
result = Autility::VodafoneSpain.scrape(options.user, options.password, options.month, options.output_folder)
|
46
|
+
puts "Path: #{result}"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
data/lib/autility.rb
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
require_relative 'autility/version'
|
2
|
+
require_relative 'autility/document'
|
3
|
+
require_relative 'autility/command'
|
4
|
+
require_relative 'autility/cookie'
|
5
|
+
|
6
|
+
require_relative 'autility/endesa'
|
7
|
+
require_relative 'autility/vodafone_spain'
|
8
|
+
|
9
|
+
module Autility
|
10
|
+
end
|
11
|
+
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Autility
|
2
|
+
# Public: A Command is a system cURL command builder.
|
3
|
+
#
|
4
|
+
# Examples
|
5
|
+
#
|
6
|
+
# command = Command.build(url, params, cookie, "foo.pdf")
|
7
|
+
# # => "curl --data ..."
|
8
|
+
#
|
9
|
+
module Command
|
10
|
+
# Public: Builds a command to fetch and save a remote document via cURL.
|
11
|
+
#
|
12
|
+
# url - The String url where we can get the document
|
13
|
+
# method - the String HTTP method to use.
|
14
|
+
# cookie - the session Cookie needed to access the url
|
15
|
+
# params - The Array of POST params needed to fetch it
|
16
|
+
# path - the String path to save the document to.
|
17
|
+
#
|
18
|
+
# Returns the String command ready to execute.
|
19
|
+
def build(url, method, cookie, params, path)
|
20
|
+
out = "curl"
|
21
|
+
|
22
|
+
if method == :post
|
23
|
+
out = "curl --data \""
|
24
|
+
|
25
|
+
out << params.to_a.map do |name, value|
|
26
|
+
"#{name}=#{value}"
|
27
|
+
end.join("&")
|
28
|
+
|
29
|
+
out << "\""
|
30
|
+
end
|
31
|
+
|
32
|
+
out << " #{cookie.to_command}" if cookie
|
33
|
+
out << " \"#{url}\""
|
34
|
+
out << " -o "
|
35
|
+
out << path
|
36
|
+
out
|
37
|
+
end
|
38
|
+
module_function :build
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Autility
|
2
|
+
# Public: Represents a browser cookie.
|
3
|
+
#
|
4
|
+
# Examples
|
5
|
+
#
|
6
|
+
# cookie = Cookie.new("JSESSIONID", "123")
|
7
|
+
# cookie.to_command
|
8
|
+
# # => "--cookie \"JSESSIONID=123\""
|
9
|
+
#
|
10
|
+
class Cookie < Struct.new(:name, :value)
|
11
|
+
# Public: Converts the cookie to a cURL option.
|
12
|
+
#
|
13
|
+
# Returns the String cURL option.
|
14
|
+
def to_command
|
15
|
+
%Q(--cookie "#{name}=#{value}")
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Autility
|
2
|
+
# Public: Represents a remote document we are interested in.
|
3
|
+
#
|
4
|
+
# Examples
|
5
|
+
#
|
6
|
+
# document = Document.new(url, params, cookie)
|
7
|
+
# document.save("~/utilities/endesa_11_2012.pdf")
|
8
|
+
#
|
9
|
+
class Document
|
10
|
+
# Public: Initializes a new document.
|
11
|
+
#
|
12
|
+
# url - The String url where we can get the document
|
13
|
+
# params - The Array of POST params needed to fetch it
|
14
|
+
# cookie - the session Cookie needed to access the url
|
15
|
+
def initialize(url, params, cookie)
|
16
|
+
@url = url
|
17
|
+
@params = params
|
18
|
+
@cookie = cookie
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Autility
|
2
|
+
# Public: Represents a remote document we are interested in.
|
3
|
+
#
|
4
|
+
# Examples
|
5
|
+
#
|
6
|
+
# document = Document.new(url, :post, cookie, params)
|
7
|
+
# document.save("~/utilities/endesa_11_2012.pdf")
|
8
|
+
#
|
9
|
+
class Document
|
10
|
+
attr_reader :url, :params, :cookie
|
11
|
+
|
12
|
+
# Public: Initializes a new document.
|
13
|
+
#
|
14
|
+
# url - The String url where we can get the document
|
15
|
+
# method - The String HTTP method to use.
|
16
|
+
# cookie - the session Cookie needed to access the url
|
17
|
+
# params - The Array of POST params needed to fetch it
|
18
|
+
def initialize(url, method=:get, cookie=nil, params={})
|
19
|
+
@url = url
|
20
|
+
@method = method
|
21
|
+
@params = params
|
22
|
+
@cookie = cookie
|
23
|
+
end
|
24
|
+
|
25
|
+
# Public: Fetches the document and saves it to a particular path.
|
26
|
+
#
|
27
|
+
# path - The String path where we will save the document.
|
28
|
+
#
|
29
|
+
# Returns nothing.
|
30
|
+
def save(path)
|
31
|
+
command = Command.build(@url, @method, @cookie, @params, path)
|
32
|
+
system(command)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,110 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'capybara'
|
3
|
+
require 'capybara/dsl'
|
4
|
+
require 'show_me_the_cookies'
|
5
|
+
require 'fileutils'
|
6
|
+
|
7
|
+
module Autility
|
8
|
+
# Public: A scraper for all the utility invoices i Endesa.
|
9
|
+
#
|
10
|
+
# Examples
|
11
|
+
#
|
12
|
+
# # Download the invoice from September this year and store it in /tmp.
|
13
|
+
# Endesa.scrape("user", "password", 9, "/tmp") # Download all invoices from September this year
|
14
|
+
#
|
15
|
+
class Endesa
|
16
|
+
include Capybara::DSL
|
17
|
+
include ShowMeTheCookies
|
18
|
+
|
19
|
+
# Public: Instantiates a new scraper and fires it to download the utility
|
20
|
+
# invoices from Endesa.
|
21
|
+
#
|
22
|
+
# Returns nothing.
|
23
|
+
def self.scrape(*args)
|
24
|
+
new(*args).scrape
|
25
|
+
end
|
26
|
+
|
27
|
+
def initialize(user, password, month, folder)
|
28
|
+
@user = user
|
29
|
+
@password = password
|
30
|
+
@month = month
|
31
|
+
@folder = folder
|
32
|
+
@year = Time.now.year
|
33
|
+
end
|
34
|
+
|
35
|
+
# Public: Scrapes the endesa website and gets the invoice for the current
|
36
|
+
# month, saving it to @folder.
|
37
|
+
#
|
38
|
+
# Returns the String path of the saved document.
|
39
|
+
def scrape
|
40
|
+
setup_capybara
|
41
|
+
log_in
|
42
|
+
|
43
|
+
FileUtils.mkdir_p(@folder)
|
44
|
+
filename = "#{@folder}/endesa_#{month}_#{@year}.pdf"
|
45
|
+
document.save(filename)
|
46
|
+
filename
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
# Internal: Logs in to the Endesa website.
|
52
|
+
#
|
53
|
+
# Returns nothing.
|
54
|
+
def log_in
|
55
|
+
visit "/gp/login.do"
|
56
|
+
fill_in "id", :with => @user
|
57
|
+
fill_in "clave", :with => @password
|
58
|
+
first("#entrar").click
|
59
|
+
end
|
60
|
+
|
61
|
+
# Internal: Gets the latest invoice and returns it as a Document (not
|
62
|
+
# fetched yet).
|
63
|
+
#
|
64
|
+
# Returns the Document to be fetched.
|
65
|
+
def document
|
66
|
+
@document ||= begin
|
67
|
+
visit "/gp/GenericForward.do?TO=ListadoFacturas"
|
68
|
+
links = all("#capa_contenido form table table td > a")
|
69
|
+
invoices = lambda { |link| link.text =~ %r{\d{2}/\d{2}/\d{2}} }
|
70
|
+
|
71
|
+
invoice = links.select(&invoices).detect do |link|
|
72
|
+
link.text =~ %r{/#{month}/}
|
73
|
+
# link.text =~ %r{/11/}
|
74
|
+
end
|
75
|
+
|
76
|
+
raise "Endesa invoice for month #{month} is not available yet." unless invoice
|
77
|
+
|
78
|
+
cfactura, cd_contrext = invoice[:onclick].scan(/'(\w+)','(\d+)'/).flatten
|
79
|
+
url = "https://www.gp.endesaonline.com/gp/obtenerFacturaPDF.do?cfactura=#{cfactura}&cd_contrext=#{cd_contrext}"
|
80
|
+
cookie = Cookie.new("JSESSIONID", get_me_the_cookie("JSESSIONID")[:value])
|
81
|
+
Document.new(url, :post, cookie)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Internal: Hack for ShowMeTheCookies.
|
86
|
+
def driver
|
87
|
+
Capybara.current_driver
|
88
|
+
end
|
89
|
+
|
90
|
+
# Internal: Hack for ShowMeTheCookies.
|
91
|
+
def drivers
|
92
|
+
send :adapters
|
93
|
+
end
|
94
|
+
|
95
|
+
# Internal: Returns the String current month padded with zeros to the left.
|
96
|
+
def month
|
97
|
+
@month.to_s.rjust(2, '0')
|
98
|
+
end
|
99
|
+
|
100
|
+
# Internal: Sets the configuration for capybara to work with the Endesa
|
101
|
+
# website.
|
102
|
+
#
|
103
|
+
# Returns nothing.
|
104
|
+
def setup_capybara
|
105
|
+
Capybara.run_server = false
|
106
|
+
Capybara.current_driver = :selenium
|
107
|
+
Capybara.app_host = 'https://www.gp.endesaonline.com'
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'capybara'
|
3
|
+
require 'capybara/dsl'
|
4
|
+
require 'fileutils'
|
5
|
+
|
6
|
+
module Autility
|
7
|
+
# Public: A scraper for all the utility invoices in VodafoneSpain.
|
8
|
+
#
|
9
|
+
# Examples
|
10
|
+
#
|
11
|
+
# # Download the invoice from September this year and store it in /tmp.
|
12
|
+
# VodafoneSpain.scrape("user", "password", 9, "/tmp") # Download all invoices from September this year
|
13
|
+
#
|
14
|
+
class VodafoneSpain
|
15
|
+
include Capybara::DSL
|
16
|
+
|
17
|
+
# Public: Instantiates a new scraper and fires it to download the utility
|
18
|
+
# invoices from VodafoneSpain.
|
19
|
+
#
|
20
|
+
# Returns nothing.
|
21
|
+
def self.scrape(*args)
|
22
|
+
new(*args).scrape
|
23
|
+
end
|
24
|
+
|
25
|
+
def initialize(user, password, month, folder)
|
26
|
+
@user = user
|
27
|
+
@password = password
|
28
|
+
@month = month
|
29
|
+
@folder = folder
|
30
|
+
@year = Time.now.year
|
31
|
+
end
|
32
|
+
|
33
|
+
# Public: Scrapes the vodafone website and gets the invoice for the current
|
34
|
+
# month, saving it to @folder.
|
35
|
+
#
|
36
|
+
# Returns the String path of the saved document.
|
37
|
+
def scrape
|
38
|
+
setup_capybara
|
39
|
+
log_in
|
40
|
+
|
41
|
+
FileUtils.mkdir_p(@folder)
|
42
|
+
filename = "#{@folder}/vodafone_#{month}_#{@year}.pdf"
|
43
|
+
document.save(filename)
|
44
|
+
filename
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
# Internal: Logs in to the VodafoneSpain website.
|
50
|
+
#
|
51
|
+
# Returns nothing.
|
52
|
+
def log_in
|
53
|
+
visit "/cpar/do/home/loginVodafone?type=empresas"
|
54
|
+
fill_in "user", :with => @user
|
55
|
+
fill_in "pass", :with => @password
|
56
|
+
first(".botonera_entrar #login").click
|
57
|
+
end
|
58
|
+
|
59
|
+
# Internal: Gets the latest invoice and returns it as a Document (not
|
60
|
+
# fetched yet).
|
61
|
+
#
|
62
|
+
# Returns the Document to be fetched.
|
63
|
+
def document
|
64
|
+
@document ||= begin
|
65
|
+
invoice = nil
|
66
|
+
|
67
|
+
visit "https://areaclientes.vodafone.es/cwgc/do/ebilling/get?ebplink=/tbmb/main/dashboard/invoices.do"
|
68
|
+
|
69
|
+
within "table#recent_invoices_tab1" do
|
70
|
+
row = all("tr").detect do |tr|
|
71
|
+
tr.text =~ /-#{month}-#{@year}/
|
72
|
+
end
|
73
|
+
invoice = row.find('#scriptOnly a')
|
74
|
+
end
|
75
|
+
|
76
|
+
raise "VodafoneSpain invoice for month #{month} is not available yet." unless invoice
|
77
|
+
|
78
|
+
url = "https://b2b.ebilling.vodafone.es/tbmb"
|
79
|
+
url += invoice[:onclick].gsub("downloadDocument('", '')[0..-2]
|
80
|
+
|
81
|
+
p url
|
82
|
+
|
83
|
+
Document.new(url)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
# Internal: Returns the String current month padded with zeros to the left.
|
88
|
+
def month
|
89
|
+
@month.to_s.rjust(2, '0')
|
90
|
+
end
|
91
|
+
|
92
|
+
# Internal: Sets the configuration for capybara to work with the VodafoneSpain
|
93
|
+
# website.
|
94
|
+
#
|
95
|
+
# Returns nothing.
|
96
|
+
def setup_capybara
|
97
|
+
Capybara.run_server = false
|
98
|
+
Capybara.current_driver = :selenium
|
99
|
+
Capybara.app_host = 'https://canalonline.vodafone.es'
|
100
|
+
end
|
101
|
+
end
|
102
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module Autility
|
4
|
+
describe Command do
|
5
|
+
let(:url) { "http://example.com/some_document.pdf" }
|
6
|
+
let(:params) do
|
7
|
+
{ foo: "bar", baz: "yeah" }
|
8
|
+
end
|
9
|
+
let(:cookie) { stub(to_command: "--cookie \"JSESSIONID=123\"") }
|
10
|
+
|
11
|
+
describe ".build" do
|
12
|
+
it 'builds a system command' do
|
13
|
+
command = Command.build(url, :post, cookie, params, "foo.pdf")
|
14
|
+
|
15
|
+
command.must_equal(
|
16
|
+
%Q{curl --data "foo=bar&baz=yeah" --cookie "JSESSIONID=123" "http://example.com/some_document.pdf" -o foo.pdf}
|
17
|
+
)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module Autility
|
4
|
+
describe Cookie do
|
5
|
+
subject { Cookie.new("JSESSIONID", "123") }
|
6
|
+
|
7
|
+
describe "#to_command" do
|
8
|
+
it 'converts the cookie to a cURL option' do
|
9
|
+
subject.to_command.must_equal "--cookie \"JSESSIONID=123\""
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module Autility
|
4
|
+
describe Document do
|
5
|
+
describe "#save" do
|
6
|
+
let(:url) { stub }
|
7
|
+
let(:params) { stub }
|
8
|
+
let(:cookie) { stub }
|
9
|
+
|
10
|
+
subject { Document.new(url, :get, cookie, params) }
|
11
|
+
|
12
|
+
it "fetches the document and saves it to a path" do
|
13
|
+
Command.stubs(:build).with(url, :get, cookie, params, "foo.pdf").returns command = stub
|
14
|
+
subject.expects(:system).with command
|
15
|
+
subject.save("foo.pdf")
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative '../test_helper'
|
3
|
+
|
4
|
+
module Autility
|
5
|
+
describe Endesa do
|
6
|
+
subject { Endesa.new("foo", "bar", 11, "/tmp/utilities") }
|
7
|
+
|
8
|
+
describe ".scrape" do
|
9
|
+
it 'delegates to an instance' do
|
10
|
+
Endesa.stubs(:new).with("foo", "bar", 11, "/tmp/utilities").returns subject
|
11
|
+
subject.expects(:scrape)
|
12
|
+
|
13
|
+
Endesa.scrape("foo", "bar", 11, "/tmp/utilities")
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#scrape' do
|
18
|
+
it 'gets the document in' do
|
19
|
+
subject.expects(:setup_capybara)
|
20
|
+
subject.expects(:log_in)
|
21
|
+
document = Document.new(url = stub, cookie = stub, {})
|
22
|
+
subject.expects(:document).returns(document)
|
23
|
+
document.expects(:save)
|
24
|
+
|
25
|
+
month = 11
|
26
|
+
year = Time.now.year
|
27
|
+
|
28
|
+
subject.scrape.must_equal "/tmp/utilities/endesa_#{month}_#{year}.pdf"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require_relative '../test_helper'
|
3
|
+
|
4
|
+
module Autility
|
5
|
+
describe VodafoneSpain do
|
6
|
+
subject { VodafoneSpain.new("foo", "bar", 11, "/tmp/utilities") }
|
7
|
+
|
8
|
+
describe ".scrape" do
|
9
|
+
it 'delegates to an instance' do
|
10
|
+
VodafoneSpain.stubs(:new).with("foo", "bar", 11, "/tmp/utilities").returns subject
|
11
|
+
subject.expects(:scrape)
|
12
|
+
|
13
|
+
VodafoneSpain.scrape("foo", "bar", 11, "/tmp/utilities")
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#scrape' do
|
18
|
+
it 'gets the document in' do
|
19
|
+
subject.expects(:setup_capybara)
|
20
|
+
subject.expects(:log_in)
|
21
|
+
document = Document.new(url = stub, cookie = stub, {})
|
22
|
+
subject.expects(:document).returns(document)
|
23
|
+
document.expects(:save)
|
24
|
+
|
25
|
+
month = 11
|
26
|
+
year = Time.now.year
|
27
|
+
|
28
|
+
subject.scrape.must_equal "/tmp/utilities/vodafone_#{month}_#{year}.pdf"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
data/test/test_helper.rb
ADDED
metadata
ADDED
@@ -0,0 +1,140 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: autility
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Josep M. Bach
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2011-12-14 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: capybara
|
16
|
+
requirement: &70363203567720 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *70363203567720
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: capybara-webkit
|
27
|
+
requirement: &70363203567120 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *70363203567120
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: show_me_the_cookies
|
38
|
+
requirement: &70363203566080 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
type: :runtime
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *70363203566080
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: commander
|
49
|
+
requirement: &70363203565080 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :runtime
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *70363203565080
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: mocha
|
60
|
+
requirement: &70363203564520 !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - ! '>='
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: '0'
|
66
|
+
type: :development
|
67
|
+
prerelease: false
|
68
|
+
version_requirements: *70363203564520
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: minitest
|
71
|
+
requirement: &70363203564040 !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: *70363203564040
|
80
|
+
description: Downloads utility invoices from common spanish firms such as Endesa or
|
81
|
+
Vodafone.
|
82
|
+
email:
|
83
|
+
- josep.m.bach@gmail.com
|
84
|
+
executables:
|
85
|
+
- autility
|
86
|
+
extensions: []
|
87
|
+
extra_rdoc_files: []
|
88
|
+
files:
|
89
|
+
- .gitignore
|
90
|
+
- .rvmrc
|
91
|
+
- Gemfile
|
92
|
+
- Rakefile
|
93
|
+
- Readme.md
|
94
|
+
- autility.gemspec
|
95
|
+
- bin/autility
|
96
|
+
- lib/autility.rb
|
97
|
+
- lib/autility/command.rb
|
98
|
+
- lib/autility/cookie.rb
|
99
|
+
- lib/autility/data.rb
|
100
|
+
- lib/autility/document.rb
|
101
|
+
- lib/autility/endesa.rb
|
102
|
+
- lib/autility/version.rb
|
103
|
+
- lib/autility/vodafone_spain.rb
|
104
|
+
- test/autility/command_test.rb
|
105
|
+
- test/autility/cookie_test.rb
|
106
|
+
- test/autility/document_test.rb
|
107
|
+
- test/autility/endesa_test.rb
|
108
|
+
- test/autility/vodafone_spain_test.rb
|
109
|
+
- test/test_helper.rb
|
110
|
+
homepage: http://github.com/codegram/utilities
|
111
|
+
licenses: []
|
112
|
+
post_install_message:
|
113
|
+
rdoc_options: []
|
114
|
+
require_paths:
|
115
|
+
- lib
|
116
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
117
|
+
none: false
|
118
|
+
requirements:
|
119
|
+
- - ! '>='
|
120
|
+
- !ruby/object:Gem::Version
|
121
|
+
version: '0'
|
122
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
123
|
+
none: false
|
124
|
+
requirements:
|
125
|
+
- - ! '>='
|
126
|
+
- !ruby/object:Gem::Version
|
127
|
+
version: '0'
|
128
|
+
requirements: []
|
129
|
+
rubyforge_project: utilities
|
130
|
+
rubygems_version: 1.8.10
|
131
|
+
signing_key:
|
132
|
+
specification_version: 3
|
133
|
+
summary: Downloads utility invoices from common spanish firms such as Endesa or Vodafone.
|
134
|
+
test_files:
|
135
|
+
- test/autility/command_test.rb
|
136
|
+
- test/autility/cookie_test.rb
|
137
|
+
- test/autility/document_test.rb
|
138
|
+
- test/autility/endesa_test.rb
|
139
|
+
- test/autility/vodafone_spain_test.rb
|
140
|
+
- test/test_helper.rb
|