traffic-scraper 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
data/.gitignore ADDED
@@ -0,0 +1,4 @@
1
+ *.gem
2
+ .bundle
3
+ Gemfile.lock
4
+ pkg/*
data/.travis.yml ADDED
@@ -0,0 +1,11 @@
1
+ script: "bundle exec rspec spec"
2
+ notifications:
3
+ email:
4
+ - mail@rodrigoflores.org
5
+ rvm:
6
+ - 1.9.2
7
+ - 1.8.7
8
+ - ree
9
+ - rbx
10
+ - rbx-2.0
11
+ - jruby
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in sp-traffic-scraper.gemspec
4
+ gemspec
data/README.md ADDED
@@ -0,0 +1,12 @@
1
+ # Traffic Scraper #
2
+
3
+ ![Build Status](https://secure.travis-ci.org/rodrigoflores/Traffic-Scraper.png)
4
+
5
+ A gem to retrieve Sao Paulo traffic information from CET-SP website. It retrieves the HTML and parsed it to get the information.
6
+
7
+ ## How to use ##
8
+
9
+ Only two methods are available right now:
10
+
11
+ Traffic::Scrapper.overall_traffic #=> Returns the amount of overall traffic (in km) that we currently have
12
+ Traffic::Scrapper.zone_traffic #=> Returns a Hash with the amount of traffic in each zone (in km) that we currently have.
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require "bundler/gem_tasks"
@@ -0,0 +1,5 @@
1
+ module Traffic
2
+ class Scraper
3
+ VERSION = "0.0.1"
4
+ end
5
+ end
@@ -0,0 +1,48 @@
1
+ require 'open-uri'
2
+ require 'nokogiri'
3
+ require "traffic-scraper/version"
4
+
5
+ module Traffic
6
+ class Scraper
7
+ class << self
8
+ TRAFFIC_PAGE_URL = "http://cetsp1.cetsp.com.br/monitransmapa/agora/"
9
+
10
+ # Returns the amount of overall traffic we currently have
11
+ #
12
+ # @return [Integer] the amount of traffic
13
+ def overall_traffic
14
+ page = Nokogiri::HTML(open(TRAFFIC_PAGE_URL))
15
+ page.css("#lentidao b").inner_html.to_i
16
+ rescue
17
+ raise CouldNotRetrievePageError
18
+ end
19
+
20
+ # Returns the amount of overall traffic we currently have
21
+ #
22
+ # @return [Hash] containing all the traffic information for all zones
23
+ def zone_traffic
24
+ page = Nokogiri::HTML(open(TRAFFIC_PAGE_URL))
25
+
26
+ result_hash = {}
27
+
28
+ { "Norte" => :north ,
29
+ "Sul" => :south,
30
+ "Leste" => :east,
31
+ "Oeste" => :west,
32
+ "Centro" => :downtown
33
+ }.each do |zone, zone_translation|
34
+ page.css("##{zone}Lentidao").inner_html =~ /^(\d+)\s*km$/
35
+
36
+ result_hash[zone_translation] = $1.to_i
37
+ end
38
+
39
+ result_hash
40
+ rescue
41
+ raise CouldNotRetrievePageError
42
+ end
43
+ end
44
+ end
45
+ end
46
+
47
+ class CouldNotRetrievePageError < RuntimeError
48
+ end
@@ -0,0 +1,154 @@
1
+
2
+
3
+ <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
4
+ <html>
5
+ <head>
6
+ <title>CET - Tr&acirc;nsito Agora</title>
7
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
8
+ <meta http-equiv="Refresh" Content="600">
9
+ <script language="JavaScript" type="text/JavaScript">
10
+ <!--
11
+ function MM_reloadPage(init) { //reloads the window if Nav4 resized
12
+ if (init==true) with (navigator) {if ((appName=="Netscape")&&(parseInt(appVersion)==4)) {
13
+ document.MM_pgW=innerWidth; document.MM_pgH=innerHeight; onresize=MM_reloadPage; }}
14
+ else if (innerWidth!=document.MM_pgW || innerHeight!=document.MM_pgH) location.reload();
15
+ }
16
+ MM_reloadPage(true);
17
+ //-->
18
+ </script>
19
+ <style type="text/css">
20
+ <!--
21
+ .lentidao {
22
+ font-family: Verdana, Tahoma, Arial;
23
+ font-weight: normal;
24
+ color: #CC0000;
25
+ font-size: 14px;
26
+ text-align: left;
27
+ }
28
+ .hora {
29
+ font-family: Verdana, Tahoma, Arial;
30
+ font-weight: bold;
31
+ color: #999999;
32
+ font-size: 18px;
33
+ text-align: left;
34
+ }
35
+ .noticia {
36
+ font-family: Verdana, Tahoma, Arial;
37
+ font-size: 10px;
38
+ color: #990000;
39
+ text-align: left;
40
+ }
41
+
42
+ -->
43
+ </style>
44
+ <style type="text/css">
45
+ <!--
46
+ .dados1 {
47
+ font-family: Arial, Helvetica, sans-serif;
48
+ font-weight: normal;
49
+ color: #B33A00;
50
+ font-size: 11px;
51
+ text-align: right;
52
+ }
53
+
54
+ .dados2 {
55
+ font-family: Verdana, Tahoma, Arial;
56
+ font-weight: bold;
57
+ color: #FFFFCC;
58
+ font-size: 10px;
59
+ text-align: left;
60
+ }
61
+ #dicasTXT {
62
+ font-family: Verdana, Tahoma;
63
+ font-size: 11px;
64
+ color: #003366;
65
+ text-align: left;
66
+ overflow: auto;
67
+ }
68
+ -->
69
+ </style>
70
+ <style type="text/css">
71
+ <!--
72
+ .dados3 {
73
+ font-family: Verdana, Tahoma, Arial;
74
+ font-weight: normal;
75
+ color: #006699;
76
+ font-size: 10px;
77
+ text-align: left;
78
+ }
79
+ -->
80
+ </style>
81
+ </head>
82
+ <body>
83
+
84
+ <script language=javascript>
85
+
86
+ function OpenWindow( pagina, largura, altura, janela )
87
+ {
88
+ window.status = "";
89
+
90
+ strFeatures = "top=100,left=200,width=" + largura + ",height=" + altura + ",toolbar=no,"
91
+ + "menubar=no,location=no,directories=no,resizable=1,status=no,scrollbars=1";
92
+ objNewWindow = window.open(pagina ,janela , strFeatures);
93
+ objNewWindow.focus()
94
+ }
95
+ </script>
96
+
97
+ <center>
98
+ <div id="fundo" style="position:absolute; width:200px; height:348px; z-index:1; left: 50px; top: 25px;">
99
+ <!--div id="fundo" style="position:relative; width:200px; height:115px; z-index:1; left: 20px; top: 19px;"-->
100
+ <!--<img src="transitoagora.jpg" width="597" height="348" border="0">-->
101
+ <img src="versao163.jpg" width="597" height="348" usemap="#transitoagora" border="0" >
102
+ <map name="transitoagora">
103
+ <area shape="poly" coords="222,135,223,122,242,108,245,90,235,67,229,60,221,61,208,30,218,24,226,26,238,16,257,19,274,37,283,32,291,38,307,34,320,15,343,2,350,4,340,25,334,54,341,80,360,63,387,65,394,68,410,59,417,67,424,61,428,80,424,99,424,129,407,153,360,136,339,138,332,135,330,151,314,161,305,178,315,188,299,199,299,212,310,224,310,244,308,254,314,270,318,300,321,322,308,330,300,340,281,334,268,336,255,336,240,344,245,330,229,313,220,296,222,280,237,284,236,259,243,252,225,240,239,215,226,205,224,178,225,152,245,132" href="http://cetsp1.cetsp.com.br/monitransmapa/painel" title="Ver detalhes">
104
+ <area shape="rect" coords="428,242,589,281" href="http://cetsp1.cetsp.com.br/monitransmapa/painel/default.asp?regiao=2" title="Ver detalhes da Zona Sul" >
105
+ <area shape="rect" coords="428,200,589,239" href="http://cetsp1.cetsp.com.br/monitransmapa/painel/default.asp?regiao=4" title="Ver detalhes da Zona Oeste" >
106
+ <area shape="rect" coords="428,154,589,193" href="http://cetsp1.cetsp.com.br/monitransmapa/painel/default.asp?regiao=1" title="Ver detalhes do Centro" >
107
+ <area shape="rect" coords="428,113,589,152" href="http://cetsp1.cetsp.com.br/monitransmapa/painel/default.asp?regiao=5" title="Ver detalhes da Zona Leste" >
108
+ <area shape="rect" coords="428,72,589,111" href="http://cetsp1.cetsp.com.br/monitransmapa/painel/default.asp?regiao=3" title="Ver detalhes da Zona Norte" >
109
+ </map>
110
+ <div id="bTendencias" style="position:absolute; width:29px; height:23px; z-index:1; left: 445px; top: 293px; cursor:hand"><a href="javascript:OpenWindow('http://cetsp1.cetsp.com.br/monitransmapa/IMG1/eixos.asp', '790', '440','eixos');"><img src="img/bTendencia2.jpg" width="29" height="22" border="0" title="Tend�ncia por Eixo"></a></div>
111
+ <div id="bGraficos" style="position:absolute; width:29px; height:23px; z-index:1; left: 475px; top: 293px; cursor:hand"><a href="javascript:OpenWindow('graficolimite.asp', '760', '535','grafico');"><img src="img/bGraficos2.jpg" width="28" height="22" border="0" title="Gr�fico de Lentid�o"></a></div>
112
+ <div id="bLentidoes" style="position:absolute; width:29px; height:23px; z-index:1; left: 505px; top: 293px; cursor:hand"><a href="javascript:OpenWindow('http://cetsp1.cetsp.com.br/monitransmapa/IMG1/lentidao.asp', '790', '600','lentidao');"><img src="img/bLentidao2.jpg" width="28" height="22" border="0" title="Lentid�o por Corredor"></a></div>
113
+ <div id="bOcorrencias" style="position:absolute; width:29px; height:23px; z-index:1; left: 535px; top: 293px; cursor:hand"><a href="javascript:OpenWindow('http://cetsp1.cetsp.com.br/monitransmapa/IMG1/ocorrencias.asp', '790', '660','ocorrencias');"><img src="img/bOcorrencias2.jpg" width="28" height="22" border="0" title="Ocorr�ncias"></a></div>
114
+ <div id="bAjuda" style="position:absolute; width:28px; height:23px; z-index:1; left: 565px; top: 293px; cursor:hand"><a href="javascript:OpenWindow('ajuda.htm', '790', '660','ajuda');"><img src="img/bAjuda2.jpg" width="27" height="22" border="0" title="P�gina de Ajuda"></a></div>
115
+ <div id="hora" style="position:absolute; width:52px; height:19px; z-index:2; left: 17px; top: 72px; visibility: inherit;" class="hora"><b>10h44m</b></div>
116
+ <div id="tamanhoTotal" align="right" style="position:absolute; width:50px; height:19px; z-index:2; left: 17px; top: 114px; visibility: inherit;" class="dados3"><b>868</b></div>
117
+ <div id="lentidao" style="position:absolute; width:47px; height:19px; z-index:2; left: 17px; top: 95px; visibility: inherit;" class="lentidao"><b>322</b></div>
118
+ <div id="percentualLentidao" style="position:absolute; width:47px; height:14px; z-index:2; left:77px; top: 95px; visibility: inherit;" class="lentidao"><b>32,4%</b></div>
119
+ <div id="tendencia" style="position:absolute; width:52px; height:14px; z-index:2; left: 176px; top: 127px; visibility: inherit;" class="hora"><img src="img\ESTAVEL.gif" title="ESTAVEL"></div>
120
+ <div id="CentroLentidao" style="position:absolute; width:80px; height:19px; z-index:2; left: 489px; top: 176px; visibility: inherit;" class="dados2">22 km</div>
121
+ <div id="CentroTendencia" style="position:absolute; width:80px; height:19px; z-index:2; left: 568px; top: 177px; visibility: inherit;" class="dados2"><img src="img\ESTAVEL.gif" title="ESTAVEL"></div>
122
+ <div id="LesteLentidao" style="position:absolute; width:80px; height:19px; z-index:2; left: 490px; top: 134px; visibility: inherit;" class="dados2">444 km</div>
123
+ <div id="LesteTendencia" style="position:absolute; width:80px; height:19px; z-index:2; left: 568px; top: 134px; visibility: inherit;" class="dados2"><img src="img\ESTAVEL.gif" title="ESTAVEL"></div>
124
+ <div id="NorteLentidao" style="position:absolute; width:80px; height:19px; z-index:2; left: 490px; top: 92px; visibility: inherit;" class="dados2">25 km</div>
125
+ <div id="NorteTendencia" style="position:absolute; width:80px; height:19px; z-index:2; left: 568px; top: 94px; visibility: inherit;" class="dados2"><img src="img\ESTAVEL.gif" title="ESTAVEL"></div>
126
+ <div id="OesteLentidao" style="position:absolute; width:80px; height:19px; z-index:2; left: 488px; top: 218px; visibility: inherit;" class="dados2">32 km</div>
127
+ <div id="OesteTendencia" style="position:absolute; width:80px; height:19px; z-index:2; left: 568px; top: 219px; visibility: inherit;" class="dados2"><img src="img\ESTAVEL.gif" title="ESTAVEL"></div>
128
+ <div id="SulLentidao" style="position:absolute; width:80px; height:19px; z-index:2; left: 487px; top: 262px; visibility: inherit;" class="dados2">34 km</div>
129
+ <div id="SulTendencia" style="position:absolute; width:80px; height:19px; z-index:2; left: 568px; top: 264px; visibility: inherit;" class="dados2"><img src="img\ESTAVEL.gif" title="ESTAVEL"></div>
130
+
131
+ </div>
132
+ <div id="dica" style="position:absolute; width:593px; height:93px; z-index:8; left: 50px; top: 380px; background-image: url(img/bgdicas.gif); layer-background-image: url(img/bgdicas.gif); border: 0px none;">
133
+ <div class="dicastxt" id="dicasTXT" style="position:absolute; width:530px; height:40px; z-index:1; left: 40px; top: 42px; border: 0px none; visibility: inherit;">
134
+ <script src="quoter.js" type="text/javascript"></script>
135
+
136
+ </div>
137
+ </div>
138
+ </center>
139
+
140
+
141
+ <!-- GOOGLE ANALYTICS -->
142
+ <script src="http://www.google-analytics.com/urchin.js" type="text/javascript">
143
+ </script>
144
+ <script type="text/javascript">
145
+ _uacct = "UA-5340069-1";
146
+ urchinTracker();
147
+ </script>
148
+
149
+
150
+ </body>
151
+ </html>
152
+
153
+
154
+
@@ -0,0 +1,42 @@
1
+ require 'traffic-scraper'
2
+ require 'fakeweb'
3
+
4
+ describe Traffic::Scraper do
5
+ describe ".overall_traffic" do
6
+ it "returns the overall traffic when success" do
7
+ FakeWeb.register_uri(:get, "http://cetsp1.cetsp.com.br/monitransmapa/agora/", :body => File.read("spec/fixtures/traffic-info-page.html"))
8
+
9
+ Traffic::Scraper.overall_traffic.should == 322
10
+ end
11
+
12
+ it "raises a CouldNotRetrievePageError when something goes wrong" do
13
+ Traffic::Scraper.should_receive(:open).with("http://cetsp1.cetsp.com.br/monitransmapa/agora/").and_raise RuntimeError
14
+
15
+ expect {
16
+ Traffic::Scraper.overall_traffic
17
+ }.to raise_error(CouldNotRetrievePageError)
18
+ end
19
+ end
20
+
21
+ describe ".zone_traffic" do
22
+ it 'returns a hash with the traffic on all the zones' do
23
+ FakeWeb.register_uri(:get, "http://cetsp1.cetsp.com.br/monitransmapa/agora/", :body => File.read("spec/fixtures/traffic-info-page.html"))
24
+
25
+ Traffic::Scraper.zone_traffic.should == {
26
+ :north => 25,
27
+ :south => 34,
28
+ :east => 444,
29
+ :west => 32,
30
+ :downtown => 22
31
+ }
32
+ end
33
+
34
+ it "raises a CouldNotRetrievePageError when something goes wrong" do
35
+ Traffic::Scraper.should_receive(:open).with("http://cetsp1.cetsp.com.br/monitransmapa/agora/").and_raise RuntimeError
36
+
37
+ expect {
38
+ Traffic::Scraper.zone_traffic
39
+ }.to raise_error(CouldNotRetrievePageError)
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,26 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "traffic-scraper/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "traffic-scraper"
7
+ s.version = Traffic::Scraper::VERSION
8
+ s.authors = ["Rodrigo Flores"]
9
+ s.email = ["mail@rodrigoflores.org"]
10
+ s.homepage = ""
11
+ s.summary = %q{A gem to retrieve data from CET SP website}
12
+ s.description = %q{This gem parses the traffic engineering company and retrieves traffic information for the city of Sao Paulo Brazil}
13
+
14
+ s.rubyforge_project = "traffic-scraper"
15
+
16
+ s.files = `git ls-files`.split("\n")
17
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
18
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
19
+ s.require_paths = ["lib"]
20
+
21
+ s.add_runtime_dependency "nokogiri"
22
+
23
+ s.add_development_dependency "rspec"
24
+ s.add_development_dependency "fakeweb"
25
+ s.add_development_dependency "yard"
26
+ end
metadata ADDED
@@ -0,0 +1,103 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: traffic-scraper
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ prerelease:
6
+ platform: ruby
7
+ authors:
8
+ - Rodrigo Flores
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2011-11-06 00:00:00.000000000Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: nokogiri
16
+ requirement: &70119666200500 !ruby/object:Gem::Requirement
17
+ none: false
18
+ requirements:
19
+ - - ! '>='
20
+ - !ruby/object:Gem::Version
21
+ version: '0'
22
+ type: :runtime
23
+ prerelease: false
24
+ version_requirements: *70119666200500
25
+ - !ruby/object:Gem::Dependency
26
+ name: rspec
27
+ requirement: &70119666199440 !ruby/object:Gem::Requirement
28
+ none: false
29
+ requirements:
30
+ - - ! '>='
31
+ - !ruby/object:Gem::Version
32
+ version: '0'
33
+ type: :development
34
+ prerelease: false
35
+ version_requirements: *70119666199440
36
+ - !ruby/object:Gem::Dependency
37
+ name: fakeweb
38
+ requirement: &70119666199000 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ! '>='
42
+ - !ruby/object:Gem::Version
43
+ version: '0'
44
+ type: :development
45
+ prerelease: false
46
+ version_requirements: *70119666199000
47
+ - !ruby/object:Gem::Dependency
48
+ name: yard
49
+ requirement: &70119666198580 !ruby/object:Gem::Requirement
50
+ none: false
51
+ requirements:
52
+ - - ! '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ type: :development
56
+ prerelease: false
57
+ version_requirements: *70119666198580
58
+ description: This gem parses the traffic engineering company and retrieves traffic
59
+ information for the city of Sao Paulo Brazil
60
+ email:
61
+ - mail@rodrigoflores.org
62
+ executables: []
63
+ extensions: []
64
+ extra_rdoc_files: []
65
+ files:
66
+ - .gitignore
67
+ - .travis.yml
68
+ - Gemfile
69
+ - README.md
70
+ - Rakefile
71
+ - lib/traffic-scraper.rb
72
+ - lib/traffic-scraper/version.rb
73
+ - spec/fixtures/traffic-info-page.html
74
+ - spec/traffic-scraper_spec.rb
75
+ - traffic-scraper.gemspec
76
+ homepage: ''
77
+ licenses: []
78
+ post_install_message:
79
+ rdoc_options: []
80
+ require_paths:
81
+ - lib
82
+ required_ruby_version: !ruby/object:Gem::Requirement
83
+ none: false
84
+ requirements:
85
+ - - ! '>='
86
+ - !ruby/object:Gem::Version
87
+ version: '0'
88
+ required_rubygems_version: !ruby/object:Gem::Requirement
89
+ none: false
90
+ requirements:
91
+ - - ! '>='
92
+ - !ruby/object:Gem::Version
93
+ version: '0'
94
+ requirements: []
95
+ rubyforge_project: traffic-scraper
96
+ rubygems_version: 1.8.10
97
+ signing_key:
98
+ specification_version: 3
99
+ summary: A gem to retrieve data from CET SP website
100
+ test_files:
101
+ - spec/fixtures/traffic-info-page.html
102
+ - spec/traffic-scraper_spec.rb
103
+ has_rdoc: