sports_db 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/MIT-LICENSE +20 -0
- data/README.rdoc +3 -0
- data/Rakefile +38 -0
- data/app/controllers/foo_controller.rb +5 -0
- data/app/models/media.rb +35 -0
- data/app/models/twitter.rb +14 -0
- data/config/routes.rb +3 -0
- data/lib/sports_db/engine.rb +5 -0
- data/lib/sports_db/media_builder.rb +58 -0
- data/lib/sports_db/twitter_builder.rb +216 -0
- data/lib/sports_db/version.rb +3 -0
- data/lib/sports_db.rb +8 -0
- data/lib/tasks/sports_db_tasks.rake +4 -0
- data/test/dummy/README.rdoc +261 -0
- data/test/dummy/Rakefile +7 -0
- data/test/dummy/app/assets/javascripts/application.js +15 -0
- data/test/dummy/app/assets/stylesheets/application.css +13 -0
- data/test/dummy/app/controllers/application_controller.rb +3 -0
- data/test/dummy/app/helpers/application_helper.rb +2 -0
- data/test/dummy/app/views/layouts/application.html.erb +14 -0
- data/test/dummy/config/application.rb +59 -0
- data/test/dummy/config/boot.rb +10 -0
- data/test/dummy/config/database.yml +25 -0
- data/test/dummy/config/environment.rb +5 -0
- data/test/dummy/config/environments/development.rb +37 -0
- data/test/dummy/config/environments/production.rb +67 -0
- data/test/dummy/config/environments/test.rb +37 -0
- data/test/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/test/dummy/config/initializers/inflections.rb +15 -0
- data/test/dummy/config/initializers/mime_types.rb +5 -0
- data/test/dummy/config/initializers/secret_token.rb +7 -0
- data/test/dummy/config/initializers/session_store.rb +8 -0
- data/test/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/test/dummy/config/locales/en.yml +5 -0
- data/test/dummy/config/routes.rb +58 -0
- data/test/dummy/config.ru +4 -0
- data/test/dummy/public/404.html +26 -0
- data/test/dummy/public/422.html +26 -0
- data/test/dummy/public/500.html +25 -0
- data/test/dummy/public/favicon.ico +0 -0
- data/test/dummy/script/rails +6 -0
- data/test/sports_db_test.rb +7 -0
- data/test/test_helper.rb +15 -0
- metadata +129 -0
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright 2013 YOURNAME
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
begin
|
3
|
+
require 'bundler/setup'
|
4
|
+
rescue LoadError
|
5
|
+
puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
|
6
|
+
end
|
7
|
+
begin
|
8
|
+
require 'rdoc/task'
|
9
|
+
rescue LoadError
|
10
|
+
require 'rdoc/rdoc'
|
11
|
+
require 'rake/rdoctask'
|
12
|
+
RDoc::Task = Rake::RDocTask
|
13
|
+
end
|
14
|
+
|
15
|
+
RDoc::Task.new(:rdoc) do |rdoc|
|
16
|
+
rdoc.rdoc_dir = 'rdoc'
|
17
|
+
rdoc.title = 'SportsDb'
|
18
|
+
rdoc.options << '--line-numbers'
|
19
|
+
rdoc.rdoc_files.include('README.rdoc')
|
20
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
|
25
|
+
|
26
|
+
Bundler::GemHelper.install_tasks
|
27
|
+
|
28
|
+
require 'rake/testtask'
|
29
|
+
|
30
|
+
Rake::TestTask.new(:test) do |t|
|
31
|
+
t.libs << 'lib'
|
32
|
+
t.libs << 'test'
|
33
|
+
t.pattern = 'test/**/*_test.rb'
|
34
|
+
t.verbose = false
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
task :default => :test
|
data/app/models/media.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
|
3
|
+
class Media < ActiveRecord::Base
|
4
|
+
|
5
|
+
# Create a hash based on publication time and title that should uniquely identify
|
6
|
+
# this publication of the document.
|
7
|
+
before_save :before_save_method
|
8
|
+
|
9
|
+
def before_save_method
|
10
|
+
if (self["title"].nil?)
|
11
|
+
raise "This is an issue.... there's no title"
|
12
|
+
Rails.logger.warn("Found video with no title")
|
13
|
+
return false
|
14
|
+
end
|
15
|
+
str = (self["published_at"].nil?) ? self["title"] : (self["title"] + self["published_at"].to_s)
|
16
|
+
self["digest"] = Digest::SHA1.hexdigest(str)
|
17
|
+
end
|
18
|
+
|
19
|
+
def is_video?
|
20
|
+
true
|
21
|
+
end
|
22
|
+
|
23
|
+
def is_twitter?
|
24
|
+
false
|
25
|
+
end
|
26
|
+
|
27
|
+
def is_article?
|
28
|
+
false
|
29
|
+
end
|
30
|
+
|
31
|
+
def thumbnail_url
|
32
|
+
self.thumb_image_url
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
data/config/routes.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
module SportsDb
|
2
|
+
class MediaBuilder
|
3
|
+
|
4
|
+
def self.update_sn_video
|
5
|
+
require 'open-uri'
|
6
|
+
config = SimpleConfig.for(:feeds)
|
7
|
+
|
8
|
+
config.sn_video_feeds.each do |feed_title, url|
|
9
|
+
p "Video - #{url}"
|
10
|
+
open( url ) do |file|
|
11
|
+
doc = Nokogiri::XML(file.read)
|
12
|
+
|
13
|
+
nodes = doc.xpath('//item')
|
14
|
+
media_nodes = doc.xpath('/rss/channel/item/media:content', 'media' => 'http://search.yahoo.com/mrss/')
|
15
|
+
|
16
|
+
if media_nodes.length > 0
|
17
|
+
Media.delete_all( "media_type = 'sportingnews_video'")
|
18
|
+
|
19
|
+
nodes.each do |entry|
|
20
|
+
high_bandwidth_node = entry.at('.//media:content[@medium="video"]', 'media' => 'http://search.yahoo.com/mrss/')
|
21
|
+
high_bandwidth_url = high_bandwidth_node.nil? ? '' : high_bandwidth_node['url']
|
22
|
+
low_bandwidth_url = high_bandwidth_url
|
23
|
+
|
24
|
+
image_url_node = entry.at('./media:thumbnail', 'media' => 'http://search.yahoo.com/mrss/')
|
25
|
+
if image_url_node.nil?
|
26
|
+
image_url = ""
|
27
|
+
thumb_url = ""
|
28
|
+
else
|
29
|
+
image_url = image_url_node['url']
|
30
|
+
thumb_url = CONFIG.image_service + "crop/w/55/h/55/url/#{CGI::escape(CGI::escape(image_url))}"
|
31
|
+
end
|
32
|
+
|
33
|
+
p "Video added - #{entry.xpath('title').text}"
|
34
|
+
|
35
|
+
Media.create(
|
36
|
+
:feed_title => feed_title,
|
37
|
+
:title => entry.xpath('title').text,
|
38
|
+
:description => entry.xpath('content:encoded', 'content' => 'http://purl.org/rss/1.0/modules/content/').text,
|
39
|
+
:contents => entry.xpath('content:encoded', 'content' => 'http://purl.org/rss/1.0/modules/content/').text,
|
40
|
+
:high_bandwidth_url => high_bandwidth_url,
|
41
|
+
:low_bandwidth_url => low_bandwidth_url,
|
42
|
+
:media_type => 'sportingnews_video',
|
43
|
+
:thumb_image_url => thumb_url,
|
44
|
+
:article_image_url => image_url,
|
45
|
+
:published_at => DateTime.parse(entry.xpath('pubDate').text)
|
46
|
+
)
|
47
|
+
end
|
48
|
+
else
|
49
|
+
Rails.logger.info("No URLs available - Did not update videos")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
rescue Exception => e
|
54
|
+
Zumobi::ExceptionHandler.error e
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,216 @@
|
|
1
|
+
module SportsDb
|
2
|
+
class TwitterBuilder
|
3
|
+
|
4
|
+
def self.update_twitter
|
5
|
+
config = SimpleConfig.for(:feeds)
|
6
|
+
config.twitter_feeds.each do |feed_title, url|
|
7
|
+
parse_feed(url, feed_title)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.update_ncaa_twitter
|
12
|
+
config = SimpleConfig.for(:feeds)
|
13
|
+
config.twitter_feeds.each do |feed_title, url|
|
14
|
+
parse_feed(url, feed_title)
|
15
|
+
end
|
16
|
+
|
17
|
+
ExternalFeed.find(:all, :conditions => ["content_type = ? and provider = ?", "twitter", "Sporting News"]).each do |feed_obj|
|
18
|
+
feed_url = feed_obj.woven_feed_url
|
19
|
+
if SimpleConfig.for(:application).woven_feed_server != "woven.zumobi.net"
|
20
|
+
feed_url = feed_url.gsub("woven.zumobi.net", SimpleConfig.for(:application).woven_feed_server)
|
21
|
+
end
|
22
|
+
|
23
|
+
team_obj = Team.find(feed_obj.team_id)
|
24
|
+
feed_name = (team_obj.nil? ? "Unknown" : team_obj.city_name)
|
25
|
+
parse_feed(feed_url, feed_name, feed_obj.team_id)
|
26
|
+
end
|
27
|
+
rescue Exception => e
|
28
|
+
Zumobi::ExceptionHandler.error e
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
def self.parse_feed(url, feed_title, team_id=nil)
|
33
|
+
require 'open-uri'
|
34
|
+
|
35
|
+
p "Twitter - #{feed_title} - #{url}"
|
36
|
+
new_tweets = []
|
37
|
+
|
38
|
+
begin
|
39
|
+
open(url) do |file|
|
40
|
+
doc = Nokogiri::XML(file.read)
|
41
|
+
source = doc.xpath('/rss/channel/title').text
|
42
|
+
source = source.gsub('Twitter', '').strip
|
43
|
+
source = title_changes(source)
|
44
|
+
|
45
|
+
if !doc.nil? && !source.nil?
|
46
|
+
doc.xpath('//item').each do |node|
|
47
|
+
t = Twitter.new
|
48
|
+
t.source = source
|
49
|
+
t.title = node.xpath('content:encoded').text.strip
|
50
|
+
t.title = t.title.gsub('>', '>')
|
51
|
+
t.author = node.xpath('dc:creator').text
|
52
|
+
t.thumb_image_url = node.xpath('media:thumbnail/@url').text
|
53
|
+
t.published_at = node.xpath('pubDate').text
|
54
|
+
t.link = node.xpath('link').text
|
55
|
+
t.guid = node.xpath('guid').text
|
56
|
+
t.contents = t.title
|
57
|
+
|
58
|
+
team_key = team_mapping(t.source)
|
59
|
+
|
60
|
+
if !team_key.blank?
|
61
|
+
if Team.column_names.include?("tsn_key") && !Team.find_by_tsn_key(team_key).blank?
|
62
|
+
t.team_id = Team.find_by_tsn_key(team_key).id
|
63
|
+
elsif !Team.find_by_key(team_key).blank?
|
64
|
+
t.team_id = Team.find_by_key(team_key).id
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
new_tweets << t
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
rescue OpenURI::HTTPError => http_e
|
73
|
+
# unable to retrieve feed
|
74
|
+
rescue EOFError
|
75
|
+
#
|
76
|
+
rescue Errno::ECONNRESET
|
77
|
+
#
|
78
|
+
rescue Errno::ECONNREFUSED
|
79
|
+
#
|
80
|
+
end
|
81
|
+
|
82
|
+
|
83
|
+
Twitter.transaction do
|
84
|
+
Twitter.delete_all()
|
85
|
+
new_tweets.each {|t| t.save}
|
86
|
+
end
|
87
|
+
|
88
|
+
rescue Exception => e
|
89
|
+
Zumobi::ExceptionHandler.error e
|
90
|
+
end
|
91
|
+
|
92
|
+
def self.title_changes(source)
|
93
|
+
titles = {'Official NFL Twitter Feed' => 'NFL',
|
94
|
+
'The Trenches Twitter' => 'The Trenches'}
|
95
|
+
|
96
|
+
return (titles[source].nil?) ? source : titles[source]
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.add_hash_highlight(str)
|
100
|
+
formatted_text = []
|
101
|
+
each_word = str.split(' ')
|
102
|
+
each_word.each do |word|
|
103
|
+
if word[0,7] == "http://"
|
104
|
+
new_word = "<a href='#{word}'>#{word}</a>"
|
105
|
+
formatted_text << new_word
|
106
|
+
elsif word.include? "http://"
|
107
|
+
phrase = word
|
108
|
+
pieces = word.split("http://")
|
109
|
+
|
110
|
+
phrase = "#{pieces[0]} <a href='http://#{pieces[1]}'>http://#{pieces[1]}</a>"
|
111
|
+
formatted_text << phrase
|
112
|
+
else
|
113
|
+
formatted_text << word
|
114
|
+
end
|
115
|
+
end
|
116
|
+
formatted_text.join(' ')
|
117
|
+
end
|
118
|
+
|
119
|
+
def self.team_mapping(source)
|
120
|
+
|
121
|
+
links = {'New England Patriots' => 'l.nfl.com-t.4',
|
122
|
+
'Buffalo Bills' => 'l.nfl.com-t.1',
|
123
|
+
'Miami Dolphins' => 'l.nfl.com-t.3',
|
124
|
+
'New York Jets' => 'l.nfl.com-t.5',
|
125
|
+
'Denver Broncos' => 'l.nfl.com-t.12',
|
126
|
+
'Kansas City Chiefs' => 'l.nfl.com-t.13',
|
127
|
+
'Oakland Raiders' => 'l.nfl.com-t.14',
|
128
|
+
'San Diego Chargers' => 'l.nfl.com-t.15',
|
129
|
+
'Baltimore Ravens' => 'l.nfl.com-t.6',
|
130
|
+
'Cincinnati Bengals' => 'l.nfl.com-t.7',
|
131
|
+
'Cleveland Browns' => 'l.nfl.com-t.8',
|
132
|
+
'Pittsburgh Steelers' => 'l.nfl.com-t.10',
|
133
|
+
'Houston Texans' => 'l.nfl.com-t.32',
|
134
|
+
'Jacksonville Jaguars' => 'l.nfl.com-t.9',
|
135
|
+
'Tennessee Titans' => 'l.nfl.com-t.11',
|
136
|
+
'Dallas Cowboys' => 'l.nfl.com-t.18',
|
137
|
+
'New York Giants' => 'l.nfl.com-t.19',
|
138
|
+
'Washington Redskins' => 'l.nfl.com-t.21',
|
139
|
+
'Arizona Cardinals' => 'l.nfl.com-t.17',
|
140
|
+
'San Francisco 49ers' => 'l.nfl.com-t.31',
|
141
|
+
'Seattle Seahawks' => 'l.nfl.com-t.16',
|
142
|
+
'St. Louis Rams' => 'l.nfl.com-t.28',
|
143
|
+
'Chicago Bears' => 'l.nfl.com-t.22',
|
144
|
+
'Detroit Lions' => 'l.nfl.com-t.23',
|
145
|
+
'Green Bay Packers' => 'l.nfl.com-t.24',
|
146
|
+
'Minnesota Vikings' => 'l.nfl.com-t.25',
|
147
|
+
'Atlanta Falcons' => 'l.nfl.com-t.27',
|
148
|
+
'Carolina Panthers' => 'l.nfl.com-t.29',
|
149
|
+
'New Orleans Saints' => 'l.nfl.com-t.30',
|
150
|
+
'Tampa Bay Bucs' => 'l.nfl.com-t.26',
|
151
|
+
'Arizona Diamondbacks' => 'l.mlb.com-t.26',
|
152
|
+
'Atlanta Braves' => 'l.mlb.com-t.15',
|
153
|
+
'Baltimore Orioles' => 'l.mlb.com-t.1',
|
154
|
+
'Boston Red Sox' => 'l.mlb.com-t.2',
|
155
|
+
'Chicago Cubs' => 'l.mlb.com-t.20',
|
156
|
+
'Chicago White Sox' => 'l.mlb.com-t.6',
|
157
|
+
'Cincinnati Reds' => 'l.mlb.com-t.21',
|
158
|
+
'Cleveland Indians' => 'l.mlb.com-t.7',
|
159
|
+
'Colorado Rockies' => 'l.mlb.com-t.27',
|
160
|
+
'Detroit Tigers' => 'l.mlb.com-t.8',
|
161
|
+
'Miami Marlins' => 'l.mlb.com-t.16',
|
162
|
+
'Houston Astros' => 'l.mlb.com-t.22',
|
163
|
+
'Kansas City Royals' => 'l.mlb.com-t.9',
|
164
|
+
'Los Angeles Angels' => 'l.mlb.com-t.11',
|
165
|
+
'Los Angeles Dodgers' => 'l.mlb.com-t.28',
|
166
|
+
'Milwaukee Brewers' => 'l.mlb.com-t.23',
|
167
|
+
'Minnesota Twins' => 'l.mlb.com-t.10',
|
168
|
+
'New York Mets' => 'l.mlb.com-t.18',
|
169
|
+
'New York Yankees' => 'l.mlb.com-t.3',
|
170
|
+
'Oakland Athletics' => 'l.mlb.com-t.12',
|
171
|
+
'Philadelphia Phillies' => 'l.mlb.com-t.19',
|
172
|
+
'Pittsburgh Pirates' => 'l.mlb.com-t.24',
|
173
|
+
'San Diego Padres' => 'l.mlb.com-t.29',
|
174
|
+
'San Francisco Giants' => 'l.mlb.com-t.30',
|
175
|
+
'Seattle Mariners' => 'l.mlb.com-t.13',
|
176
|
+
'St. Louis Cardinals' => 'l.mlb.com-t.25',
|
177
|
+
'Tampa Bay Rays' => 'l.mlb.com-t.4',
|
178
|
+
'Texas Rangers' => 'l.mlb.com-t.14',
|
179
|
+
'Toronto Blue Jays' => 'l.mlb.com-t.5',
|
180
|
+
'Washington Nationals' => 'l.mlb.com-t.17',
|
181
|
+
'Boston Celtics' => 'l.nba.com-t.1',
|
182
|
+
'Brooklyn Nets' => 'l.nba.com-t.3',
|
183
|
+
'New York Knicks' => 'l.nba.com-t.4',
|
184
|
+
'Philadelphia 76ers' => 'l.nba.com-t.6',
|
185
|
+
'Toronto Raptors' => 'l.nba.com-t.15',
|
186
|
+
'Chicago Bulls' => 'l.nba.com-t.10',
|
187
|
+
'Cleveland Cavaliers' => 'l.nba.com-t.11',
|
188
|
+
'Detroit Pistons' => 'l.nba.com-t.12',
|
189
|
+
'Indiana Pacers' => 'l.nba.com-t.13',
|
190
|
+
'Milwaukee Bucks' => 'l.nba.com-t.14',
|
191
|
+
'Atlanta Hawks' => 'l.nba.com-t.8',
|
192
|
+
'Charlotte Bobcats' => 'l.nba.com-t.32',
|
193
|
+
'Miami Heat' => 'l.nba.com-t.2',
|
194
|
+
'Orlando Magic' => 'l.nba.com-t.5',
|
195
|
+
'Washington Wizards' => 'l.nba.com-t.7',
|
196
|
+
'Denver Nuggets' => 'l.nba.com-t.17',
|
197
|
+
'Minnesota Timberwolves' => 'l.nba.com-t.20',
|
198
|
+
'Oklahoma City Thunder' => 'l.nba.com-t.29',
|
199
|
+
'Portland Trail Blazers' => 'l.nba.com-t.27',
|
200
|
+
'Utah Jazz' => 'l.nba.com-t.22',
|
201
|
+
'Golden State Warriors' => 'l.nba.com-t.23',
|
202
|
+
'Los Angeles Clippers' => 'l.nba.com-t.24',
|
203
|
+
'Los Angeles Lakers' => 'l.nba.com-t.25',
|
204
|
+
'Phoenix Suns' => 'l.nba.com-t.26',
|
205
|
+
'Sacramento Kings' => 'l.nba.com-t.28',
|
206
|
+
'Dallas Mavericks' => 'l.nba.com-t.16',
|
207
|
+
'Houston Rockets' => 'l.nba.com-t.18',
|
208
|
+
'Memphis Grizzlies' => 'l.nba.com-t.19',
|
209
|
+
'New Orleans Hornets' => 'l.nba.com-t.9',
|
210
|
+
'San Antonio Spurs' => 'l.nba.com-t.21'}
|
211
|
+
|
212
|
+
return (links[source].nil?) ? "" : links[source]
|
213
|
+
end
|
214
|
+
|
215
|
+
end
|
216
|
+
end
|
data/lib/sports_db.rb
ADDED