sports_db 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/MIT-LICENSE +20 -0
- data/README.rdoc +3 -0
- data/Rakefile +38 -0
- data/app/controllers/foo_controller.rb +5 -0
- data/app/models/media.rb +35 -0
- data/app/models/twitter.rb +14 -0
- data/config/routes.rb +3 -0
- data/lib/sports_db/engine.rb +5 -0
- data/lib/sports_db/media_builder.rb +58 -0
- data/lib/sports_db/twitter_builder.rb +216 -0
- data/lib/sports_db/version.rb +3 -0
- data/lib/sports_db.rb +8 -0
- data/lib/tasks/sports_db_tasks.rake +4 -0
- data/test/dummy/README.rdoc +261 -0
- data/test/dummy/Rakefile +7 -0
- data/test/dummy/app/assets/javascripts/application.js +15 -0
- data/test/dummy/app/assets/stylesheets/application.css +13 -0
- data/test/dummy/app/controllers/application_controller.rb +3 -0
- data/test/dummy/app/helpers/application_helper.rb +2 -0
- data/test/dummy/app/views/layouts/application.html.erb +14 -0
- data/test/dummy/config/application.rb +59 -0
- data/test/dummy/config/boot.rb +10 -0
- data/test/dummy/config/database.yml +25 -0
- data/test/dummy/config/environment.rb +5 -0
- data/test/dummy/config/environments/development.rb +37 -0
- data/test/dummy/config/environments/production.rb +67 -0
- data/test/dummy/config/environments/test.rb +37 -0
- data/test/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/test/dummy/config/initializers/inflections.rb +15 -0
- data/test/dummy/config/initializers/mime_types.rb +5 -0
- data/test/dummy/config/initializers/secret_token.rb +7 -0
- data/test/dummy/config/initializers/session_store.rb +8 -0
- data/test/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/test/dummy/config/locales/en.yml +5 -0
- data/test/dummy/config/routes.rb +58 -0
- data/test/dummy/config.ru +4 -0
- data/test/dummy/public/404.html +26 -0
- data/test/dummy/public/422.html +26 -0
- data/test/dummy/public/500.html +25 -0
- data/test/dummy/public/favicon.ico +0 -0
- data/test/dummy/script/rails +6 -0
- data/test/sports_db_test.rb +7 -0
- data/test/test_helper.rb +15 -0
- metadata +129 -0
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright 2013 YOURNAME
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
ADDED
data/Rakefile
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
begin
|
3
|
+
require 'bundler/setup'
|
4
|
+
rescue LoadError
|
5
|
+
puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
|
6
|
+
end
|
7
|
+
begin
|
8
|
+
require 'rdoc/task'
|
9
|
+
rescue LoadError
|
10
|
+
require 'rdoc/rdoc'
|
11
|
+
require 'rake/rdoctask'
|
12
|
+
RDoc::Task = Rake::RDocTask
|
13
|
+
end
|
14
|
+
|
15
|
+
RDoc::Task.new(:rdoc) do |rdoc|
|
16
|
+
rdoc.rdoc_dir = 'rdoc'
|
17
|
+
rdoc.title = 'SportsDb'
|
18
|
+
rdoc.options << '--line-numbers'
|
19
|
+
rdoc.rdoc_files.include('README.rdoc')
|
20
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
21
|
+
end
|
22
|
+
|
23
|
+
|
24
|
+
|
25
|
+
|
26
|
+
Bundler::GemHelper.install_tasks
|
27
|
+
|
28
|
+
require 'rake/testtask'
|
29
|
+
|
30
|
+
Rake::TestTask.new(:test) do |t|
|
31
|
+
t.libs << 'lib'
|
32
|
+
t.libs << 'test'
|
33
|
+
t.pattern = 'test/**/*_test.rb'
|
34
|
+
t.verbose = false
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
task :default => :test
|
data/app/models/media.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'digest/md5'
|
2
|
+
|
3
|
+
class Media < ActiveRecord::Base
|
4
|
+
|
5
|
+
# Create a hash based on publication time and title that should uniquely identify
|
6
|
+
# this publication of the document.
|
7
|
+
before_save :before_save_method
|
8
|
+
|
9
|
+
def before_save_method
|
10
|
+
if (self["title"].nil?)
|
11
|
+
raise "This is an issue.... there's no title"
|
12
|
+
Rails.logger.warn("Found video with no title")
|
13
|
+
return false
|
14
|
+
end
|
15
|
+
str = (self["published_at"].nil?) ? self["title"] : (self["title"] + self["published_at"].to_s)
|
16
|
+
self["digest"] = Digest::SHA1.hexdigest(str)
|
17
|
+
end
|
18
|
+
|
19
|
+
def is_video?
|
20
|
+
true
|
21
|
+
end
|
22
|
+
|
23
|
+
def is_twitter?
|
24
|
+
false
|
25
|
+
end
|
26
|
+
|
27
|
+
def is_article?
|
28
|
+
false
|
29
|
+
end
|
30
|
+
|
31
|
+
def thumbnail_url
|
32
|
+
self.thumb_image_url
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
data/config/routes.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
module SportsDb
|
2
|
+
class MediaBuilder
|
3
|
+
|
4
|
+
def self.update_sn_video
|
5
|
+
require 'open-uri'
|
6
|
+
config = SimpleConfig.for(:feeds)
|
7
|
+
|
8
|
+
config.sn_video_feeds.each do |feed_title, url|
|
9
|
+
p "Video - #{url}"
|
10
|
+
open( url ) do |file|
|
11
|
+
doc = Nokogiri::XML(file.read)
|
12
|
+
|
13
|
+
nodes = doc.xpath('//item')
|
14
|
+
media_nodes = doc.xpath('/rss/channel/item/media:content', 'media' => 'http://search.yahoo.com/mrss/')
|
15
|
+
|
16
|
+
if media_nodes.length > 0
|
17
|
+
Media.delete_all( "media_type = 'sportingnews_video'")
|
18
|
+
|
19
|
+
nodes.each do |entry|
|
20
|
+
high_bandwidth_node = entry.at('.//media:content[@medium="video"]', 'media' => 'http://search.yahoo.com/mrss/')
|
21
|
+
high_bandwidth_url = high_bandwidth_node.nil? ? '' : high_bandwidth_node['url']
|
22
|
+
low_bandwidth_url = high_bandwidth_url
|
23
|
+
|
24
|
+
image_url_node = entry.at('./media:thumbnail', 'media' => 'http://search.yahoo.com/mrss/')
|
25
|
+
if image_url_node.nil?
|
26
|
+
image_url = ""
|
27
|
+
thumb_url = ""
|
28
|
+
else
|
29
|
+
image_url = image_url_node['url']
|
30
|
+
thumb_url = CONFIG.image_service + "crop/w/55/h/55/url/#{CGI::escape(CGI::escape(image_url))}"
|
31
|
+
end
|
32
|
+
|
33
|
+
p "Video added - #{entry.xpath('title').text}"
|
34
|
+
|
35
|
+
Media.create(
|
36
|
+
:feed_title => feed_title,
|
37
|
+
:title => entry.xpath('title').text,
|
38
|
+
:description => entry.xpath('content:encoded', 'content' => 'http://purl.org/rss/1.0/modules/content/').text,
|
39
|
+
:contents => entry.xpath('content:encoded', 'content' => 'http://purl.org/rss/1.0/modules/content/').text,
|
40
|
+
:high_bandwidth_url => high_bandwidth_url,
|
41
|
+
:low_bandwidth_url => low_bandwidth_url,
|
42
|
+
:media_type => 'sportingnews_video',
|
43
|
+
:thumb_image_url => thumb_url,
|
44
|
+
:article_image_url => image_url,
|
45
|
+
:published_at => DateTime.parse(entry.xpath('pubDate').text)
|
46
|
+
)
|
47
|
+
end
|
48
|
+
else
|
49
|
+
Rails.logger.info("No URLs available - Did not update videos")
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
rescue Exception => e
|
54
|
+
Zumobi::ExceptionHandler.error e
|
55
|
+
end
|
56
|
+
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,216 @@
|
|
1
|
+
module SportsDb
|
2
|
+
class TwitterBuilder
|
3
|
+
|
4
|
+
def self.update_twitter
|
5
|
+
config = SimpleConfig.for(:feeds)
|
6
|
+
config.twitter_feeds.each do |feed_title, url|
|
7
|
+
parse_feed(url, feed_title)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.update_ncaa_twitter
|
12
|
+
config = SimpleConfig.for(:feeds)
|
13
|
+
config.twitter_feeds.each do |feed_title, url|
|
14
|
+
parse_feed(url, feed_title)
|
15
|
+
end
|
16
|
+
|
17
|
+
ExternalFeed.find(:all, :conditions => ["content_type = ? and provider = ?", "twitter", "Sporting News"]).each do |feed_obj|
|
18
|
+
feed_url = feed_obj.woven_feed_url
|
19
|
+
if SimpleConfig.for(:application).woven_feed_server != "woven.zumobi.net"
|
20
|
+
feed_url = feed_url.gsub("woven.zumobi.net", SimpleConfig.for(:application).woven_feed_server)
|
21
|
+
end
|
22
|
+
|
23
|
+
team_obj = Team.find(feed_obj.team_id)
|
24
|
+
feed_name = (team_obj.nil? ? "Unknown" : team_obj.city_name)
|
25
|
+
parse_feed(feed_url, feed_name, feed_obj.team_id)
|
26
|
+
end
|
27
|
+
rescue Exception => e
|
28
|
+
Zumobi::ExceptionHandler.error e
|
29
|
+
end
|
30
|
+
|
31
|
+
|
32
|
+
def self.parse_feed(url, feed_title, team_id=nil)
|
33
|
+
require 'open-uri'
|
34
|
+
|
35
|
+
p "Twitter - #{feed_title} - #{url}"
|
36
|
+
new_tweets = []
|
37
|
+
|
38
|
+
begin
|
39
|
+
open(url) do |file|
|
40
|
+
doc = Nokogiri::XML(file.read)
|
41
|
+
source = doc.xpath('/rss/channel/title').text
|
42
|
+
source = source.gsub('Twitter', '').strip
|
43
|
+
source = title_changes(source)
|
44
|
+
|
45
|
+
if !doc.nil? && !source.nil?
|
46
|
+
doc.xpath('//item').each do |node|
|
47
|
+
t = Twitter.new
|
48
|
+
t.source = source
|
49
|
+
t.title = node.xpath('content:encoded').text.strip
|
50
|
+
t.title = t.title.gsub('>', '>')
|
51
|
+
t.author = node.xpath('dc:creator').text
|
52
|
+
t.thumb_image_url = node.xpath('media:thumbnail/@url').text
|
53
|
+
t.published_at = node.xpath('pubDate').text
|
54
|
+
t.link = node.xpath('link').text
|
55
|
+
t.guid = node.xpath('guid').text
|
56
|
+
t.contents = t.title
|
57
|
+
|
58
|
+
team_key = team_mapping(t.source)
|
59
|
+
|
60
|
+
if !team_key.blank?
|
61
|
+
if Team.column_names.include?("tsn_key") && !Team.find_by_tsn_key(team_key).blank?
|
62
|
+
t.team_id = Team.find_by_tsn_key(team_key).id
|
63
|
+
elsif !Team.find_by_key(team_key).blank?
|
64
|
+
t.team_id = Team.find_by_key(team_key).id
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
new_tweets << t
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
rescue OpenURI::HTTPError => http_e
|
73
|
+
# unable to retrieve feed
|
74
|
+
rescue EOFError
|
75
|
+
#
|
76
|
+
rescue Errno::ECONNRESET
|
77
|
+
#
|
78
|
+
rescue Errno::ECONNREFUSED
|
79
|
+
#
|
80
|
+
end
|
81
|
+
|
82
|
+
|
83
|
+
Twitter.transaction do
|
84
|
+
Twitter.delete_all()
|
85
|
+
new_tweets.each {|t| t.save}
|
86
|
+
end
|
87
|
+
|
88
|
+
rescue Exception => e
|
89
|
+
Zumobi::ExceptionHandler.error e
|
90
|
+
end
|
91
|
+
|
92
|
+
def self.title_changes(source)
|
93
|
+
titles = {'Official NFL Twitter Feed' => 'NFL',
|
94
|
+
'The Trenches Twitter' => 'The Trenches'}
|
95
|
+
|
96
|
+
return (titles[source].nil?) ? source : titles[source]
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.add_hash_highlight(str)
|
100
|
+
formatted_text = []
|
101
|
+
each_word = str.split(' ')
|
102
|
+
each_word.each do |word|
|
103
|
+
if word[0,7] == "http://"
|
104
|
+
new_word = "<a href='#{word}'>#{word}</a>"
|
105
|
+
formatted_text << new_word
|
106
|
+
elsif word.include? "http://"
|
107
|
+
phrase = word
|
108
|
+
pieces = word.split("http://")
|
109
|
+
|
110
|
+
phrase = "#{pieces[0]} <a href='http://#{pieces[1]}'>http://#{pieces[1]}</a>"
|
111
|
+
formatted_text << phrase
|
112
|
+
else
|
113
|
+
formatted_text << word
|
114
|
+
end
|
115
|
+
end
|
116
|
+
formatted_text.join(' ')
|
117
|
+
end
|
118
|
+
|
119
|
+
def self.team_mapping(source)
|
120
|
+
|
121
|
+
links = {'New England Patriots' => 'l.nfl.com-t.4',
|
122
|
+
'Buffalo Bills' => 'l.nfl.com-t.1',
|
123
|
+
'Miami Dolphins' => 'l.nfl.com-t.3',
|
124
|
+
'New York Jets' => 'l.nfl.com-t.5',
|
125
|
+
'Denver Broncos' => 'l.nfl.com-t.12',
|
126
|
+
'Kansas City Chiefs' => 'l.nfl.com-t.13',
|
127
|
+
'Oakland Raiders' => 'l.nfl.com-t.14',
|
128
|
+
'San Diego Chargers' => 'l.nfl.com-t.15',
|
129
|
+
'Baltimore Ravens' => 'l.nfl.com-t.6',
|
130
|
+
'Cincinnati Bengals' => 'l.nfl.com-t.7',
|
131
|
+
'Cleveland Browns' => 'l.nfl.com-t.8',
|
132
|
+
'Pittsburgh Steelers' => 'l.nfl.com-t.10',
|
133
|
+
'Houston Texans' => 'l.nfl.com-t.32',
|
134
|
+
'Jacksonville Jaguars' => 'l.nfl.com-t.9',
|
135
|
+
'Tennessee Titans' => 'l.nfl.com-t.11',
|
136
|
+
'Dallas Cowboys' => 'l.nfl.com-t.18',
|
137
|
+
'New York Giants' => 'l.nfl.com-t.19',
|
138
|
+
'Washington Redskins' => 'l.nfl.com-t.21',
|
139
|
+
'Arizona Cardinals' => 'l.nfl.com-t.17',
|
140
|
+
'San Francisco 49ers' => 'l.nfl.com-t.31',
|
141
|
+
'Seattle Seahawks' => 'l.nfl.com-t.16',
|
142
|
+
'St. Louis Rams' => 'l.nfl.com-t.28',
|
143
|
+
'Chicago Bears' => 'l.nfl.com-t.22',
|
144
|
+
'Detroit Lions' => 'l.nfl.com-t.23',
|
145
|
+
'Green Bay Packers' => 'l.nfl.com-t.24',
|
146
|
+
'Minnesota Vikings' => 'l.nfl.com-t.25',
|
147
|
+
'Atlanta Falcons' => 'l.nfl.com-t.27',
|
148
|
+
'Carolina Panthers' => 'l.nfl.com-t.29',
|
149
|
+
'New Orleans Saints' => 'l.nfl.com-t.30',
|
150
|
+
'Tampa Bay Bucs' => 'l.nfl.com-t.26',
|
151
|
+
'Arizona Diamondbacks' => 'l.mlb.com-t.26',
|
152
|
+
'Atlanta Braves' => 'l.mlb.com-t.15',
|
153
|
+
'Baltimore Orioles' => 'l.mlb.com-t.1',
|
154
|
+
'Boston Red Sox' => 'l.mlb.com-t.2',
|
155
|
+
'Chicago Cubs' => 'l.mlb.com-t.20',
|
156
|
+
'Chicago White Sox' => 'l.mlb.com-t.6',
|
157
|
+
'Cincinnati Reds' => 'l.mlb.com-t.21',
|
158
|
+
'Cleveland Indians' => 'l.mlb.com-t.7',
|
159
|
+
'Colorado Rockies' => 'l.mlb.com-t.27',
|
160
|
+
'Detroit Tigers' => 'l.mlb.com-t.8',
|
161
|
+
'Miami Marlins' => 'l.mlb.com-t.16',
|
162
|
+
'Houston Astros' => 'l.mlb.com-t.22',
|
163
|
+
'Kansas City Royals' => 'l.mlb.com-t.9',
|
164
|
+
'Los Angeles Angels' => 'l.mlb.com-t.11',
|
165
|
+
'Los Angeles Dodgers' => 'l.mlb.com-t.28',
|
166
|
+
'Milwaukee Brewers' => 'l.mlb.com-t.23',
|
167
|
+
'Minnesota Twins' => 'l.mlb.com-t.10',
|
168
|
+
'New York Mets' => 'l.mlb.com-t.18',
|
169
|
+
'New York Yankees' => 'l.mlb.com-t.3',
|
170
|
+
'Oakland Athletics' => 'l.mlb.com-t.12',
|
171
|
+
'Philadelphia Phillies' => 'l.mlb.com-t.19',
|
172
|
+
'Pittsburgh Pirates' => 'l.mlb.com-t.24',
|
173
|
+
'San Diego Padres' => 'l.mlb.com-t.29',
|
174
|
+
'San Francisco Giants' => 'l.mlb.com-t.30',
|
175
|
+
'Seattle Mariners' => 'l.mlb.com-t.13',
|
176
|
+
'St. Louis Cardinals' => 'l.mlb.com-t.25',
|
177
|
+
'Tampa Bay Rays' => 'l.mlb.com-t.4',
|
178
|
+
'Texas Rangers' => 'l.mlb.com-t.14',
|
179
|
+
'Toronto Blue Jays' => 'l.mlb.com-t.5',
|
180
|
+
'Washington Nationals' => 'l.mlb.com-t.17',
|
181
|
+
'Boston Celtics' => 'l.nba.com-t.1',
|
182
|
+
'Brooklyn Nets' => 'l.nba.com-t.3',
|
183
|
+
'New York Knicks' => 'l.nba.com-t.4',
|
184
|
+
'Philadelphia 76ers' => 'l.nba.com-t.6',
|
185
|
+
'Toronto Raptors' => 'l.nba.com-t.15',
|
186
|
+
'Chicago Bulls' => 'l.nba.com-t.10',
|
187
|
+
'Cleveland Cavaliers' => 'l.nba.com-t.11',
|
188
|
+
'Detroit Pistons' => 'l.nba.com-t.12',
|
189
|
+
'Indiana Pacers' => 'l.nba.com-t.13',
|
190
|
+
'Milwaukee Bucks' => 'l.nba.com-t.14',
|
191
|
+
'Atlanta Hawks' => 'l.nba.com-t.8',
|
192
|
+
'Charlotte Bobcats' => 'l.nba.com-t.32',
|
193
|
+
'Miami Heat' => 'l.nba.com-t.2',
|
194
|
+
'Orlando Magic' => 'l.nba.com-t.5',
|
195
|
+
'Washington Wizards' => 'l.nba.com-t.7',
|
196
|
+
'Denver Nuggets' => 'l.nba.com-t.17',
|
197
|
+
'Minnesota Timberwolves' => 'l.nba.com-t.20',
|
198
|
+
'Oklahoma City Thunder' => 'l.nba.com-t.29',
|
199
|
+
'Portland Trail Blazers' => 'l.nba.com-t.27',
|
200
|
+
'Utah Jazz' => 'l.nba.com-t.22',
|
201
|
+
'Golden State Warriors' => 'l.nba.com-t.23',
|
202
|
+
'Los Angeles Clippers' => 'l.nba.com-t.24',
|
203
|
+
'Los Angeles Lakers' => 'l.nba.com-t.25',
|
204
|
+
'Phoenix Suns' => 'l.nba.com-t.26',
|
205
|
+
'Sacramento Kings' => 'l.nba.com-t.28',
|
206
|
+
'Dallas Mavericks' => 'l.nba.com-t.16',
|
207
|
+
'Houston Rockets' => 'l.nba.com-t.18',
|
208
|
+
'Memphis Grizzlies' => 'l.nba.com-t.19',
|
209
|
+
'New Orleans Hornets' => 'l.nba.com-t.9',
|
210
|
+
'San Antonio Spurs' => 'l.nba.com-t.21'}
|
211
|
+
|
212
|
+
return (links[source].nil?) ? "" : links[source]
|
213
|
+
end
|
214
|
+
|
215
|
+
end
|
216
|
+
end
|
data/lib/sports_db.rb
ADDED