vnews 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,24 @@
1
+ require 'nokogiri'
2
+
3
+ class Vnews
4
+ class AutodiscoveryFailed < StandardError; end
5
+
6
+ module Autodiscoverer
7
+ def auto_discover(feed_url)
8
+ html = open(feed_url)
9
+ doc = Nokogiri::HTML.parse(html)
10
+ feed_url = [ 'head link[@type=application/atom+xml]',
11
+ 'head link[@type=application/rss+xml]',
12
+ "head link[@type=text/xml]"].detect do |path|
13
+ doc.at(path)
14
+ end
15
+ if feed_url
16
+ feed_url
17
+ else
18
+ nil
19
+ end
20
+ rescue Errno::ECONNRESET, Nokogiri::CSS::SyntaxError
21
+ $stderr.puts $!
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,122 @@
1
+ require 'vnews/sql'
2
+ require 'yaml'
3
+
4
+ class Vnews
5
+ def self.sql_client
6
+ $sql_client ||= Config.load_config
7
+ end
8
+
9
+ module Config
10
+ def self.generate_config
11
+ config = Vnews.sql_client.config.inject({}) do |memo, (k, v)|
12
+ memo[k.to_s] = v
13
+ memo
14
+ end
15
+ out = [config.to_yaml.sub(/---\s+/, ''), '']
16
+ Vnews.sql_client.configured_folders.map do |x|
17
+ folder = x["folder"]
18
+ out << folder
19
+ Vnews.sql_client.feeds_in_folder(folder).each do |feed|
20
+ out << feed
21
+ end
22
+ out << ""
23
+ end
24
+ out.join("\n")
25
+ end
26
+
27
+ def self.rewrite_config
28
+ out = generate_config
29
+ f = File.open(File.expand_path(CONFIGPATH), 'w') {|f| f.write(out)}
30
+ end
31
+
32
+ def self.stub_config
33
+ stub = <<-END.gsub(/^ */, '')
34
+ host: localhost
35
+ database: vnews
36
+ username: root
37
+ password:
38
+
39
+ General News
40
+ http://feedproxy.google.com/economist/full_print_edition
41
+ http://feeds.feedburner.com/TheAtlanticWire
42
+
43
+ Humor
44
+ http://feed.dilbert.com/dilbert/blog
45
+
46
+ Tech
47
+ http://rss.slashdot.org/Slashdot/slashdot
48
+ http://feeds2.feedburner.com/readwriteweb
49
+ http://feedproxy.google.com/programmableweb
50
+ http://news.ycombinator.com/rss
51
+ http://daringfireball.net/index.xml
52
+ http://dailyvim.blogspot.com/feeds/posts/default
53
+ END
54
+ end
55
+
56
+ CONFIGPATH = "#{ENV['HOME']}/.vnewsrc"
57
+
58
+ def self.update_folders
59
+ f = File.read(File.expand_path(CONFIGPATH))
60
+ db, list = f.split(/^\s*$/,2)
61
+ current_folder = nil
62
+ ff = []
63
+ list.split("\n").each do |line|
64
+ line = line.strip
65
+ if line =~ /^\s*http/ # feed
66
+ ff << [line, current_folder]
67
+ elsif line =~ /^\s*\w+/ # folder
68
+ current_folder = line
69
+ end
70
+ end
71
+ puts "Using feeds and folders: #{ff.inspect}"
72
+
73
+ old_feeds = Vnews.sql_client.feeds(0).map {|x| x["feed_url"]}
74
+ new_feeds = ff.map {|feed,folder| feed}
75
+ rm_feeds = old_feeds - new_feeds
76
+ puts "Removing feeds: #{rm_feeds.inspect}"
77
+ rm_feeds.each {|x| Vnews.sql_client.delete_feed(x)}
78
+
79
+ # ff is an association between a feed and a folder
80
+ old_ff = Vnews.sql_client.configured_feeds_folders
81
+ rm_ff = old_ff - ff
82
+ puts "Removing feed-folder associations: #{rm_ff.inspect}"
83
+ rm_ff.each {|feed,folder| Vnews.sql_client.delete_feed_folder(feed,folder)}
84
+
85
+ puts "Adding feeds: #{(new_feeds - old_feeds).inspect}"
86
+ puts "Adding folder-feed associations: #{(ff - old_ff).inspect}"
87
+ feeds2 = []
88
+ threads = []
89
+ ff.each do |feed_url, folder|
90
+ threads << Thread.new do
91
+ feeds2 << Vnews::Feed.fetch_feed(feed_url, folder)
92
+ end
93
+ end
94
+ threads.each {|t| t.join}
95
+ feeds2.each do |x|
96
+ feed_url, f, folder = *x
97
+ folder ||= "Misc"
98
+ if f.nil?
99
+ $stderr.print "\nNo feed found for #{feed_url}\n"
100
+ else
101
+ Vnews::Feed.save_feed(feed_url, f, folder)
102
+ end
103
+ end
104
+ $stderr.puts "\nDone."
105
+ end
106
+
107
+ def self.load_config
108
+ if ! File.exists?(File.expand_path(CONFIGPATH))
109
+ return false
110
+ end
111
+ return if $sql_client
112
+ f = File.read(File.expand_path(CONFIGPATH))
113
+ top, bottom = f.split(/^\s*$/,2)
114
+ dbconfig = YAML::load top
115
+ Sql.new(dbconfig)
116
+ end
117
+ end
118
+ end
119
+
120
+ if __FILE__ == $0
121
+ puts Vnews::Config.generate_config
122
+ end
@@ -0,0 +1,157 @@
1
+ require 'vnews/config'
2
+ require 'yaml'
3
+ require 'date'
4
+
5
+ class Vnews
6
+ class Display
7
+
8
+ def initialize
9
+ @sqliteclient = Vnews.sql_client
10
+ @window_width = 140
11
+ end
12
+
13
+ # returns folders as a list
14
+ def folders
15
+ @sqliteclient.folders.map do |x|
16
+ "#{x["folder"]} (#{x['count']})"
17
+ end.join("\n")
18
+ end
19
+
20
+ # returns feeds as a list, sorted alphabetically
21
+ # e.g.
22
+ # {"title"=>"Bits",
23
+ # "feed_url"=>"http://bits.blogs.nytimes.com/feed/",
24
+ # "link"=>"http://bits.blogs.nytimes.com/"}
25
+ def feeds(order)
26
+ # '0' is alphabetical, '1' is most popular first
27
+ @sqliteclient.feeds(order.to_i).map.with_index do |x, idx|
28
+ "#{ x['title'] } (#{x['item_count']})"
29
+ end
30
+ end
31
+
32
+ # returns items as a list, most recent first
33
+ # e.g.
34
+ # {"title"=>"Episode 96: Git on Rails", "guid"=>"git-on-rails",
35
+ # "feed"=>"http://feeds.feedburner.com/railscasts",
36
+ # "feed_title"=>"Railscasts", "pub_date"=>2008-03-10 00:00:00 -0400,
37
+ # "word_count"=>41}
38
+
39
+ def col(string, width)
40
+ return unless string
41
+ string[0,width].ljust(width)
42
+ end
43
+
44
+ def format_date(d)
45
+ if d.nil?
46
+ "no date"
47
+ elsif d.year != Time.now.year
48
+ d.strftime("%b %Y")
49
+ elsif d.to_date == Time.now.to_date
50
+ d.strftime("%I:%M%P")
51
+ else
52
+ d.strftime("%b %d")
53
+ end
54
+ end
55
+
56
+ # for item display
57
+ def format_long_date(d)
58
+ if d.nil?
59
+ "[no date]"
60
+ else
61
+ d.strftime("%a %m/%d/%Y at %I:%M%p %Z")
62
+ end
63
+ end
64
+
65
+ def format_item_summary(i, width)
66
+ varwidth = width.to_i - 31
67
+ feed_title = col i['feed_title'], varwidth * 0.25
68
+ title = col i['title'], varwidth * 0.75
69
+ word_count = i['word_count'].to_s.rjust(6)
70
+ date = format_date(i['pub_date']).rjust(7)
71
+ spacer = " " * 20 # to push guid all the way off screen
72
+ guid = i['guid']
73
+
74
+ flag = i['unread'] == 1 ? '+' : ' '
75
+ flag = i['starred'] == 1 ? '*' : flag
76
+ "%s | %s | %s | %s | %s | %s | %s" % [flag, feed_title, title, word_count, date, spacer, guid]
77
+ end
78
+
79
+ # look up feed up idx
80
+ def feed_items(*feed_selection)
81
+ window_width = feed_selection[0]
82
+ feed_title = feed_selection[1].split(' ')[0..-2].join(' ')
83
+ @sqliteclient.feed_items(feed_title).map do |x|
84
+ format_item_summary x, window_width
85
+ end
86
+ end
87
+
88
+ def self.strip_item_count(folder)
89
+ folder.gsub(/\(\d+\)$/, '').strip
90
+ end
91
+
92
+ def folder_items(window_width, folder)
93
+ # strip off the count summary
94
+ folder = self.class.strip_item_count(folder)
95
+ @sqliteclient.folder_items(folder).map do |x|
96
+ format_item_summary x, window_width
97
+ end
98
+ end
99
+
100
+ def format_item(item)
101
+ res = <<-END
102
+ #{item['feed']}
103
+ #{item['feed_title']}
104
+ #{format_long_date item['pub_date']} #{item['word_count']} words
105
+
106
+
107
+ #{item['title']}#{item['author'] ? ("\n" + item['author']) : '' }
108
+
109
+ #{item['text']}
110
+
111
+ ---
112
+ #{item['link']}
113
+
114
+ END
115
+ end
116
+
117
+ def show_item(guid, increment_read_count=false)
118
+ inc_read_count = increment_read_count == '1'
119
+ res = @sqliteclient.show_item(guid.strip, inc_read_count).first
120
+ if res
121
+ format_item(res)
122
+ else
123
+ "No item found"
124
+ end
125
+ end
126
+
127
+ def star_item(guid)
128
+ @sqliteclient.star_item guid, true
129
+ end
130
+
131
+ def unstar_item(guid)
132
+ @sqliteclient.star_item guid, false
133
+ end
134
+
135
+ def delete_item(guid)
136
+ @sqliteclient.delete_item guid
137
+ end
138
+
139
+ def search_items(window_width, term)
140
+ res = @sqliteclient.search_items(term).map do |x|
141
+ format_item_summary x, window_width
142
+ end
143
+ res.empty? ? "No matches" : res
144
+ end
145
+
146
+ def cat_items(*guids)
147
+ text = guids.map do |guid|
148
+ show_item(guid, 0)
149
+ end.join( "\n+" + ('-' * 78) + "+\n" )
150
+ end
151
+ end
152
+
153
+ end
154
+
155
+ if __FILE__ == $0
156
+ puts Vnews::Display.new.send *ARGV
157
+ end
@@ -0,0 +1,4 @@
1
+ class Vnews
2
+ class SubscribeFailed < StandardError; end
3
+ end
4
+
data/lib/vnews/feed.rb ADDED
@@ -0,0 +1,77 @@
1
+ # encoding: utf-8
2
+ require 'open-uri'
3
+ require 'feed_yamlizer'
4
+ #require 'vnews/autodiscoverer'
5
+ require 'vnews/config'
6
+
7
+ class Vnews
8
+ class Feed
9
+ #include Autodiscoverer
10
+
11
+ USER_AGENT = "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4"
12
+
13
+ def self.get_feed(feed_url)
14
+ response = open(feed_url, "User-Agent" => USER_AGENT)
15
+
16
+ xml = response.read
17
+ # puts response.last_modified
18
+ $stderr.print "#{feed_url} -> Found #{response.content_type} #{response.charset}\n"
19
+
20
+ charset = response.charset || "UTF-8"
21
+
22
+ not_xml = response.content_type !~ /xml/ && xml[0,900] !~ /<?xml|<rss/
23
+ if not_xml
24
+ return nil
25
+ end
26
+ feed_yaml = FeedYamlizer.run(xml, charset)
27
+ feed_yaml
28
+ rescue OpenURI::HTTPError, REXML::ParseException, NoMethodError, Timeout::Error
29
+ $stderr.puts "ERROR (#{feed_url}): #{$!} => #{$!.message}"
30
+ end
31
+
32
+ def self.fetch_feed(xml_url, folder=nil)
33
+ f = self.get_feed(xml_url)
34
+ [xml_url, f, folder]
35
+ end
36
+
37
+ # f is the feed hash
38
+ def self.save_feed(feed_url, f, folder=nil)
39
+ # if no folder, we're just updating a feed
40
+ if folder
41
+ Vnews.sql_client.insert_feed(f[:meta][:title], feed_url, f[:meta][:link], folder)
42
+ end
43
+ f[:items].each do |item|
44
+ if item[:guid].nil? || item[:guid].strip == ''
45
+ item[:guid] = [f[:meta][:link], f[:link]].join(":::")
46
+ end
47
+ Vnews.sql_client.insert_item item.merge(:feed => feed_url, :feed_title => f[:meta][:title])
48
+ $stderr.print "."
49
+ end
50
+ rescue
51
+ puts "ERROR: dump: #{feed_url.inspect}, #{folder.inspect}: #{f.inspect} "
52
+ raise
53
+ end
54
+
55
+ def self.update_feed(feed_title)
56
+ require 'vnews/display'
57
+ feed_title = Vnews::Display.strip_item_count(feed_title)
58
+ puts "Updating feed: #{feed_title}"
59
+ feed_url = Vnews.sql_client.feed_by_title feed_title
60
+ puts "Fetching data from feed url: #{feed_url}"
61
+ # puts "Deleting feed items for #{feed_url}"
62
+ # Vnews.sql_client.delete_feed_items feed_url
63
+ f = Vnews::Feed.get_feed feed_url
64
+ save_feed feed_url, f, nil
65
+ puts "\nFeed updated"
66
+ end
67
+
68
+ def self.log(text)
69
+ $stderr.puts text
70
+ end
71
+ end
72
+ end
73
+
74
+ if __FILE__ == $0
75
+ Vnews::Feed.new(ARGV.first, ARGV.last).fetch
76
+ end
77
+
@@ -0,0 +1,29 @@
1
+ require 'vnews/feed'
2
+
3
+ class Vnews
4
+ class Folder
5
+ def self.update_folder(folder)
6
+ if folder.strip == "Starred"
7
+ puts "Sorry, you can't update the starred folder."
8
+ return
9
+ else
10
+ require 'vnews/display'
11
+ folder = Vnews::Display.strip_item_count(folder)
12
+ puts "Updating folder: #{folder.inspect}"
13
+ threads = []
14
+ feeds = []
15
+ Vnews.sql_client.feeds_in_folder(folder.strip).each do |feed|
16
+ threads << Thread.new do
17
+ feeds << Vnews::Feed.fetch_feed(feed, folder)
18
+ end
19
+ end
20
+ threads.each {|t| t.join}
21
+ puts "Saving data to database"
22
+ feeds.select {|x| x[1]}.compact.each do |feed_url, f, folder|
23
+ Vnews::Feed.save_feed feed_url, f, folder
24
+ end
25
+ puts "\nDone"
26
+ end
27
+ end
28
+ end
29
+ end
data/lib/vnews/opml.rb ADDED
@@ -0,0 +1,50 @@
1
+ require 'nokogiri'
2
+ require 'vnews/feed'
3
+
4
+ class Vnews
5
+ class Opml
6
+
7
+ CONCURRENCY = 18
8
+
9
+ def self.import(opml)
10
+ sqlclient = Vnews.sql_client
11
+ doc = Nokogiri::XML.parse(opml)
12
+ feeds = []
13
+ doc.xpath('/opml/body/outline').each_slice(CONCURRENCY) do |xs|
14
+ threads = []
15
+ xs.each do |n|
16
+ threads << Thread.new do
17
+ if n.attributes['xmlUrl']
18
+ feeds << Vnews::Feed.fetch_feed(n.attributes['xmlUrl'].to_s)
19
+ else
20
+ folder = n.attributes["title"].to_s
21
+ $stderr.print "Found folder: #{folder}\n"
22
+ n.xpath("outline[@xmlUrl]").each do |m|
23
+ feeds << Vnews::Feed.fetch_feed(m.attributes['xmlUrl'].to_s, folder)
24
+ end
25
+ end
26
+ end
27
+ end
28
+ threads.each {|t| t.join}
29
+ end
30
+
31
+ $stderr.puts "Making database records"
32
+ feeds.each do |x|
33
+ feed_url, f, folder = *x
34
+ folder ||= "Misc"
35
+ if f.nil?
36
+ $stderr.print "\nNo feed found for #{feed_url}\n"
37
+ else
38
+ Vnews::Feed.save_feed(feed_url, f, folder)
39
+ end
40
+ end
41
+ $stderr.puts "\nDone."
42
+
43
+ end
44
+ end
45
+ end
46
+
47
+ if __FILE__ == $0
48
+ opml = STDIN.read
49
+ Vnews::Opml.import(opml)
50
+ end