vnews 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +9 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +19 -0
- data/MIT-LICENSE.txt +21 -0
- data/Rakefile +37 -0
- data/bin/vnews-client +19 -0
- data/lib/create.sql +36 -0
- data/lib/vnews.rb +63 -0
- data/lib/vnews.vim +472 -0
- data/lib/vnews/autodiscoverer.rb +24 -0
- data/lib/vnews/config.rb +122 -0
- data/lib/vnews/display.rb +157 -0
- data/lib/vnews/exceptions.rb +4 -0
- data/lib/vnews/feed.rb +77 -0
- data/lib/vnews/folder.rb +29 -0
- data/lib/vnews/opml.rb +50 -0
- data/lib/vnews/sql.rb +190 -0
- data/lib/vnews/version.rb +4 -0
- data/recreate.sh +3 -0
- data/vnews.gemspec +27 -0
- metadata +111 -0
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'nokogiri'
|
2
|
+
|
3
|
+
class Vnews
|
4
|
+
class AutodiscoveryFailed < StandardError; end
|
5
|
+
|
6
|
+
module Autodiscoverer
|
7
|
+
def auto_discover(feed_url)
|
8
|
+
html = open(feed_url)
|
9
|
+
doc = Nokogiri::HTML.parse(html)
|
10
|
+
feed_url = [ 'head link[@type=application/atom+xml]',
|
11
|
+
'head link[@type=application/rss+xml]',
|
12
|
+
"head link[@type=text/xml]"].detect do |path|
|
13
|
+
doc.at(path)
|
14
|
+
end
|
15
|
+
if feed_url
|
16
|
+
feed_url
|
17
|
+
else
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
rescue Errno::ECONNRESET, Nokogiri::CSS::SyntaxError
|
21
|
+
$stderr.puts $!
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
data/lib/vnews/config.rb
ADDED
@@ -0,0 +1,122 @@
|
|
1
|
+
require 'vnews/sql'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
class Vnews
|
5
|
+
def self.sql_client
|
6
|
+
$sql_client ||= Config.load_config
|
7
|
+
end
|
8
|
+
|
9
|
+
module Config
|
10
|
+
def self.generate_config
|
11
|
+
config = Vnews.sql_client.config.inject({}) do |memo, (k, v)|
|
12
|
+
memo[k.to_s] = v
|
13
|
+
memo
|
14
|
+
end
|
15
|
+
out = [config.to_yaml.sub(/---\s+/, ''), '']
|
16
|
+
Vnews.sql_client.configured_folders.map do |x|
|
17
|
+
folder = x["folder"]
|
18
|
+
out << folder
|
19
|
+
Vnews.sql_client.feeds_in_folder(folder).each do |feed|
|
20
|
+
out << feed
|
21
|
+
end
|
22
|
+
out << ""
|
23
|
+
end
|
24
|
+
out.join("\n")
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.rewrite_config
|
28
|
+
out = generate_config
|
29
|
+
f = File.open(File.expand_path(CONFIGPATH), 'w') {|f| f.write(out)}
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.stub_config
|
33
|
+
stub = <<-END.gsub(/^ */, '')
|
34
|
+
host: localhost
|
35
|
+
database: vnews
|
36
|
+
username: root
|
37
|
+
password:
|
38
|
+
|
39
|
+
General News
|
40
|
+
http://feedproxy.google.com/economist/full_print_edition
|
41
|
+
http://feeds.feedburner.com/TheAtlanticWire
|
42
|
+
|
43
|
+
Humor
|
44
|
+
http://feed.dilbert.com/dilbert/blog
|
45
|
+
|
46
|
+
Tech
|
47
|
+
http://rss.slashdot.org/Slashdot/slashdot
|
48
|
+
http://feeds2.feedburner.com/readwriteweb
|
49
|
+
http://feedproxy.google.com/programmableweb
|
50
|
+
http://news.ycombinator.com/rss
|
51
|
+
http://daringfireball.net/index.xml
|
52
|
+
http://dailyvim.blogspot.com/feeds/posts/default
|
53
|
+
END
|
54
|
+
end
|
55
|
+
|
56
|
+
CONFIGPATH = "#{ENV['HOME']}/.vnewsrc"
|
57
|
+
|
58
|
+
def self.update_folders
|
59
|
+
f = File.read(File.expand_path(CONFIGPATH))
|
60
|
+
db, list = f.split(/^\s*$/,2)
|
61
|
+
current_folder = nil
|
62
|
+
ff = []
|
63
|
+
list.split("\n").each do |line|
|
64
|
+
line = line.strip
|
65
|
+
if line =~ /^\s*http/ # feed
|
66
|
+
ff << [line, current_folder]
|
67
|
+
elsif line =~ /^\s*\w+/ # folder
|
68
|
+
current_folder = line
|
69
|
+
end
|
70
|
+
end
|
71
|
+
puts "Using feeds and folders: #{ff.inspect}"
|
72
|
+
|
73
|
+
old_feeds = Vnews.sql_client.feeds(0).map {|x| x["feed_url"]}
|
74
|
+
new_feeds = ff.map {|feed,folder| feed}
|
75
|
+
rm_feeds = old_feeds - new_feeds
|
76
|
+
puts "Removing feeds: #{rm_feeds.inspect}"
|
77
|
+
rm_feeds.each {|x| Vnews.sql_client.delete_feed(x)}
|
78
|
+
|
79
|
+
# ff is an association between a feed and a folder
|
80
|
+
old_ff = Vnews.sql_client.configured_feeds_folders
|
81
|
+
rm_ff = old_ff - ff
|
82
|
+
puts "Removing feed-folder associations: #{rm_ff.inspect}"
|
83
|
+
rm_ff.each {|feed,folder| Vnews.sql_client.delete_feed_folder(feed,folder)}
|
84
|
+
|
85
|
+
puts "Adding feeds: #{(new_feeds - old_feeds).inspect}"
|
86
|
+
puts "Adding folder-feed associations: #{(ff - old_ff).inspect}"
|
87
|
+
feeds2 = []
|
88
|
+
threads = []
|
89
|
+
ff.each do |feed_url, folder|
|
90
|
+
threads << Thread.new do
|
91
|
+
feeds2 << Vnews::Feed.fetch_feed(feed_url, folder)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
threads.each {|t| t.join}
|
95
|
+
feeds2.each do |x|
|
96
|
+
feed_url, f, folder = *x
|
97
|
+
folder ||= "Misc"
|
98
|
+
if f.nil?
|
99
|
+
$stderr.print "\nNo feed found for #{feed_url}\n"
|
100
|
+
else
|
101
|
+
Vnews::Feed.save_feed(feed_url, f, folder)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
$stderr.puts "\nDone."
|
105
|
+
end
|
106
|
+
|
107
|
+
def self.load_config
|
108
|
+
if ! File.exists?(File.expand_path(CONFIGPATH))
|
109
|
+
return false
|
110
|
+
end
|
111
|
+
return if $sql_client
|
112
|
+
f = File.read(File.expand_path(CONFIGPATH))
|
113
|
+
top, bottom = f.split(/^\s*$/,2)
|
114
|
+
dbconfig = YAML::load top
|
115
|
+
Sql.new(dbconfig)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
if __FILE__ == $0
|
121
|
+
puts Vnews::Config.generate_config
|
122
|
+
end
|
@@ -0,0 +1,157 @@
|
|
1
|
+
require 'vnews/config'
|
2
|
+
require 'yaml'
|
3
|
+
require 'date'
|
4
|
+
|
5
|
+
class Vnews
|
6
|
+
class Display
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@sqliteclient = Vnews.sql_client
|
10
|
+
@window_width = 140
|
11
|
+
end
|
12
|
+
|
13
|
+
# returns folders as a list
|
14
|
+
def folders
|
15
|
+
@sqliteclient.folders.map do |x|
|
16
|
+
"#{x["folder"]} (#{x['count']})"
|
17
|
+
end.join("\n")
|
18
|
+
end
|
19
|
+
|
20
|
+
# returns feeds as a list, sorted alphabetically
|
21
|
+
# e.g.
|
22
|
+
# {"title"=>"Bits",
|
23
|
+
# "feed_url"=>"http://bits.blogs.nytimes.com/feed/",
|
24
|
+
# "link"=>"http://bits.blogs.nytimes.com/"}
|
25
|
+
def feeds(order)
|
26
|
+
# '0' is alphabetical, '1' is most popular first
|
27
|
+
@sqliteclient.feeds(order.to_i).map.with_index do |x, idx|
|
28
|
+
"#{ x['title'] } (#{x['item_count']})"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# returns items as a list, most recent first
|
33
|
+
# e.g.
|
34
|
+
# {"title"=>"Episode 96: Git on Rails", "guid"=>"git-on-rails",
|
35
|
+
# "feed"=>"http://feeds.feedburner.com/railscasts",
|
36
|
+
# "feed_title"=>"Railscasts", "pub_date"=>2008-03-10 00:00:00 -0400,
|
37
|
+
# "word_count"=>41}
|
38
|
+
|
39
|
+
def col(string, width)
|
40
|
+
return unless string
|
41
|
+
string[0,width].ljust(width)
|
42
|
+
end
|
43
|
+
|
44
|
+
def format_date(d)
|
45
|
+
if d.nil?
|
46
|
+
"no date"
|
47
|
+
elsif d.year != Time.now.year
|
48
|
+
d.strftime("%b %Y")
|
49
|
+
elsif d.to_date == Time.now.to_date
|
50
|
+
d.strftime("%I:%M%P")
|
51
|
+
else
|
52
|
+
d.strftime("%b %d")
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# for item display
|
57
|
+
def format_long_date(d)
|
58
|
+
if d.nil?
|
59
|
+
"[no date]"
|
60
|
+
else
|
61
|
+
d.strftime("%a %m/%d/%Y at %I:%M%p %Z")
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def format_item_summary(i, width)
|
66
|
+
varwidth = width.to_i - 31
|
67
|
+
feed_title = col i['feed_title'], varwidth * 0.25
|
68
|
+
title = col i['title'], varwidth * 0.75
|
69
|
+
word_count = i['word_count'].to_s.rjust(6)
|
70
|
+
date = format_date(i['pub_date']).rjust(7)
|
71
|
+
spacer = " " * 20 # to push guid all the way off screen
|
72
|
+
guid = i['guid']
|
73
|
+
|
74
|
+
flag = i['unread'] == 1 ? '+' : ' '
|
75
|
+
flag = i['starred'] == 1 ? '*' : flag
|
76
|
+
"%s | %s | %s | %s | %s | %s | %s" % [flag, feed_title, title, word_count, date, spacer, guid]
|
77
|
+
end
|
78
|
+
|
79
|
+
# look up feed up idx
|
80
|
+
def feed_items(*feed_selection)
|
81
|
+
window_width = feed_selection[0]
|
82
|
+
feed_title = feed_selection[1].split(' ')[0..-2].join(' ')
|
83
|
+
@sqliteclient.feed_items(feed_title).map do |x|
|
84
|
+
format_item_summary x, window_width
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def self.strip_item_count(folder)
|
89
|
+
folder.gsub(/\(\d+\)$/, '').strip
|
90
|
+
end
|
91
|
+
|
92
|
+
def folder_items(window_width, folder)
|
93
|
+
# strip off the count summary
|
94
|
+
folder = self.class.strip_item_count(folder)
|
95
|
+
@sqliteclient.folder_items(folder).map do |x|
|
96
|
+
format_item_summary x, window_width
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def format_item(item)
|
101
|
+
res = <<-END
|
102
|
+
#{item['feed']}
|
103
|
+
#{item['feed_title']}
|
104
|
+
#{format_long_date item['pub_date']} #{item['word_count']} words
|
105
|
+
|
106
|
+
|
107
|
+
#{item['title']}#{item['author'] ? ("\n" + item['author']) : '' }
|
108
|
+
|
109
|
+
#{item['text']}
|
110
|
+
|
111
|
+
---
|
112
|
+
#{item['link']}
|
113
|
+
|
114
|
+
END
|
115
|
+
end
|
116
|
+
|
117
|
+
def show_item(guid, increment_read_count=false)
|
118
|
+
inc_read_count = increment_read_count == '1'
|
119
|
+
res = @sqliteclient.show_item(guid.strip, inc_read_count).first
|
120
|
+
if res
|
121
|
+
format_item(res)
|
122
|
+
else
|
123
|
+
"No item found"
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def star_item(guid)
|
128
|
+
@sqliteclient.star_item guid, true
|
129
|
+
end
|
130
|
+
|
131
|
+
def unstar_item(guid)
|
132
|
+
@sqliteclient.star_item guid, false
|
133
|
+
end
|
134
|
+
|
135
|
+
def delete_item(guid)
|
136
|
+
@sqliteclient.delete_item guid
|
137
|
+
end
|
138
|
+
|
139
|
+
def search_items(window_width, term)
|
140
|
+
res = @sqliteclient.search_items(term).map do |x|
|
141
|
+
format_item_summary x, window_width
|
142
|
+
end
|
143
|
+
res.empty? ? "No matches" : res
|
144
|
+
end
|
145
|
+
|
146
|
+
def cat_items(*guids)
|
147
|
+
text = guids.map do |guid|
|
148
|
+
show_item(guid, 0)
|
149
|
+
end.join( "\n+" + ('-' * 78) + "+\n" )
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
end
|
154
|
+
|
155
|
+
if __FILE__ == $0
|
156
|
+
puts Vnews::Display.new.send *ARGV
|
157
|
+
end
|
data/lib/vnews/feed.rb
ADDED
@@ -0,0 +1,77 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'open-uri'
|
3
|
+
require 'feed_yamlizer'
|
4
|
+
#require 'vnews/autodiscoverer'
|
5
|
+
require 'vnews/config'
|
6
|
+
|
7
|
+
class Vnews
|
8
|
+
class Feed
|
9
|
+
#include Autodiscoverer
|
10
|
+
|
11
|
+
USER_AGENT = "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4"
|
12
|
+
|
13
|
+
def self.get_feed(feed_url)
|
14
|
+
response = open(feed_url, "User-Agent" => USER_AGENT)
|
15
|
+
|
16
|
+
xml = response.read
|
17
|
+
# puts response.last_modified
|
18
|
+
$stderr.print "#{feed_url} -> Found #{response.content_type} #{response.charset}\n"
|
19
|
+
|
20
|
+
charset = response.charset || "UTF-8"
|
21
|
+
|
22
|
+
not_xml = response.content_type !~ /xml/ && xml[0,900] !~ /<?xml|<rss/
|
23
|
+
if not_xml
|
24
|
+
return nil
|
25
|
+
end
|
26
|
+
feed_yaml = FeedYamlizer.run(xml, charset)
|
27
|
+
feed_yaml
|
28
|
+
rescue OpenURI::HTTPError, REXML::ParseException, NoMethodError, Timeout::Error
|
29
|
+
$stderr.puts "ERROR (#{feed_url}): #{$!} => #{$!.message}"
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.fetch_feed(xml_url, folder=nil)
|
33
|
+
f = self.get_feed(xml_url)
|
34
|
+
[xml_url, f, folder]
|
35
|
+
end
|
36
|
+
|
37
|
+
# f is the feed hash
|
38
|
+
def self.save_feed(feed_url, f, folder=nil)
|
39
|
+
# if no folder, we're just updating a feed
|
40
|
+
if folder
|
41
|
+
Vnews.sql_client.insert_feed(f[:meta][:title], feed_url, f[:meta][:link], folder)
|
42
|
+
end
|
43
|
+
f[:items].each do |item|
|
44
|
+
if item[:guid].nil? || item[:guid].strip == ''
|
45
|
+
item[:guid] = [f[:meta][:link], f[:link]].join(":::")
|
46
|
+
end
|
47
|
+
Vnews.sql_client.insert_item item.merge(:feed => feed_url, :feed_title => f[:meta][:title])
|
48
|
+
$stderr.print "."
|
49
|
+
end
|
50
|
+
rescue
|
51
|
+
puts "ERROR: dump: #{feed_url.inspect}, #{folder.inspect}: #{f.inspect} "
|
52
|
+
raise
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.update_feed(feed_title)
|
56
|
+
require 'vnews/display'
|
57
|
+
feed_title = Vnews::Display.strip_item_count(feed_title)
|
58
|
+
puts "Updating feed: #{feed_title}"
|
59
|
+
feed_url = Vnews.sql_client.feed_by_title feed_title
|
60
|
+
puts "Fetching data from feed url: #{feed_url}"
|
61
|
+
# puts "Deleting feed items for #{feed_url}"
|
62
|
+
# Vnews.sql_client.delete_feed_items feed_url
|
63
|
+
f = Vnews::Feed.get_feed feed_url
|
64
|
+
save_feed feed_url, f, nil
|
65
|
+
puts "\nFeed updated"
|
66
|
+
end
|
67
|
+
|
68
|
+
def self.log(text)
|
69
|
+
$stderr.puts text
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
if __FILE__ == $0
|
75
|
+
Vnews::Feed.new(ARGV.first, ARGV.last).fetch
|
76
|
+
end
|
77
|
+
|
data/lib/vnews/folder.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'vnews/feed'
|
2
|
+
|
3
|
+
class Vnews
|
4
|
+
class Folder
|
5
|
+
def self.update_folder(folder)
|
6
|
+
if folder.strip == "Starred"
|
7
|
+
puts "Sorry, you can't update the starred folder."
|
8
|
+
return
|
9
|
+
else
|
10
|
+
require 'vnews/display'
|
11
|
+
folder = Vnews::Display.strip_item_count(folder)
|
12
|
+
puts "Updating folder: #{folder.inspect}"
|
13
|
+
threads = []
|
14
|
+
feeds = []
|
15
|
+
Vnews.sql_client.feeds_in_folder(folder.strip).each do |feed|
|
16
|
+
threads << Thread.new do
|
17
|
+
feeds << Vnews::Feed.fetch_feed(feed, folder)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
threads.each {|t| t.join}
|
21
|
+
puts "Saving data to database"
|
22
|
+
feeds.select {|x| x[1]}.compact.each do |feed_url, f, folder|
|
23
|
+
Vnews::Feed.save_feed feed_url, f, folder
|
24
|
+
end
|
25
|
+
puts "\nDone"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/vnews/opml.rb
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
require 'nokogiri'
|
2
|
+
require 'vnews/feed'
|
3
|
+
|
4
|
+
class Vnews
|
5
|
+
class Opml
|
6
|
+
|
7
|
+
CONCURRENCY = 18
|
8
|
+
|
9
|
+
def self.import(opml)
|
10
|
+
sqlclient = Vnews.sql_client
|
11
|
+
doc = Nokogiri::XML.parse(opml)
|
12
|
+
feeds = []
|
13
|
+
doc.xpath('/opml/body/outline').each_slice(CONCURRENCY) do |xs|
|
14
|
+
threads = []
|
15
|
+
xs.each do |n|
|
16
|
+
threads << Thread.new do
|
17
|
+
if n.attributes['xmlUrl']
|
18
|
+
feeds << Vnews::Feed.fetch_feed(n.attributes['xmlUrl'].to_s)
|
19
|
+
else
|
20
|
+
folder = n.attributes["title"].to_s
|
21
|
+
$stderr.print "Found folder: #{folder}\n"
|
22
|
+
n.xpath("outline[@xmlUrl]").each do |m|
|
23
|
+
feeds << Vnews::Feed.fetch_feed(m.attributes['xmlUrl'].to_s, folder)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
threads.each {|t| t.join}
|
29
|
+
end
|
30
|
+
|
31
|
+
$stderr.puts "Making database records"
|
32
|
+
feeds.each do |x|
|
33
|
+
feed_url, f, folder = *x
|
34
|
+
folder ||= "Misc"
|
35
|
+
if f.nil?
|
36
|
+
$stderr.print "\nNo feed found for #{feed_url}\n"
|
37
|
+
else
|
38
|
+
Vnews::Feed.save_feed(feed_url, f, folder)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
$stderr.puts "\nDone."
|
42
|
+
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
if __FILE__ == $0
|
48
|
+
opml = STDIN.read
|
49
|
+
Vnews::Opml.import(opml)
|
50
|
+
end
|