murlsh 1.4.1 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/.htaccess +0 -3
- data/Gemfile +41 -0
- data/README.textile +68 -6
- data/Rakefile +65 -152
- data/config.ru +13 -2
- data/config.yaml +9 -4
- data/db/migrate/20110213023123_init.rb +20 -0
- data/lib/murlsh/atom_body.rb +78 -0
- data/lib/murlsh/atom_server.rb +38 -0
- data/lib/murlsh/auth.rb +12 -0
- data/lib/murlsh/build_query.rb +1 -1
- data/lib/murlsh/cat_files.rb +17 -0
- data/lib/murlsh/delicious_parse.rb +1 -0
- data/lib/murlsh/dispatch.rb +25 -11
- data/lib/murlsh/etag_add_encoding.rb +1 -1
- data/lib/murlsh/feed_body.rb +36 -0
- data/lib/murlsh/img_store.rb +27 -28
- data/lib/murlsh/install.rb +1 -0
- data/lib/murlsh/json_body.rb +5 -2
- data/lib/murlsh/json_server.rb +9 -13
- data/lib/murlsh/m3u_body.rb +28 -0
- data/lib/murlsh/m3u_server.rb +50 -0
- data/lib/murlsh/markup.rb +1 -1
- data/lib/murlsh/plugin.rb +5 -0
- data/lib/murlsh/podcast_server.rb +44 -0
- data/lib/murlsh/pop_server.rb +78 -0
- data/lib/murlsh/random_server.rb +41 -0
- data/lib/murlsh/rss_body.rb +46 -0
- data/lib/murlsh/rss_server.rb +38 -0
- data/lib/murlsh/search_conditions.rb +2 -2
- data/lib/murlsh/uri_ask.rb +2 -2
- data/lib/murlsh/url_body.rb +21 -6
- data/lib/murlsh/url_result_set.rb +2 -2
- data/lib/murlsh/url_server.rb +19 -16
- data/lib/murlsh/write_ordered_hash.rb +17 -0
- data/lib/murlsh.rb +13 -2
- data/murlsh.gemspec +41 -194
- data/plugins/add_post_60_notify_hubs.rb +3 -2
- data/plugins/add_pre_30_unajax_twitter.rb +1 -1
- data/plugins/add_pre_40_thumbnail_shortcuts.rb +23 -0
- data/plugins/add_pre_45_supplied_thumbnail.rb +4 -9
- data/plugins/add_pre_50_media_thumbnail.rb +4 -9
- data/plugins/add_pre_50_open_graph_image.rb +4 -8
- data/plugins/add_pre_60_github_title.rb +1 -1
- data/plugins/add_pre_65_html_thumb.rb +3 -8
- data/plugins/add_pre_65_img_thumb.rb +4 -9
- data/plugins/avatar_50_gravatar.rb +2 -1
- data/plugins/store_asset_40_s3.rb +40 -0
- data/plugins/store_asset_50_local.rb +22 -0
- data/public/js/js.js +0 -7
- data/spec/auth_spec.rb +7 -0
- data/spec/cat_files_spec.rb +49 -0
- data/spec/img_store_spec.rb +24 -8
- metadata +119 -76
- data/VERSION +0 -1
- data/lib/murlsh/build_md5.rb +0 -12
- data/lib/murlsh/head_from_get.rb +0 -15
- data/plugins/add_post_50_update_feed.rb +0 -84
- data/plugins/add_post_50_update_m3u.rb +0 -35
- data/plugins/add_post_50_update_podcast.rb +0 -44
- data/plugins/add_post_50_update_rss.rb +0 -51
- data/plugins/add_pre_60_s3_image.rb +0 -35
data/.gitignore
ADDED
data/.htaccess
CHANGED
@@ -1,11 +1,8 @@
|
|
1
1
|
Options -Indexes
|
2
2
|
|
3
|
-
AddOutputFilterByType DEFLATE application/atom+xml
|
4
3
|
AddOutputFilterByType DEFLATE application/javascript
|
5
|
-
AddOutputFilterByType DEFLATE application/rss+xml
|
6
4
|
AddOutputFilterByType DEFLATE application/xhtml+xml
|
7
5
|
AddOutputFilterByType DEFLATE application/xml
|
8
|
-
AddOutputFilterByType DEFLATE audio/x-mpegurl
|
9
6
|
AddOutputFilterByType DEFLATE text/css
|
10
7
|
AddOutputFilterByType DEFLATE text/html
|
11
8
|
|
data/Gemfile
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
source :rubygems
|
2
|
+
|
3
|
+
# This does not use 'gemspec' because it creates a Gemfile.lock that treats
|
4
|
+
# murlsh as a "gem" and will not work on Heroku, which sees murlsh as an
|
5
|
+
# "app".
|
6
|
+
|
7
|
+
# Dependencies duplicated in gemspec and here until a better solution is
|
8
|
+
# found.
|
9
|
+
|
10
|
+
%w{
|
11
|
+
activerecord >= 2.3.4
|
12
|
+
aws-s3 ~> 0.6
|
13
|
+
bcrypt-ruby >= 2.1.2
|
14
|
+
builder > 0
|
15
|
+
htmlentities >= 4.2.0
|
16
|
+
json >= 1.2.3
|
17
|
+
nokogiri ~> 1.0
|
18
|
+
plumnailer >= 0.1.3
|
19
|
+
postrank-uri ~> 1.0
|
20
|
+
public_suffix_service ~> 0.0
|
21
|
+
push-notify >= 0.1.0
|
22
|
+
rack >= 1.0.0
|
23
|
+
rack-cache >= 0.5.2
|
24
|
+
rack-rewrite >= 1.0.2
|
25
|
+
rack-throttle >= 0.3.0
|
26
|
+
rmagick >= 1.15.14
|
27
|
+
rmail ~> 1.0
|
28
|
+
sqlite3 ~> 1.3
|
29
|
+
tinyatom >= 0.3.4
|
30
|
+
treetop ~> 1.4
|
31
|
+
twitter >= 0.9.12
|
32
|
+
}.each_slice(3) { |g,o,v| gem g, "#{o} #{v}" }
|
33
|
+
|
34
|
+
group :development do
|
35
|
+
%w{
|
36
|
+
fakeweb ~> 1.3
|
37
|
+
flog >= 2.5.0
|
38
|
+
rack-test ~> 0.5
|
39
|
+
rspec ~> 2.0
|
40
|
+
}.each_slice(3) { |g,o,v| gem g, "#{o} #{v}" }
|
41
|
+
end
|
data/README.textile
CHANGED
@@ -6,7 +6,7 @@ Host your bookmarks or maintain a link blog.
|
|
6
6
|
* generates Atom and RSS feeds
|
7
7
|
* generates podcast RSS feed and m3u file for all audio urls
|
8
8
|
* generates json and jsonp feeds for client-side inclusion in other sites
|
9
|
-
* search
|
9
|
+
* search, all output formats can be filtered by search criteria
|
10
10
|
* uses HTML5 audio for mp3 and ogg urls
|
11
11
|
* looks good on iPhone
|
12
12
|
* PubSubHubbub notification
|
@@ -14,6 +14,7 @@ Host your bookmarks or maintain a link blog.
|
|
14
14
|
* rack interface
|
15
15
|
* Gravatar support
|
16
16
|
* generates import scripts from delicious api exports
|
17
|
+
* optionally store thumbnails in S3
|
17
18
|
|
18
19
|
See "http://urls.matthewm.boedicker.org/":http://urls.matthewm.boedicker.org/ for example.
|
19
20
|
|
@@ -34,13 +35,60 @@ rake init
|
|
34
35
|
</code>
|
35
36
|
</pre>
|
36
37
|
|
38
|
+
h2. Heroku
|
39
|
+
|
40
|
+
<pre>
|
41
|
+
<code>
|
42
|
+
gem install heroku
|
43
|
+
heroku keys:add
|
44
|
+
</code>
|
45
|
+
</pre>
|
46
|
+
|
47
|
+
Create a fork on github and clone it or clone public url:
|
48
|
+
|
49
|
+
<pre>
|
50
|
+
<code>
|
51
|
+
git clone git://github.com/mmb/murlsh.git
|
52
|
+
cd murlsh
|
53
|
+
bundle install
|
54
|
+
heroku create <choose a name>
|
55
|
+
rake heroku:config
|
56
|
+
heroku info
|
57
|
+
rake config[root_url,<your app's Heroku url>]
|
58
|
+
rake config[s3_bucket,<your S3 bucket name>]
|
59
|
+
rake config[s3_id,<your S3 id>]
|
60
|
+
rake config[s3_secret,<your S3 secret>]
|
61
|
+
</code>
|
62
|
+
</pre>
|
63
|
+
|
64
|
+
S3 is used for thumbnail storage because Heroku cannot write local files.
|
65
|
+
|
66
|
+
Other config in config.yaml is optional.
|
67
|
+
|
68
|
+
<pre>
|
69
|
+
<code>
|
70
|
+
rake user:add
|
71
|
+
git add .
|
72
|
+
git commit
|
73
|
+
git push heroku master
|
74
|
+
heroku rake db:migrate
|
75
|
+
</code>
|
76
|
+
</pre>
|
77
|
+
|
37
78
|
h2. Development
|
38
79
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
80
|
+
Create a fork on Github and clone it.
|
81
|
+
|
82
|
+
<pre>
|
83
|
+
<code>
|
84
|
+
rake config[root_url,http://localhost:9292/]
|
85
|
+
rake db:migrate
|
86
|
+
rake user:add
|
87
|
+
rackup
|
88
|
+
</code>
|
89
|
+
</pre>
|
90
|
+
|
91
|
+
Browse to http://localhost:9292/
|
44
92
|
|
45
93
|
h1. Updating
|
46
94
|
|
@@ -60,6 +108,18 @@ h2. Recent urls
|
|
60
108
|
* http://your_root/json.json
|
61
109
|
* http://your_root/json.json?callback=x (jsonp)
|
62
110
|
|
111
|
+
h1. Thumbnails
|
112
|
+
|
113
|
+
Thumbnail images are generated from added urls using plumnailer. They are
|
114
|
+
scaled down to 'thumbnail_max_side' in config.yaml and stored locally.
|
115
|
+
|
116
|
+
Thumbnails can also be manually specified by passing their url as the
|
117
|
+
'thumbnail' parameter when adding a url. They are also scaled and stored
|
118
|
+
locally.
|
119
|
+
|
120
|
+
The plugin add_pre_40_thumbnail_shortcuts.rb can be used to specify
|
121
|
+
short names for frequently used thumbnail urls that can be passed in instead.
|
122
|
+
|
63
123
|
h1. Plugins
|
64
124
|
|
65
125
|
Classes in the plugins directory can be used to change behavior at certain
|
@@ -79,7 +139,9 @@ Plugin hooks
|
|
79
139
|
|add_pre|called before a new url is saved|url, config hash|undefined|
|
80
140
|
|add_post|called after a new url is saved|url, config hash|undefined|
|
81
141
|
|avatar|called to get an avatar url from an email md5 sum|avatar url, url, config hash|avatar url|
|
142
|
+
|store_asset|store an asset somewhere where it can be loaded by url|name, data, config hash|asset url if successfully stored|
|
82
143
|
|url_display_add|called to display additional information after urls|markup builder, url, config hash|undefined|
|
144
|
+
|url_display_pre|called to modify a url on-the-fly before display, does not change database|url, rack request, config hash|undefined|
|
83
145
|
|
84
146
|
h1. PubSubHubbub
|
85
147
|
|
data/Rakefile
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
$:.unshift(File.join(File.dirname(__FILE__), 'lib'))
|
2
2
|
|
3
|
-
require 'cgi'
|
4
3
|
require 'digest/md5'
|
5
4
|
require 'net/http'
|
5
|
+
require 'logger'
|
6
6
|
require 'open-uri'
|
7
7
|
require 'pp'
|
8
8
|
require 'set'
|
@@ -11,18 +11,23 @@ require 'yaml'
|
|
11
11
|
|
12
12
|
require 'active_record'
|
13
13
|
require 'RMagick'
|
14
|
-
require 'sqlite3'
|
15
14
|
|
16
15
|
require 'murlsh'
|
17
16
|
|
18
17
|
def gem_not_found(gem_name)
|
19
|
-
puts "#{gem_name} not found
|
18
|
+
puts "#{gem_name} gem not found"
|
20
19
|
end
|
21
20
|
|
22
21
|
config = YAML.load_file('config.yaml')
|
23
22
|
|
23
|
+
# for Heroku
|
24
|
+
db_config_file = File.join(File.dirname(__FILE__), 'config', 'database.yml')
|
25
|
+
if File.exist?(db_config_file)
|
26
|
+
config['db'] = YAML.load_file(db_config_file)['production']
|
27
|
+
end
|
28
|
+
|
24
29
|
desc 'Initialize a new installation.'
|
25
|
-
task :init => %w{db:
|
30
|
+
task :init => %w{db:migrate user:add compress} do
|
26
31
|
puts <<-eos
|
27
32
|
|
28
33
|
Things you might want to do now:
|
@@ -46,8 +51,7 @@ namespace :db do
|
|
46
51
|
|
47
52
|
desc 'Delete the last url added.'
|
48
53
|
task :delete_last_url do
|
49
|
-
ActiveRecord::Base.establish_connection(
|
50
|
-
:database => config.fetch('db_file'))
|
54
|
+
ActiveRecord::Base.establish_connection config.fetch('db')
|
51
55
|
|
52
56
|
last = Murlsh::Url.find(:last, :order => 'time')
|
53
57
|
pp last
|
@@ -57,46 +61,43 @@ namespace :db do
|
|
57
61
|
|
58
62
|
desc 'Check for duplicate URLs.'
|
59
63
|
task :dupcheck do
|
60
|
-
|
61
|
-
|
64
|
+
ActiveRecord::Base.establish_connection config.fetch('db')
|
65
|
+
|
62
66
|
h = {}
|
63
|
-
|
64
|
-
h[
|
67
|
+
Murlsh::Url.all.each do |mu|
|
68
|
+
h[mu.url] = h.fetch(mu.url, []).push([mu.id, mu.time])
|
65
69
|
end
|
66
70
|
h.find_all { |k,v| v.size > 1 }.each do |k,v|
|
67
71
|
puts k
|
68
|
-
v.each { |id,time| puts " #{id} #{time}" }
|
72
|
+
v.each { |id,time| puts " id #{id} (#{time})" }
|
69
73
|
end
|
70
74
|
end
|
71
75
|
|
72
|
-
desc '
|
73
|
-
task :
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
time TIMESTAMP,
|
79
|
-
url TEXT,
|
80
|
-
email TEXT,
|
81
|
-
name TEXT,
|
82
|
-
title TEXT,
|
83
|
-
content_length INTEGER,
|
84
|
-
content_type TEXT,
|
85
|
-
via TEXT,
|
86
|
-
thumbnail_url TEXT);
|
87
|
-
'
|
88
|
-
db.execute 'CREATE INDEX IF NOT EXISTS urls_time_desc ON urls (time DESC);'
|
76
|
+
desc 'Migrate the database.'
|
77
|
+
task :migrate do
|
78
|
+
ActiveRecord::Base.establish_connection config.fetch('db')
|
79
|
+
ActiveRecord::Base.logger = Logger.new($stdout)
|
80
|
+
ActiveRecord::Migration.verbose = true
|
81
|
+
ActiveRecord::Migrator.migrate 'db/migrate'
|
89
82
|
end
|
90
83
|
|
91
84
|
desc 'Interact with the database.'
|
92
85
|
task :shell do
|
93
|
-
|
86
|
+
db = config.fetch('db')
|
87
|
+
command = case db.fetch('adapter')
|
88
|
+
when 'sqlite3'; "sqlite3 #{db.fetch('database')}"
|
89
|
+
end
|
90
|
+
|
91
|
+
if command
|
92
|
+
exec command
|
93
|
+
else
|
94
|
+
puts "Don't know how to launch shell for database '#{db.fetch('adapter')}'"
|
95
|
+
end
|
94
96
|
end
|
95
97
|
|
96
98
|
desc 'Search urls and titles in the database.'
|
97
99
|
task :grep, :search do |t,args|
|
98
|
-
ActiveRecord::Base.establish_connection(
|
99
|
-
:database => config.fetch('db_file'))
|
100
|
+
ActiveRecord::Base.establish_connection config.fetch('db')
|
100
101
|
|
101
102
|
like = "%#{args.search}%"
|
102
103
|
Murlsh::Url.all(:conditions =>
|
@@ -140,7 +141,7 @@ begin
|
|
140
141
|
t.verbose = true
|
141
142
|
end
|
142
143
|
rescue LoadError
|
143
|
-
|
144
|
+
gem_not_found 'rspec'
|
144
145
|
end
|
145
146
|
|
146
147
|
desc 'Test remote title fetch for a URL and show errors.'
|
@@ -151,8 +152,7 @@ end
|
|
151
152
|
|
152
153
|
desc 'Try to fetch the title for a url and update it in the database.'
|
153
154
|
task :title_fetch, :url_id do |t, args|
|
154
|
-
ActiveRecord::Base.establish_connection(
|
155
|
-
:database => config.fetch('db_file'))
|
155
|
+
ActiveRecord::Base.establish_connection config.fetch('db')
|
156
156
|
url = Murlsh::Url.find(args.url_id)
|
157
157
|
puts "Url: #{url.url}"
|
158
158
|
puts "Previous title: #{url.title}"
|
@@ -176,51 +176,6 @@ namespace :user do
|
|
176
176
|
|
177
177
|
end
|
178
178
|
|
179
|
-
# Validate a document with the W3C validation service.
|
180
|
-
def validate_html(check_url, options={})
|
181
|
-
opts = {
|
182
|
-
:validator_host => 'validator.w3.org',
|
183
|
-
:validator_port => 80,
|
184
|
-
:validator_path =>
|
185
|
-
"/check?uri=#{CGI::escape(check_url)}&charset=(detect+automatically)&doctype=Inline&group=0",
|
186
|
-
}.merge options
|
187
|
-
|
188
|
-
net_http = Net::HTTP.new(opts[:validator_host], opts[:validator_port])
|
189
|
-
# net_http.set_debug_output(STDOUT)
|
190
|
-
|
191
|
-
net_http.start do |http|
|
192
|
-
resp = http.request_head(opts[:validator_path])
|
193
|
-
result = {
|
194
|
-
:response => resp
|
195
|
-
}
|
196
|
-
if Net::HTTPSuccess === resp
|
197
|
-
result.merge!(
|
198
|
-
:status => resp['X-W3C-Validator-Status'],
|
199
|
-
:errors => resp['X-W3C-Validator-Errors'],
|
200
|
-
:warnings => resp['X-W3C-Validator-Warnings']
|
201
|
-
)
|
202
|
-
end
|
203
|
-
result
|
204
|
-
end
|
205
|
-
|
206
|
-
end
|
207
|
-
|
208
|
-
namespace :validate do
|
209
|
-
|
210
|
-
desc 'Validate HTML.'
|
211
|
-
task :html do
|
212
|
-
check_url = config['root_url']
|
213
|
-
print "validating #{check_url} : "
|
214
|
-
result = validate_html(check_url)
|
215
|
-
if Net::HTTPSuccess === result[:response]
|
216
|
-
puts "#{result[:status]} (#{result[:errors]} errors, #{result[:warnings]} warnings)"
|
217
|
-
else
|
218
|
-
puts result[:response]
|
219
|
-
end
|
220
|
-
end
|
221
|
-
|
222
|
-
end
|
223
|
-
|
224
179
|
desc 'Generate a shell script that will post a new url.'
|
225
180
|
task :post_sh do
|
226
181
|
puts <<EOS
|
@@ -238,25 +193,14 @@ curl \\
|
|
238
193
|
EOS
|
239
194
|
end
|
240
195
|
|
241
|
-
# Concatenate some files and return the result as a string.
|
242
|
-
def cat(in_files, sep=nil)
|
243
|
-
result = ''
|
244
|
-
in_files.each do |fname|
|
245
|
-
open(fname) do |h|
|
246
|
-
while (line = h.gets) do; result << line; end
|
247
|
-
result << sep if sep
|
248
|
-
end
|
249
|
-
end
|
250
|
-
result
|
251
|
-
end
|
252
|
-
|
253
196
|
directory 'public/css'
|
254
197
|
|
255
198
|
namespace :css do
|
256
199
|
|
257
200
|
desc 'Combine and compress css.'
|
258
201
|
task :compress => ['public/css'] do
|
259
|
-
combined =
|
202
|
+
combined = Murlsh.cat_files(
|
203
|
+
config['css_files'].map { |x| "public/#{x}" }, "\n")
|
260
204
|
|
261
205
|
md5sum = Digest::MD5.hexdigest(combined)
|
262
206
|
|
@@ -271,11 +215,7 @@ namespace :css do
|
|
271
215
|
|
272
216
|
unless config['css_compressed'] == compressed_url
|
273
217
|
config['css_compressed'] = compressed_url
|
274
|
-
config.
|
275
|
-
config.each_value do |v|
|
276
|
-
v.extend(Murlsh::YamlOrderedHash) if v.is_a?(Hash)
|
277
|
-
end
|
278
|
-
open('config.yaml', 'w') { |f| YAML.dump(config, f) }
|
218
|
+
Murlsh.write_ordered_hash config, 'config.yaml'
|
279
219
|
puts "updated config with css_compressed = #{compressed_url}"
|
280
220
|
end
|
281
221
|
end
|
@@ -290,7 +230,8 @@ namespace :js do
|
|
290
230
|
|
291
231
|
desc 'Combine and compress javascript.'
|
292
232
|
task :compress => ['public/js'] do
|
293
|
-
combined =
|
233
|
+
combined = Murlsh.cat_files(
|
234
|
+
config['js_files'].map { |x| "public/#{x}" } )
|
294
235
|
|
295
236
|
compressed = Net::HTTP.post_form(
|
296
237
|
URI.parse('http://closure-compiler.appspot.com/compile'), {
|
@@ -313,11 +254,7 @@ namespace :js do
|
|
313
254
|
|
314
255
|
unless config['js_compressed'] == compressed_url
|
315
256
|
config['js_compressed'] = compressed_url
|
316
|
-
config.
|
317
|
-
config.each_value do |v|
|
318
|
-
v.extend(Murlsh::YamlOrderedHash) if v.is_a?(Hash)
|
319
|
-
end
|
320
|
-
open('config.yaml', 'w') { |f| YAML.dump(config, f) }
|
257
|
+
Murlsh.write_ordered_hash config, 'config.yaml'
|
321
258
|
puts "updated config with js_compressed = #{compressed_url}"
|
322
259
|
end
|
323
260
|
end
|
@@ -326,7 +263,7 @@ namespace :js do
|
|
326
263
|
task :jslint do
|
327
264
|
local_jslint = 'jslint_rhino.js'
|
328
265
|
open(local_jslint, 'w') do |f|
|
329
|
-
f.write(
|
266
|
+
f.write(Murlsh.cat_files(%w{
|
330
267
|
https://github.com/AndyStricker/JSLint/raw/rhinocmdline/fulljslint.js
|
331
268
|
https://github.com/AndyStricker/JSLint/raw/rhinocmdline/rhino.js
|
332
269
|
}))
|
@@ -352,11 +289,10 @@ namespace :thumb do
|
|
352
289
|
|
353
290
|
desc 'Check that local thumbnails in database are consistent with filesystem.'
|
354
291
|
task :check do
|
355
|
-
ActiveRecord::Base.establish_connection
|
356
|
-
:database => config.fetch('db_file')
|
292
|
+
ActiveRecord::Base.establish_connection config.fetch('db')
|
357
293
|
used_thumbnails = Set.new
|
358
294
|
Murlsh::Url.all(
|
359
|
-
:conditions => "thumbnail_url
|
295
|
+
:conditions => "thumbnail_url LIKE 'img/thumb/%'").each do |u|
|
360
296
|
identity = "url #{u.id} (#{u.url})"
|
361
297
|
|
362
298
|
path = File.join(%w{public}.concat(File.split(u.thumbnail_url)))
|
@@ -424,50 +360,27 @@ EOS
|
|
424
360
|
|
425
361
|
end
|
426
362
|
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
gemspec.executables = %w{murlsh}
|
437
|
-
|
438
|
-
# gemspec.signing_key = '/home/mmb/src/keys/gem-private_key.pem'
|
439
|
-
# gemspec.cert_chain = %w{/home/mmb/src/keys/gem-public_cert.pem}
|
440
|
-
|
441
|
-
%w{
|
442
|
-
activerecord >= 2.3.4
|
443
|
-
bcrypt-ruby >= 2.1.2
|
444
|
-
builder >= 2.1.2
|
445
|
-
htmlentities >= 4.2.0
|
446
|
-
json >= 1.2.3
|
447
|
-
nokogiri ~> 1.0
|
448
|
-
plumnailer >= 0.1.0
|
449
|
-
postrank-uri ~> 1.0
|
450
|
-
public_suffix_service ~> 0.0
|
451
|
-
push-notify >= 0.1.0
|
452
|
-
rack >= 1.0.0
|
453
|
-
rack-cache >= 0.5.2
|
454
|
-
rack-rewrite >= 1.0.2
|
455
|
-
rack-throttle >= 0.3.0
|
456
|
-
rmagick >= 1.15.14
|
457
|
-
sqlite3 ~> 1.3
|
458
|
-
tinyatom >= 0.3.3
|
459
|
-
treetop ~> 1.4
|
460
|
-
twitter >= 0.9.12
|
461
|
-
}.each_slice(3) { |g,o,v| gemspec.add_dependency(g, "#{o} #{v}") }
|
462
|
-
%w{
|
463
|
-
fakeweb ~> 1.3
|
464
|
-
flog >= 2.5.0
|
465
|
-
rack-test ~> 0.5
|
466
|
-
rspec ~> 2.0
|
467
|
-
}.each_slice(3) do |g,o,v|
|
468
|
-
gemspec.add_development_dependency(g, "#{o} #{v}")
|
469
|
-
end
|
363
|
+
desc 'Set options in config.yaml.'
|
364
|
+
task :config, :key, :value do |t, args|
|
365
|
+
orig_value = config[args.key]
|
366
|
+
if args.value != orig_value
|
367
|
+
config[args.key] = args.value
|
368
|
+
Murlsh.write_ordered_hash config, 'config.yaml'
|
369
|
+
puts "updated '#{args.key}' '#{orig_value}' => '#{args.value}'"
|
370
|
+
else
|
371
|
+
puts "'#{args.key}' is already set to '#{args.value}'"
|
470
372
|
end
|
471
|
-
|
472
|
-
|
373
|
+
end
|
374
|
+
|
375
|
+
namespace :heroku do
|
376
|
+
|
377
|
+
desc 'Set config options for deployment on Heroku.'
|
378
|
+
task :config => %w{compress} do
|
379
|
+
config.delete 'cache_entitystore'
|
380
|
+
config.delete 'cache_metastore'
|
381
|
+
|
382
|
+
Murlsh.write_ordered_hash config, 'config.yaml'
|
383
|
+
puts 'removed cache_entitystore and cache_metastore from config, Rack::Cache disabled'
|
384
|
+
end
|
385
|
+
|
473
386
|
end
|
data/config.ru
CHANGED
@@ -3,6 +3,7 @@ $:.unshift(File.join(File.dirname(__FILE__), 'lib'))
|
|
3
3
|
require 'uri'
|
4
4
|
require 'yaml'
|
5
5
|
|
6
|
+
require 'rack'
|
6
7
|
require 'rack/cache'
|
7
8
|
require 'rack/rewrite'
|
8
9
|
require 'rack/throttle'
|
@@ -11,18 +12,29 @@ require 'murlsh'
|
|
11
12
|
|
12
13
|
config = YAML.load_file('config.yaml')
|
13
14
|
|
15
|
+
# for Heroku
|
16
|
+
db_config_file = File.join(File.dirname(__FILE__), 'config', 'database.yml')
|
17
|
+
if File.exist?(db_config_file)
|
18
|
+
config['db'] = YAML.load_file(db_config_file)['production']
|
19
|
+
end
|
20
|
+
|
14
21
|
# use Rack::ShowExceptions
|
15
22
|
# no more than 1024 requests per day per ip
|
16
23
|
use Rack::Throttle::Daily, :max => 1024
|
17
|
-
|
24
|
+
|
25
|
+
if !config['cache_metastore'].to_s.empty? and
|
26
|
+
!config['cache_entitystore'].to_s.empty?
|
18
27
|
use Rack::Cache,
|
19
28
|
:verbose => true,
|
20
29
|
:metastore => config['cache_metastore'],
|
21
30
|
:entitystore => config['cache_entitystore']
|
22
31
|
end
|
32
|
+
|
23
33
|
use Rack::ConditionalGet
|
24
34
|
use Murlsh::EtagAddEncoding
|
25
35
|
use Rack::Deflater
|
36
|
+
use Rack::Head
|
37
|
+
use Rack::ETag
|
26
38
|
use Murlsh::FarFutureExpires, :patterns => [
|
27
39
|
%r{[\da-z]{32}\.(?:gif|jpe?g|png)$}i,
|
28
40
|
%r{\.gen\.(css|js)$}
|
@@ -32,7 +44,6 @@ feed_url = URI.join(config.fetch('root_url'), config.fetch('feed_file'))
|
|
32
44
|
use Murlsh::MustRevalidate, :patterns => %r{^#{Regexp.escape(feed_url.path)}$}
|
33
45
|
|
34
46
|
use Rack::Static, :urls => %w{/css/ /img/ /js/}, :root => 'public'
|
35
|
-
use Rack::Static, :urls => %w{/atom.atom /m3u.m3u /podcast.rss /rss.rss}
|
36
47
|
|
37
48
|
use Rack::Rewrite do
|
38
49
|
r301 '/atom.xml', feed_url.to_s
|
data/config.yaml
CHANGED
@@ -5,7 +5,9 @@ cache_metastore: file:tmp/cache/rack/meta
|
|
5
5
|
css_files:
|
6
6
|
- css/jquery.jgrowl.css
|
7
7
|
- css/screen.css
|
8
|
-
|
8
|
+
db:
|
9
|
+
adapter: sqlite3
|
10
|
+
database: murlsh.db
|
9
11
|
feed_file: atom.atom
|
10
12
|
gravatar_size: 32
|
11
13
|
js_files:
|
@@ -13,12 +15,12 @@ js_files:
|
|
13
15
|
- js/jquery.jgrowl_compressed.js
|
14
16
|
- js/twitter-text-1.3.1.js
|
15
17
|
- js/js.js
|
16
|
-
meta_tag_description:
|
18
|
+
meta_tag_description:
|
17
19
|
meta_tag_verify-v1:
|
18
20
|
meta_tag_viewport: width=device-width,minimum-scale=1.0,maximum-scale=1.0
|
19
21
|
num_posts_feed: 25
|
20
22
|
num_posts_page: 25
|
21
|
-
page_title:
|
23
|
+
page_title: murlsh
|
22
24
|
pubsubhubbub_hubs: []
|
23
25
|
|
24
26
|
quick_search:
|
@@ -29,6 +31,9 @@ quick_search:
|
|
29
31
|
video: hulu.com vimeo.com www.ted.com youtube.com
|
30
32
|
wikipedia: wikipedia.org
|
31
33
|
root_url: http://urls.matthewm.boedicker.org/
|
34
|
+
s3_bucket:
|
35
|
+
s3_id:
|
36
|
+
s3_secret:
|
32
37
|
show_names: true
|
33
38
|
thumbnail_max_side: 90
|
34
|
-
user_agent: murlsh (
|
39
|
+
user_agent: murlsh (https://github.com/mmb/murlsh)
|
@@ -0,0 +1,20 @@
|
|
1
|
+
class Init < ActiveRecord::Migration
|
2
|
+
|
3
|
+
def self.up
|
4
|
+
create_table :urls do |t|
|
5
|
+
# t.integer :id
|
6
|
+
t.integer :content_length
|
7
|
+
t.string :content_type
|
8
|
+
t.string :email
|
9
|
+
t.string :name
|
10
|
+
t.timestamp :time
|
11
|
+
t.string :thumbnail_url
|
12
|
+
t.string :title
|
13
|
+
t.string :url
|
14
|
+
t.string :via
|
15
|
+
end
|
16
|
+
|
17
|
+
add_index :urls, :time
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
require 'uri'
|
2
|
+
|
3
|
+
require 'tinyatom'
|
4
|
+
|
5
|
+
require 'murlsh'
|
6
|
+
|
7
|
+
module Murlsh
|
8
|
+
|
9
|
+
# Atom feed builder.
|
10
|
+
class AtomBody
|
11
|
+
include Murlsh::FeedBody
|
12
|
+
|
13
|
+
# Atom feed builder.
|
14
|
+
def build
|
15
|
+
if defined?(@body)
|
16
|
+
@body
|
17
|
+
else
|
18
|
+
feed = TinyAtom::Feed.new(config.fetch('root_url'), feed_title,
|
19
|
+
feed_url, :hubs => config.fetch('pubsubhubbub_hubs', []).
|
20
|
+
map { |x| x['subscribe_url'] })
|
21
|
+
|
22
|
+
urls.each do |mu|
|
23
|
+
Murlsh::Plugin.hooks('url_display_pre') do |p|
|
24
|
+
p.run mu, req, config
|
25
|
+
end
|
26
|
+
|
27
|
+
options = {
|
28
|
+
:author_name => mu.name,
|
29
|
+
:summary => mu.title_stripped
|
30
|
+
}
|
31
|
+
|
32
|
+
if EnclosureContentTypes.include?(mu.content_type)
|
33
|
+
options.merge!(
|
34
|
+
:enclosure_type => mu.content_type,
|
35
|
+
:enclosure_href => mu.url,
|
36
|
+
:enclosure_title => mu.title
|
37
|
+
)
|
38
|
+
if mu.content_length
|
39
|
+
options.merge! :enclosure_length => mu.content_length
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
if mu.thumbnail_url
|
44
|
+
begin
|
45
|
+
# Add root url to relative urls.
|
46
|
+
tu = URI(mu.thumbnail_url)
|
47
|
+
abs_url = if tu.is_a?(URI::HTTP)
|
48
|
+
tu
|
49
|
+
else
|
50
|
+
URI.join config.fetch('root_url'), tu
|
51
|
+
end
|
52
|
+
options.merge! :media_thumbnail_url => abs_url
|
53
|
+
rescue URI::InvalidURIError
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
Murlsh::failproof do
|
58
|
+
if mu.via
|
59
|
+
options.merge!(
|
60
|
+
:via_type => 'text/html',
|
61
|
+
:via_href => mu.via,
|
62
|
+
:via_title => URI(mu.via).extend(Murlsh::URIDomain).domain
|
63
|
+
)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
feed.add_entry mu.id, mu.title_stripped, mu.time, mu.url, options
|
68
|
+
end
|
69
|
+
|
70
|
+
@updated = feed.updated
|
71
|
+
@body = feed.make
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
|
78
|
+
end
|