cdn_fu 0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/MIT-LICENSE +20 -0
- data/README.md +20 -0
- data/bin/cdnfu +7 -0
- data/lib/cdn_fu/config.rb +138 -0
- data/lib/cdn_fu/config_error.rb +4 -0
- data/lib/cdn_fu/executor.rb +92 -0
- data/lib/cdn_fu/file_info.rb +14 -0
- data/lib/cdn_fu/lister.rb +47 -0
- data/lib/cdn_fu/minifier.rb +56 -0
- data/lib/cdn_fu/minifiers/yui_minifier.rb +35 -0
- data/lib/cdn_fu/uploader.rb +59 -0
- data/lib/cdn_fu/uploaders/cloudfront_uploader.rb +94 -0
- data/lib/cdn_fu/uploaders/local_uploader.rb +35 -0
- data/lib/cdn_fu.rb +35 -0
- data/rails/init.rb +4 -0
- data/tasks/cdn_fu.rake +82 -0
- metadata +95 -0
data/MIT-LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 Curtis W Spencer (@jubos)
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
## cdn fu
|
2
|
+
cdn fu is a fun domain specific language for deployment of assets to content
|
3
|
+
deliver network. There are 4 pluggable steps in the chain:
|
4
|
+
|
5
|
+
1. Preprocessing
|
6
|
+
2. Listing
|
7
|
+
3. Minification
|
8
|
+
4. Uploading
|
9
|
+
|
10
|
+
## Installation
|
11
|
+
|
12
|
+
ruby script/plugin install git://github.com/jubos/cdn_fu.git
|
13
|
+
|
14
|
+
|
15
|
+
## Example
|
16
|
+
For example, you can first run a Sass/Sprockets/AssetPacker during
|
17
|
+
preprocessing, then minify all of the resulting .js and .css assets, and
|
18
|
+
finally upload to a third party CDN.
|
19
|
+
|
20
|
+
|
data/bin/cdnfu
ADDED
@@ -0,0 +1,138 @@
|
|
1
|
+
module CdnFu
|
2
|
+
# This is just a storage class for the configuration for CdnFu
|
3
|
+
class Config
|
4
|
+
def self.configure(&block)
|
5
|
+
@@cfg ||= Config.new
|
6
|
+
@@cfg.instance_eval &block if block_given?
|
7
|
+
@@cfg
|
8
|
+
end
|
9
|
+
|
10
|
+
def self.clear
|
11
|
+
@@cfg = nil
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.config
|
15
|
+
@@cfg ||= Config.new
|
16
|
+
@@cfg
|
17
|
+
end
|
18
|
+
|
19
|
+
# Make a lister object
|
20
|
+
def initialize
|
21
|
+
@lister = Lister.new
|
22
|
+
end
|
23
|
+
|
24
|
+
# Accessors
|
25
|
+
def asset_id(*args)
|
26
|
+
return @asset_id if args.size == 0
|
27
|
+
@asset_id = args[0]
|
28
|
+
end
|
29
|
+
|
30
|
+
def verbose(*args)
|
31
|
+
return @verbose if args.size == 0
|
32
|
+
@verbose = args[0]
|
33
|
+
end
|
34
|
+
|
35
|
+
def asset_root_dir(*args)
|
36
|
+
return @asset_root_dir if args.size == 0
|
37
|
+
asset_root = args.first
|
38
|
+
# TODO add default separator so it works on windows
|
39
|
+
if asset_root[0,1] == '/'
|
40
|
+
@asset_root_dir = File.expand_path(asset_root)
|
41
|
+
else
|
42
|
+
@asset_root_dir = File.expand_path(File.join(Dir.pwd,asset_root))
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def tmp_dir(*args)
|
47
|
+
return @tmp_dir if args.size == 0
|
48
|
+
@tmp_dir = args[0]
|
49
|
+
end
|
50
|
+
|
51
|
+
def prepare_and_upload
|
52
|
+
@tmp_dir ||= "/tmp"
|
53
|
+
FileUtils.mkdir_p(@tmp_dir)
|
54
|
+
|
55
|
+
case @preprocessor
|
56
|
+
when Proc
|
57
|
+
@preprocessor.call
|
58
|
+
when Class
|
59
|
+
@preprocessor.preprocess
|
60
|
+
end
|
61
|
+
|
62
|
+
file_list = @lister.list
|
63
|
+
|
64
|
+
case @minifier
|
65
|
+
when Proc
|
66
|
+
@minifier.call(file_list)
|
67
|
+
when CdnFu::Minifier
|
68
|
+
@minifier.validate_and_minify(file_list)
|
69
|
+
end
|
70
|
+
|
71
|
+
case @uploader
|
72
|
+
when Proc
|
73
|
+
@uploader.call(file_list)
|
74
|
+
when CdnFu::Uploader
|
75
|
+
@uploader.validate_and_upload(file_list)
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
def preprocess
|
80
|
+
if block_given?
|
81
|
+
@preprocessor = proc
|
82
|
+
else
|
83
|
+
raise ConfigError,"No preprocess block given"
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def preprocessor(klass)
|
88
|
+
@preprocessor = klass
|
89
|
+
yield @preprocessor if block_given?
|
90
|
+
end
|
91
|
+
|
92
|
+
def files(&block)
|
93
|
+
if block_given?
|
94
|
+
@lister.instance_eval &block
|
95
|
+
else
|
96
|
+
@lister.list
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def minify(&block)
|
101
|
+
if block_given?
|
102
|
+
@minifier = block
|
103
|
+
else
|
104
|
+
raise ConfigError,"No minify block given"
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def minifier(*args,&block)
|
109
|
+
return @minifier if args.size == 0
|
110
|
+
minifier_class = args[0]
|
111
|
+
@minifier = minifier_class.new
|
112
|
+
@minifier.instance_eval &block if block_given?
|
113
|
+
end
|
114
|
+
|
115
|
+
def upload(&block)
|
116
|
+
if block_given?
|
117
|
+
@uploader = block
|
118
|
+
else
|
119
|
+
raise CdnFuConfigError,"No upload block given"
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def uploader(*args,&block)
|
124
|
+
return @uploader if args.size == 0
|
125
|
+
uploader_class = args[0]
|
126
|
+
@uploader = uploader_class.new
|
127
|
+
@uploader.instance_eval &block if block_given?
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
# Here we read in the config file
|
133
|
+
#cf_config_path = File.join(RAILS_ROOT,"config/cdn_fu.rb")
|
134
|
+
#if File.exists?(cf_config_path)
|
135
|
+
# require cf_config_path
|
136
|
+
#else
|
137
|
+
# puts "Please make a cdn_fu.rb file in RAILS_ROOT/config/ using rake cdn:init"
|
138
|
+
#end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module CdnFu
|
5
|
+
class CommandLine
|
6
|
+
def initialize(args)
|
7
|
+
@args = args
|
8
|
+
@options = {}
|
9
|
+
end
|
10
|
+
|
11
|
+
def parse!
|
12
|
+
begin
|
13
|
+
@opts = OptionParser.new do |opts|
|
14
|
+
opts.banner = 'Usage: cdnfu [options]'
|
15
|
+
opts.on('-c', '--config CONFIG' , 'a cdn fu configuration file') do |config|
|
16
|
+
@options[:config] = config
|
17
|
+
end
|
18
|
+
|
19
|
+
opts.on('--rails RAILS_DIR', "Install CDN Fu from the gem into a rails project") do |dir|
|
20
|
+
puts "Rails Install #{dir}"
|
21
|
+
@options[:rails] = dir
|
22
|
+
end
|
23
|
+
|
24
|
+
opts.on_tail('-?','-h','--help', 'Show this message') do
|
25
|
+
puts opts
|
26
|
+
exit
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
@opts.parse!(@args)
|
31
|
+
|
32
|
+
# Check for required arguments
|
33
|
+
if !@options[:config] && !@options[:rails]
|
34
|
+
puts @opts
|
35
|
+
exit
|
36
|
+
end
|
37
|
+
|
38
|
+
if config_file = @options[:config]
|
39
|
+
if File.exists?(config_file)
|
40
|
+
Dir.chdir(File.dirname(config_file))
|
41
|
+
eval(File.open(config_file).read)
|
42
|
+
Config.config.prepare_and_upload
|
43
|
+
else
|
44
|
+
puts "Cannot find config file"
|
45
|
+
exit
|
46
|
+
end
|
47
|
+
elsif rails_dir = @options[:rails]
|
48
|
+
plugins_dir = File.join(rails_dir,"vendor","plugins")
|
49
|
+
unless File.exists?(plugins_dir)
|
50
|
+
puts "#{plugins_dir} doesn't exist. Are you sure this is a rails project?"
|
51
|
+
exit
|
52
|
+
end
|
53
|
+
|
54
|
+
cdn_fu_dir = File.join(plugins_dir,'cdn_fu')
|
55
|
+
|
56
|
+
if File.exists?(cdn_fu_dir)
|
57
|
+
print "Directory #{cdn_fu_dir} already exists, overwrite [y/N]?"
|
58
|
+
exit if gets !~ /y/i
|
59
|
+
FileUtils.rm_rf(cdn_fu_dir)
|
60
|
+
end
|
61
|
+
|
62
|
+
tasks_dir = File.join(cdn_fu_dir,'tasks')
|
63
|
+
|
64
|
+
begin
|
65
|
+
Dir.mkdir(cdn_fu_dir)
|
66
|
+
Dir.mkdir(tasks_dir)
|
67
|
+
rescue SystemCallError
|
68
|
+
puts "Cannot create #{cdn_fu_dir}"
|
69
|
+
exit
|
70
|
+
end
|
71
|
+
|
72
|
+
File.open(File.join(cdn_fu_dir,'init.rb'), 'w') do |file|
|
73
|
+
file << File.read(File.dirname(__FILE__) + "/../../rails/init.rb")
|
74
|
+
end
|
75
|
+
|
76
|
+
File.open(File.join(tasks_dir,'cdn_fu.rake'),'w') do |file|
|
77
|
+
file << File.read(File.dirname(__FILE__) + "/../../tasks/cdn_fu.rake")
|
78
|
+
end
|
79
|
+
|
80
|
+
puts "CDN Fu plugin added to #{rails_dir}"
|
81
|
+
exit
|
82
|
+
end
|
83
|
+
rescue RuntimeError => e
|
84
|
+
puts e
|
85
|
+
e.backtrace.each do |line|
|
86
|
+
puts line
|
87
|
+
end
|
88
|
+
exit
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module CdnFu
|
2
|
+
class Lister
|
3
|
+
def initialize
|
4
|
+
@globs = []
|
5
|
+
@files = []
|
6
|
+
end
|
7
|
+
def glob(name,options = {})
|
8
|
+
options[:name] = name
|
9
|
+
@globs << options
|
10
|
+
end
|
11
|
+
|
12
|
+
def file(name,options = {})
|
13
|
+
options[:name] = name
|
14
|
+
@files << options
|
15
|
+
end
|
16
|
+
|
17
|
+
def list
|
18
|
+
file_infos ||= []
|
19
|
+
asset_root = Config.config.asset_root_dir
|
20
|
+
asset_root ||= Dir.pwd
|
21
|
+
|
22
|
+
@globs.each do |glob|
|
23
|
+
glob_str = glob[:name]
|
24
|
+
Dir.glob(File.join(asset_root,glob_str)).each do |file|
|
25
|
+
fi = FileInfo.new
|
26
|
+
fi.local_path = File.expand_path(file)
|
27
|
+
fi.gzip = glob[:gzip]
|
28
|
+
fi.minify = glob[:minify]
|
29
|
+
root_sub_path = fi.local_path.gsub(asset_root,'')
|
30
|
+
glob_sub_path = root_sub_path.gsub(glob_str[0,glob_str.index('*')],'')
|
31
|
+
fi.remote_path = glob[:path] ? File.join(glob[:path],glob_sub_path) : root_sub_path
|
32
|
+
file_infos << fi
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
@files.each do |file|
|
37
|
+
fi = FileInfo.new
|
38
|
+
fi.local_path = File.join(asset_root,file[:name])
|
39
|
+
fi.gzip = file[:gzip]
|
40
|
+
fi.minify = file[:minify]
|
41
|
+
fi.remote_path = file[:path] ? file[:path] : fi.local_path.gsub(Config.config.asset_root_dir,'')
|
42
|
+
file_infos << fi
|
43
|
+
end
|
44
|
+
file_infos
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module CdnFu
|
2
|
+
class Minifier
|
3
|
+
class << self
|
4
|
+
|
5
|
+
def optional_attributes(*arr)
|
6
|
+
return @optional_attributes if arr.size == 0
|
7
|
+
@optional_attributes ||= []
|
8
|
+
arr.each do |a|
|
9
|
+
@optional_attributes << a.to_sym
|
10
|
+
class_eval do
|
11
|
+
define_method(a.to_sym) do |*args|
|
12
|
+
return instance_variable_get("@#{a}") if args.size == 0
|
13
|
+
instance_variable_set("@#{a}",args[0])
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
alias :optional_attribute :optional_attributes
|
19
|
+
|
20
|
+
def required_attributes(*arr)
|
21
|
+
return @required_attributes if arr.size == 0
|
22
|
+
@required_attributes ||= []
|
23
|
+
arr.each do |a|
|
24
|
+
@required_attributes << a.to_sym
|
25
|
+
class_eval do
|
26
|
+
define_method(a.to_sym) do |*args|
|
27
|
+
return instance_variable_get("@#{a}") if args.size == 0
|
28
|
+
instance_variable_set("@#{a}",args[0])
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
alias :required_attribute :required_attributes
|
34
|
+
end
|
35
|
+
|
36
|
+
def validate_and_minify(file_list)
|
37
|
+
attribute_validate
|
38
|
+
validate
|
39
|
+
minify(file_list)
|
40
|
+
end
|
41
|
+
|
42
|
+
def attribute_validate
|
43
|
+
if @required_attributes
|
44
|
+
@required_attributes.each do |attr|
|
45
|
+
res = self.send(attr)
|
46
|
+
if res.nil? or res == ''
|
47
|
+
raise CdnFuConfigError, "#{attr} must be specified in the configuration"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def validate
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
class YuiMinifier < CdnFu::Minifier
|
2
|
+
required_attribute :yui_jar_path
|
3
|
+
|
4
|
+
# Essentially iterate through through the files and if minify is specified call it.
|
5
|
+
def minify(file_list)
|
6
|
+
file_list.each do |cf_file|
|
7
|
+
one_minification(cf_file) if cf_file.minify?
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
# Custom validate method ensures that the jar specified will actually work.
|
12
|
+
def validate
|
13
|
+
if @yui_jar_path
|
14
|
+
if File.exists?(@yui_jar_path)
|
15
|
+
output = `java -jar #{@yui_jar_path}`
|
16
|
+
if output !~ /java -jar yuicompressor-x\.y\.z\.jar/
|
17
|
+
raise CdnFu::ConfigError,"Invalid YUI Compressor jar specified in cdn_fu.rb"
|
18
|
+
end
|
19
|
+
else
|
20
|
+
raise CdnFu::ConfigError,"YUI Compressor jar specified in cdn_fu.rb does not exist"
|
21
|
+
end
|
22
|
+
else
|
23
|
+
raise CdnFu::ConfigError,"You must specify a yui_jar_path for YUI Compressor"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def one_minification(file)
|
30
|
+
modified_path = File.join(CdnFu::Config.config.tmp_dir,"minified_#{File.basename(file.local_path)}")
|
31
|
+
`java -jar #{@yui_jar_path} #{file.local_path} > #{modified_path}`
|
32
|
+
puts "[minify] #{file.local_path}" if CdnFu::Config.config.verbose
|
33
|
+
file.minified_path = modified_path
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module CdnFu
|
2
|
+
class Uploader
|
3
|
+
# These are class level methods, so that subclasses can do some light
|
4
|
+
# metaprogramming to specify optional and required attributes
|
5
|
+
class << self;
|
6
|
+
def required_attributes( *arr )
|
7
|
+
return @required_attributes if arr.size == 0
|
8
|
+
@required_attributes ||= []
|
9
|
+
arr.each do |a|
|
10
|
+
@required_attributes << a.to_sym
|
11
|
+
class_eval do
|
12
|
+
define_method(a.to_sym) do |*args|
|
13
|
+
return instance_variable_get("@#{a}") if args.size == 0
|
14
|
+
instance_variable_set("@#{a}",args[0])
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def optional_attributes( *arr )
|
21
|
+
return @optional_attributes if arr.size == 0
|
22
|
+
@optional_attributes ||= []
|
23
|
+
arr.each do |a|
|
24
|
+
@optional_attributes << a.to_sym
|
25
|
+
class_eval do
|
26
|
+
define_method(a.to_sym) do |*args|
|
27
|
+
return instance_variable_get("@#{a}") if args.size == 0
|
28
|
+
instance_variable_set("@#{a}",args[0])
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# aliases just for syntactic sugar
|
35
|
+
alias :required_attribute :required_attributes
|
36
|
+
alias :optional_attribute :optional_attributes
|
37
|
+
end
|
38
|
+
|
39
|
+
def validate_and_upload(file_list)
|
40
|
+
attribute_validate
|
41
|
+
validate
|
42
|
+
upload(file_list)
|
43
|
+
end
|
44
|
+
|
45
|
+
def attribute_validate
|
46
|
+
if self.class.required_attributes
|
47
|
+
self.class.required_attributes.each do |attr|
|
48
|
+
res = self.send(attr)
|
49
|
+
if res.nil? or res == ''
|
50
|
+
raise CdnFuConfigError, "#{attr} must be specified in the uploader configuration"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def validate
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# This uploader using aws:s3 gem to upload everything to the specified bucket
|
2
|
+
require 'zlib'
|
3
|
+
require 'aws/s3'
|
4
|
+
class CloudfrontUploader < CdnFu::Uploader
|
5
|
+
include AWS::S3
|
6
|
+
required_attribute :s3_bucket
|
7
|
+
optional_attributes :s3_access_key, :s3_secret_key
|
8
|
+
|
9
|
+
MAX_KEYS = 1000
|
10
|
+
|
11
|
+
def upload(file_list)
|
12
|
+
access_key = @s3_access_key ? @s3_access_key : ENV["CDN_FU_AMAZON_ACCESS_KEY"]
|
13
|
+
secret_key = @s3_secret_key ? @s3_secret_key : ENV["CDN_FU_AMAZON_SECRET_KEY"]
|
14
|
+
|
15
|
+
AWS::S3::Base.establish_connection!(
|
16
|
+
:access_key_id => access_key,
|
17
|
+
:secret_access_key => secret_key
|
18
|
+
)
|
19
|
+
|
20
|
+
checksums = populate_existing_asset_checksums
|
21
|
+
file_list.each do |file|
|
22
|
+
upload_single_file(file)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# This uploader requires a valid asset_id at the top level configuration
|
27
|
+
# because cloudfront only invalidates its files every 24 hours, so you need
|
28
|
+
# to have an incrementing id to avoid stale assets
|
29
|
+
def validate
|
30
|
+
if !CdnFu::Config.config.asset_id
|
31
|
+
raise CdnFu::ConfigError, "You must specify an asset_id at the top level"
|
32
|
+
end
|
33
|
+
access_key = @s3_access_key ? @s3_access_key : ENV["CDN_FU_AMAZON_ACCESS_KEY"]
|
34
|
+
secret_key = @s3_secret_key ? @s3_secret_key : ENV["CDN_FU_AMAZON_SECRET_KEY"]
|
35
|
+
|
36
|
+
if !access_key or !secret_key
|
37
|
+
raise CdnFu::ConfigError, "Please specify s3_access_key and s3_secret_key attributes or use the environnment variables: CDN_FU_AMAZON_ACCESS_KEY and CDN_FU_AMAZON_SECRET_KEY"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
def populate_existing_asset_checksums
|
43
|
+
@existing_asset_checksums = {}
|
44
|
+
objects = []
|
45
|
+
bucket = Bucket.find(@s3_bucket)
|
46
|
+
objects += bucket.objects(:max_keys => MAX_KEYS)
|
47
|
+
if objects.size == MAX_KEYS
|
48
|
+
loop do
|
49
|
+
new_objects = bucket.objects(:max_keys => MAX_KEYS,:marker => objects.last)
|
50
|
+
objects += new_objects
|
51
|
+
break if new_objects.size < MAX_KEYS
|
52
|
+
end
|
53
|
+
end
|
54
|
+
objects.each do |obj|
|
55
|
+
@existing_asset_checksums[obj.path] = obj.metadata['x-amz-meta-sha1-hash']
|
56
|
+
end
|
57
|
+
return objects
|
58
|
+
end
|
59
|
+
|
60
|
+
def upload_single_file(cf_file)
|
61
|
+
versioned_filename = CdnFu::Config.config.asset_id + cf_file.remote_path
|
62
|
+
options = {}
|
63
|
+
options[:access] = :public_read
|
64
|
+
path_to_upload = cf_file.minified_path
|
65
|
+
path_to_upload ||= cf_file.local_path
|
66
|
+
fstat = File.stat(path_to_upload)
|
67
|
+
sha1sum = Digest::SHA1.hexdigest(File.open(path_to_upload).read)
|
68
|
+
if remote_sha1 = @existing_asset_checksums["/" + s3_bucket + "/" + versioned_filename]
|
69
|
+
if remote_sha1 != sha1sum
|
70
|
+
puts "Your assets are different from the ones in s3 with this asset_id. Please increment your asset_id in cdn_fu.rb"
|
71
|
+
exit
|
72
|
+
end
|
73
|
+
else
|
74
|
+
options[:access] = :public_read
|
75
|
+
options["x-amz-meta-sha1_hash"] = sha1sum
|
76
|
+
options["x-amz-meta-mtime"] = fstat.mtime.getutc.to_i
|
77
|
+
options["x-amz-meta-size"] = fstat.size
|
78
|
+
file_content =open(path_to_upload).read
|
79
|
+
if cf_file.gzip?
|
80
|
+
options["Content-Encoding"] = 'gzip'
|
81
|
+
strio = StringIO.open('', 'w')
|
82
|
+
gz = Zlib::GzipWriter.new(strio)
|
83
|
+
gz.write(file_content)
|
84
|
+
gz.close
|
85
|
+
file_content = strio.string
|
86
|
+
end
|
87
|
+
|
88
|
+
options[:cache_control] = "max-age=#{8.years.to_i}"
|
89
|
+
options[:expires] = 8.years.from_now.httpdate
|
90
|
+
S3Object.store(versioned_filename,file_content,s3_bucket, options)
|
91
|
+
puts "[upload] #{s3_bucket} #{versioned_filename}" if CdnFu::Config.config.verbose
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'pathname'
|
3
|
+
class LocalUploader < CdnFu::Uploader
|
4
|
+
required_attribute :path
|
5
|
+
|
6
|
+
def validate
|
7
|
+
path = @path
|
8
|
+
if !@path
|
9
|
+
raise CdnFu::ConfigError, "Please specify a path for LocalUploader"
|
10
|
+
end
|
11
|
+
|
12
|
+
@path_obj = Pathname.new(@path)
|
13
|
+
if !@path_obj.absolute?
|
14
|
+
raise CdnFu::ConfigError, "Please specify an absolute path"
|
15
|
+
end
|
16
|
+
|
17
|
+
if File.exists?(@path) and !FileTest.directory?(@path)
|
18
|
+
raise CdnFu::ConfigError, "LocalUploader path must be a directory"
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
# Here we just iterate through the files
|
24
|
+
def upload(file_list)
|
25
|
+
FileUtils.mkdir_p(@path) if !File.exists?(@path)
|
26
|
+
file_list.each do |file|
|
27
|
+
path_to_copy = file.minified_path
|
28
|
+
path_to_copy ||= file.local_path
|
29
|
+
destination = File.join(@path,file.remote_path)
|
30
|
+
dest_dir = File.dirname(destination)
|
31
|
+
FileUtils.mkdir_p(dest_dir) if !File.exists?(dest_dir)
|
32
|
+
FileUtils.cp path_to_copy,dest_dir
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/cdn_fu.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
dir = File.dirname(__FILE__)
|
2
|
+
$LOAD_PATH.unshift dir unless $LOAD_PATH.include?(dir)
|
3
|
+
|
4
|
+
module CdnFu
|
5
|
+
VERSION = 0.5
|
6
|
+
# Do some sensible defaults for rails
|
7
|
+
def self.init_rails(binding)
|
8
|
+
cfg = CdnFu::Config.config
|
9
|
+
cfg.asset_root_dir(File.join(RAILS_ROOT,'public'))
|
10
|
+
cfg.tmp_dir(File.join(RAILS_ROOT,'tmp'))
|
11
|
+
# If there is a RAILS_ASSET_ID use that.
|
12
|
+
begin
|
13
|
+
cfg.asset_id(RAILS_ASSET_ID)
|
14
|
+
rescue NameError
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.load_rails_config
|
19
|
+
cdn_fu_config_file = File.join(RAILS_ROOT,'config','cdn_fu.rb')
|
20
|
+
if File.exists?(cdn_fu_config_file)
|
21
|
+
eval(File.open(cdn_fu_config_file).read)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
require 'cdn_fu/config_error'
|
27
|
+
require 'cdn_fu/config'
|
28
|
+
require 'cdn_fu/executor'
|
29
|
+
require 'cdn_fu/file_info.rb'
|
30
|
+
require 'cdn_fu/lister.rb'
|
31
|
+
require 'cdn_fu/minifier.rb'
|
32
|
+
require 'cdn_fu/uploader.rb'
|
33
|
+
["minifiers","listers","uploaders"].each do |subdir|
|
34
|
+
Dir[File.join(File.dirname(__FILE__),'cdn_fu',subdir, "*")].each {|f| require f}
|
35
|
+
end
|
data/rails/init.rb
ADDED
data/tasks/cdn_fu.rake
ADDED
@@ -0,0 +1,82 @@
|
|
1
|
+
namespace "cdn" do
|
2
|
+
desc "Initialize a cdn_fu.rb"
|
3
|
+
task "init" => :environment do
|
4
|
+
cfg_path = File.join(RAILS_ROOT,'config/cdn_fu.rb')
|
5
|
+
if FileTest.exists?(cfg_path)
|
6
|
+
puts "config/cdn_fu.rb already exists."
|
7
|
+
print "Do you want to overwrite? [y/n] "
|
8
|
+
if STDIN.gets !~ /^[yY]/
|
9
|
+
exit
|
10
|
+
end
|
11
|
+
end
|
12
|
+
File.open(cfg_path,'w') do |f|
|
13
|
+
f << <<-EOF
|
14
|
+
CdnFu::Config.configure do
|
15
|
+
# asset_id
|
16
|
+
# default: RAILS_ASSET_ID
|
17
|
+
# The asset id that you need to increment whenever your assets change (for
|
18
|
+
# cloudfront) This ensures that cloudfront assets are properly invalidated,
|
19
|
+
# since there is no way to explicitly invalidate entries. Set it to nil if
|
20
|
+
# you don't care about stale assets.
|
21
|
+
#
|
22
|
+
# Examples:
|
23
|
+
# asset_id nil
|
24
|
+
# asset_id '123'
|
25
|
+
# asset_id File.open(File.join(RAILS_ROOT,'revision')) {|f| f.read.strip }
|
26
|
+
# asset_id RAILS_ASSET_ID
|
27
|
+
|
28
|
+
# asset_root_dir
|
29
|
+
# default: RAILS_ROOT + 'public/'
|
30
|
+
# This is where your files specifications will start looking for assets.
|
31
|
+
|
32
|
+
# You can specify things to do before the minification step here. For
|
33
|
+
# example if you use Sass you will want to ensure you have the latest
|
34
|
+
# stylesheets processed
|
35
|
+
#
|
36
|
+
#preprocess do
|
37
|
+
# Sass::Plugin.update_stylesheets
|
38
|
+
#end
|
39
|
+
|
40
|
+
files do
|
41
|
+
glob "javascripts/*.js", :minify => true, :gzip => true
|
42
|
+
glob "stylesheets/*.css", :minify => true, :gzip => true
|
43
|
+
glob "images/**/*.*"
|
44
|
+
end
|
45
|
+
|
46
|
+
# minifier
|
47
|
+
# default: nil
|
48
|
+
# YuiMinifier is the only included backend for now. This will iterate over
|
49
|
+
# each js/css asset and do a minification.
|
50
|
+
# minifier YuiMinifier do
|
51
|
+
# yui_jar_path "/usr/bin/yuicompressor-2.4.2.jar"
|
52
|
+
#end
|
53
|
+
|
54
|
+
# uploader
|
55
|
+
# default: nil
|
56
|
+
#
|
57
|
+
# Cloudfront is the only supported backend now. You can specify access
|
58
|
+
# credentials in this file or for more safety, put them in your environment
|
59
|
+
# variables CDN_FU_AMAZON_ACCESS_KEY and CDN_FU_AMAZON_SECRET_KEY environment
|
60
|
+
# variables
|
61
|
+
#uploader CloudfrontUploader do
|
62
|
+
# s3_bucket 'mybucket'
|
63
|
+
# s3_access_key 'blah'
|
64
|
+
# s3_secret_key 'blah'
|
65
|
+
#end
|
66
|
+
end
|
67
|
+
EOF
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
desc "Uploads all your assets to your cdn"
|
72
|
+
task "upload" => :environment do
|
73
|
+
cfg_path = File.join(RAILS_ROOT,'config/cdn_fu.rb')
|
74
|
+
if !FileTest.exists?(cfg_path)
|
75
|
+
puts "#{cfg_path} doesn't exist. Please run rake cdn:init to create a basic configuration file"
|
76
|
+
exit
|
77
|
+
end
|
78
|
+
CdnFu.load_rails_config
|
79
|
+
CdnFu::Config.config.prepare_and_upload
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
metadata
ADDED
@@ -0,0 +1,95 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: cdn_fu
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: "0.5"
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Curtis Spencer
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
|
12
|
+
date: 2009-11-04 00:00:00 -08:00
|
13
|
+
default_executable:
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: rake
|
17
|
+
type: :runtime
|
18
|
+
version_requirement:
|
19
|
+
version_requirements: !ruby/object:Gem::Requirement
|
20
|
+
requirements:
|
21
|
+
- - ">="
|
22
|
+
- !ruby/object:Gem::Version
|
23
|
+
version: "0.8"
|
24
|
+
version:
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: aws-s3
|
27
|
+
type: :runtime
|
28
|
+
version_requirement:
|
29
|
+
version_requirements: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 0.6.2
|
34
|
+
version:
|
35
|
+
description: |
|
36
|
+
CDN Fu is a framework for making listing, minification and deployment of static
|
37
|
+
assets easy. It allows you to use it standalone on the command line for non
|
38
|
+
Rails project and it can be used as a Rails plugin to add useful rake tasks and
|
39
|
+
sensible defaults.
|
40
|
+
|
41
|
+
email: curtis@sevenforge.com
|
42
|
+
executables:
|
43
|
+
- cdnfu
|
44
|
+
extensions: []
|
45
|
+
|
46
|
+
extra_rdoc_files: []
|
47
|
+
|
48
|
+
files:
|
49
|
+
- README.md
|
50
|
+
- MIT-LICENSE
|
51
|
+
- bin/cdnfu
|
52
|
+
- lib/cdn_fu/config.rb
|
53
|
+
- lib/cdn_fu/config_error.rb
|
54
|
+
- lib/cdn_fu/executor.rb
|
55
|
+
- lib/cdn_fu/file_info.rb
|
56
|
+
- lib/cdn_fu/lister.rb
|
57
|
+
- lib/cdn_fu/minifier.rb
|
58
|
+
- lib/cdn_fu/minifiers/yui_minifier.rb
|
59
|
+
- lib/cdn_fu/uploader.rb
|
60
|
+
- lib/cdn_fu/uploaders/cloudfront_uploader.rb
|
61
|
+
- lib/cdn_fu/uploaders/local_uploader.rb
|
62
|
+
- lib/cdn_fu.rb
|
63
|
+
- rails/init.rb
|
64
|
+
- tasks/cdn_fu.rake
|
65
|
+
has_rdoc: true
|
66
|
+
homepage: http://www.sevenforge.com/cdn_fu
|
67
|
+
licenses: []
|
68
|
+
|
69
|
+
post_install_message:
|
70
|
+
rdoc_options:
|
71
|
+
- --inline-source
|
72
|
+
- --charset=UTF-8
|
73
|
+
require_paths:
|
74
|
+
- lib
|
75
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: "0"
|
80
|
+
version:
|
81
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ">="
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: "0"
|
86
|
+
version:
|
87
|
+
requirements: []
|
88
|
+
|
89
|
+
rubyforge_project: cdn_fu
|
90
|
+
rubygems_version: 1.3.5
|
91
|
+
signing_key:
|
92
|
+
specification_version: 3
|
93
|
+
summary: CDN Fu is a framework for making minification and deployment of static assets easy
|
94
|
+
test_files: []
|
95
|
+
|