rake-pipeline 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/.gitignore ADDED
@@ -0,0 +1,18 @@
1
+ *.gem
2
+ *.rbx
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ devbin
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ -cfs -r spec_helper.rb
data/.yardopts ADDED
@@ -0,0 +1,2 @@
1
+ --readme README.yard
2
+
data/Gemfile ADDED
@@ -0,0 +1,10 @@
1
+ source 'http://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in rake-pipeline.gemspec
4
+ gemspec
5
+ gem "flay"
6
+ gem "flog"
7
+
8
+ gem "simplecov", :require => false
9
+ gem "yard"
10
+ gem "rdiscount"
data/LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (C) 2011 by LivingSocial, Inc.
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ of this software and associated documentation files (the "Software"), to deal
5
+ in the Software without restriction, including without limitation the rights
6
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ copies of the Software, and to permit persons to whom the Software is
8
+ furnished to do so, subject to the following conditions:
9
+
10
+ The above copyright notice and this permission notice shall be included in
11
+ all copies or substantial portions of the Software.
12
+
13
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19
+ THE SOFTWARE.
20
+
data/README.markdown ADDED
@@ -0,0 +1,4 @@
1
+ # Rake::Pipeline
2
+
3
+ The canonical documentation for Rake::Pipeline is hosted at
4
+ <a href="http://rubydoc.info/github/livingsocial/rake-pipeline/master/file/README.yard">rubydoc.info</a>
data/README.yard ADDED
@@ -0,0 +1,149 @@
1
+ = Rake::Pipeline
2
+
3
+ Rake::Pipeline is a system for packaging assets for deployment to the
4
+ web. It uses Rake under the hood for dependency management and updating
5
+ output files based on input changes.
6
+
7
+ = Usage
8
+
9
+ The easiest way to use Rake::Pipeline is via a +Assetfile+ file in the
10
+ root of your project.
11
+
12
+ A sample +Assetfile+ looks like this:
13
+
14
+ !!!ruby
15
+ input "assets"
16
+ output "public"
17
+
18
+ # this block will take all JS inputs, wrap them in a closure,
19
+ # add some additional metadata, and concatenate them all into
20
+ # application.scripts.js.
21
+ match "*.js" do
22
+ filter ClosureWrapper
23
+ filter DataWrapper
24
+ filter Rake::Pipeline::ConcatFilter, "application.scripts.js"
25
+ end
26
+
27
+ # this block will take all HTML and CSS inputs, convert them
28
+ # into JavaScript
29
+ match "*/*.{html,css}" do
30
+ filter DataWrapper
31
+ filter Rake::Pipeline::ConcatFilter, "application.assets.js"
32
+ end
33
+
34
+ match "*.js" do
35
+ filter Rake::Pipeline::ConcatFilter, "application.js"
36
+ end
37
+
38
+ # copy any unprocessed files over to the output directory
39
+ filter Rake::Pipeline::ConcatFilter
40
+
41
+ The available options are:
42
+
43
+ * {Rake::Pipeline::DSL#input input}: the directory containing your input files
44
+ * {Rake::Pipeline::DSL#output output}: the directory to place your output files
45
+ like to process
46
+ * if you do not specify a block, the files will be
47
+ copied over directly.
48
+
49
+ = Filters
50
+
51
+ A filter is a simple class that inherits from
52
+ {Rake::Pipeline::Filter}. A filter must implement a single
53
+ method, called +generate_output+, which takes
54
+ two parameters: a list of input files and the output file.
55
+
56
+ Both the input and output files are {Rake::Pipeline::FileWrapper} objects.
57
+ The most important methods on a {Rake::Pipeline::FileWrapper FileWrapper} are:
58
+
59
+ * {Rake::Pipeline::FileWrapper#path path}: the path of the file, relative to its input root
60
+ * {Rake::Pipeline::FileWrapper#read read}: read the contents of the file
61
+ * {Rake::Pipeline::FileWrapper#write write(string)}: write a String to the file
62
+
63
+ For example, a simple concatenation filter would look like:
64
+
65
+ !!!ruby
66
+ class ConcatFilter < Rake::Pipeline::Filter
67
+ def generate_output(inputs, output)
68
+ inputs.each do |input|
69
+ output.write input.read
70
+ end
71
+ end
72
+ end
73
+
74
+ If you had a series of input files like:
75
+
76
+ * +app/javascripts/one.js+
77
+ * +app/javascripts/two.js+
78
+ * +app/javascripts/three.js+
79
+
80
+ and you specified the +ConcatFilter+ in your
81
+ +AssetFile+ like:
82
+
83
+ !!!ruby
84
+ filter ConcatFilter, "application.js"
85
+
86
+ The filter would receive a single call to
87
+ +generate_output+ with an Array of {Rake::Pipeline::FileWrapper FileWrapper}s
88
+ representing each of the three files, and a {Rake::Pipeline::FileWrapper FileWrapper}
89
+ representing +application.js+.
90
+
91
+ == Binary Data
92
+
93
+ If your filter is operating on binary data, like images,
94
+ rather than textual data, like source code, you can specify
95
+ that in your filter:
96
+
97
+ !!!ruby
98
+ class ConcatFilter < Rake::Pipeline::Filter
99
+ processes_binary_files
100
+
101
+ def generate_output(inputs, output)
102
+ inputs.each do |input|
103
+ output.write input.read
104
+ end
105
+ end
106
+ end
107
+
108
+ This will stop `Rake::Pipeline` from trying to interpret the
109
+ input files as `UTF-8`, which obviously will not work on
110
+ binary data.
111
+
112
+ = Built-In Filters
113
+
114
+ At the current time, +Rake::Pipeline+ comes with a single built-in
115
+ filter: {Rake::Pipeline::ConcatFilter}. Its implementation is
116
+ the same as the `ConcatFilter` shown above.
117
+
118
+ = Preview Server
119
+
120
+ To start up the preview server, run +rakep+. This will start up
121
+ a server that automatically recompiles files for you on the fly
122
+ and serves up the files you need.
123
+
124
+ This should allow you to have a single index.html file pointing
125
+ at the same files in both development and production.
126
+
127
+ = Compiling Assets
128
+
129
+ To compile all assets before deployment, simply run:
130
+
131
+ $ rakep build
132
+
133
+ = Encodings
134
+
135
+ If a filter does not specify that it processes binary files,
136
+ +Rake::Pipeline+ will open all inputs and outputs as +UTF-8+.
137
+
138
+ This means that if you have files encoded in other encodings,
139
+ like +Latin-1+, +Rake::Pipeline+ will raise an exception. In
140
+ this situation, you need to open the offending file in your
141
+ text editor and re-save it as +UTF-8+.
142
+
143
+ = Public Release Requirement
144
+
145
+ Before publicly releasing this code, we need to properly support
146
+ encodings other than UTF-8. That means using the
147
+ +default_external+ instead of hardcoding to UTF-8 and
148
+ providing a mechanism for specifying the encoding of a file using
149
+ a magic comment.
data/Rakefile ADDED
@@ -0,0 +1,12 @@
1
+ #!/usr/bin/env rake
2
+ require "bundler/gem_tasks"
3
+
4
+ directory "doc"
5
+
6
+ task :docs => Dir["lib/**"] do
7
+ sh "devbin/yard doc --readme README.yard --hide-void-return"
8
+ end
9
+
10
+ task :graph => ["doc", :docs] do
11
+ sh "devbin/yard graph --dependencies | dot -Tpng -o doc/arch.png"
12
+ end
data/bin/rakep ADDED
@@ -0,0 +1,27 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "rake-pipeline"
4
+ require "rake-pipeline/middleware"
5
+ require "rack/server"
6
+
7
+ module Rake
8
+ class Pipeline
9
+ class Server < Rack::Server
10
+ def app
11
+ not_found = proc { [404, { "Content-Type" => "text/plain" }, ["not found"]] }
12
+ config = "Assetfile"
13
+
14
+ Middleware.new(not_found, config)
15
+ end
16
+ end
17
+ end
18
+ end
19
+
20
+ if ARGV[0] == "build"
21
+ config = "Assetfile"
22
+ pipeline_source = File.read(config)
23
+ pipeline = Rake::Pipeline.class_eval "build do\n#{pipeline_source}\nend", config, 1
24
+ pipeline.invoke
25
+ else
26
+ Rake::Pipeline::Server.new.start
27
+ end
@@ -0,0 +1,365 @@
1
+ require "rake-pipeline/file_wrapper"
2
+ require "rake-pipeline/filter"
3
+ require "rake-pipeline/filters"
4
+ require "rake-pipeline/dsl"
5
+ require "rake-pipeline/matcher"
6
+ require "rake-pipeline/error"
7
+
8
+ if defined?(Rails::Railtie)
9
+ require "rake-pipeline/railtie"
10
+ elsif defined?(Rails)
11
+ require "rake-pipeline/rails_plugin"
12
+ end
13
+
14
+ require "thread"
15
+
16
+ # Use the Rake namespace
17
+ module Rake
18
+ # Override Rake::Task to support recursively re-enabling
19
+ # a task and its dependencies.
20
+ class Task
21
+
22
+ # @param [Rake::Application] app a Rake Application
23
+ # @return [void]
24
+ def recursively_reenable(app)
25
+ reenable
26
+
27
+ prerequisites.each do |dep|
28
+ app[dep].recursively_reenable(app)
29
+ end
30
+ end
31
+ end
32
+
33
+ # Override Rake::FileTask to make it sortable
34
+ class FileTask
35
+ # implement Ruby protocol for sorting
36
+ #
37
+ # @return [Fixnum]
38
+ def <=>(other)
39
+ [name, prerequisites] <=> [other.name, other.prerequisites]
40
+ end
41
+ end
42
+
43
+ # A Pipeline is responsible for taking a directory of input
44
+ # files, applying a number of filters to the inputs, and
45
+ # outputting them into an output directory.
46
+ #
47
+ # The normal way to build and configure a pipeline is by
48
+ # using {.build}. Inside the block passed to {.build}, all
49
+ # methods of {DSL} are available.
50
+ #
51
+ # @see DSL Rake::Pipeline::DSL for information on the methods
52
+ # available inside the block.
53
+ #
54
+ # @example
55
+ # !!!ruby
56
+ # Rake::Pipeline.build do
57
+ # # process all js, css and html files in app/assets
58
+ # input "app/assets", "**/*.{js,coffee,css,scss,html}"
59
+ #
60
+ # # processed files should be outputted to public
61
+ # output "public"
62
+ #
63
+ # # process all coffee files
64
+ # match "*.coffee" do
65
+ # # compile all CoffeeScript files. the output file
66
+ # # for the compilation should be the input name
67
+ # # with the .coffee extension replaced with .js
68
+ # filter(CoffeeCompiler) do |input|
69
+ # input.sub(/\.coffee$/, '.js')
70
+ # end
71
+ # end
72
+ #
73
+ # # specify filters for js files. this includes the
74
+ # # output of the previous step, which converted
75
+ # # coffee files to js files
76
+ # match "*.js" do
77
+ # # first, wrap all JS files in a custom filter
78
+ # filter ClosureFilter
79
+ # # then, concatenate all JS files into a single file
80
+ # filter Rake::Pipeline::ConcatFilter, "application.js"
81
+ # end
82
+ #
83
+ # # specify filters for css and scss files
84
+ # match "*.{css,scss}" do
85
+ # # compile CSS and SCSS files using the SCSS
86
+ # # compiler. if an input file has the extension
87
+ # # scss, replace it with css
88
+ # filter(ScssCompiler) do |input|
89
+ # input.sub(/\.scss$/, 'css')
90
+ # end
91
+ # # then, concatenate all CSS files into a single file
92
+ # filter Rake::Pipeline::ConcatFilter, "application.css"
93
+ # end
94
+ #
95
+ # # the remaining files not specified by a matcher (the
96
+ # # HTML files) are simply copied over.
97
+ #
98
+ # # you can also specify filters here that will apply to
99
+ # # all processed files (application.js and application.css)
100
+ # # up until this point, as well as the HTML files.
101
+ # end
102
+ class Pipeline
103
+ # @return [String] a glob representing the input files
104
+ attr_accessor :input_glob
105
+
106
+ # @return [String] the directory path for the input files.
107
+ attr_reader :input_root
108
+
109
+ # @return [String] the directory path for the output files.
110
+ attr_reader :output_root
111
+
112
+ # @return [String] the directory path for temporary files.
113
+ attr_reader :tmpdir
114
+
115
+ # @return [Array] an Array of Rake::Task objects. This
116
+ # property is populated by the #generate_rake_tasks
117
+ # method.
118
+ attr_reader :rake_tasks
119
+
120
+ # @return [String] a list of files that will be outputted
121
+ # to the output directory when the pipeline is invoked
122
+ attr_reader :output_files
123
+
124
+ # @return [Array] this pipeline's filters.
125
+ attr_reader :filters
126
+
127
+ attr_writer :input_files
128
+
129
+ def initialize
130
+ @filters = []
131
+ @tmpdir = "tmp"
132
+ @mutex = Mutex.new
133
+ end
134
+
135
+ # Build a new pipeline taking a block. The block will
136
+ # be evaluated by the Rake::Pipeline::DSL class.
137
+ #
138
+ # @see Rake::Pipeline::Filter Rake::Pipeline::Filter
139
+ #
140
+ # @example
141
+ # Rake::Pipeline.build do
142
+ # input "app/assets"
143
+ # output "public"
144
+ #
145
+ # filter Rake::Pipeline::ConcatFilter, "app.js"
146
+ # end
147
+ #
148
+ # @see DSL the Rake::Pipeline::DSL documentation.
149
+ # All instance methods of DSL are available inside
150
+ # the build block.
151
+ #
152
+ # @return [Rake::Pipeline] the newly configured pipeline
153
+ def self.build(&block)
154
+ pipeline = new
155
+ DSL.evaluate(pipeline, &block) if block
156
+ pipeline
157
+ end
158
+
159
+ @@tmp_id = 0
160
+
161
+ # Copy the current pipeline's attributes over.
162
+ #
163
+ # @param [Class] target_class the class to create a new
164
+ # instance of. Defaults to the class of the current
165
+ # pipeline. Is overridden in {Matcher}
166
+ # @param [Proc] block a block to pass to the {DSL DSL}
167
+ # @return [Pipeline] the new pipeline
168
+ # @api private
169
+ def copy(target_class=self.class, &block)
170
+ pipeline = target_class.build(&block)
171
+ pipeline.input_root = input_root
172
+ pipeline.tmpdir = tmpdir
173
+ pipeline.rake_application = rake_application
174
+ pipeline
175
+ end
176
+
177
+ # Set the input root of this pipeline and expand its path.
178
+ #
179
+ # @param [String] root this pipeline's input root
180
+ def input_root=(root)
181
+ @input_root = File.expand_path(root)
182
+ end
183
+
184
+ # Set the output root of this pipeline and expand its path.
185
+ #
186
+ # @param [String] root this pipeline's output root
187
+ def output_root=(root)
188
+ @output_root = File.expand_path(root)
189
+ end
190
+
191
+ # Set the temporary directory for this pipeline and expand its path.
192
+ #
193
+ # @param [String] root this pipeline's temporary directory
194
+ def tmpdir=(dir)
195
+ @tmpdir = File.expand_path(dir)
196
+ end
197
+
198
+ # If you specify a glob for #input_glob, this method will
199
+ # calculate the input files for the directory. If you supply
200
+ # input_files directly, this method will simply return the
201
+ # input_files you supplied.
202
+ #
203
+ # @return [Array<FileWrapper>] An Array of file wrappers
204
+ # that represent the inputs for the current pipeline.
205
+ def input_files
206
+ return @input_files if @input_files
207
+
208
+ assert_input_provided
209
+
210
+ expanded_root = File.expand_path(input_root)
211
+ files = Dir[File.join(expanded_root, input_glob)].select { |f| File.file?(f) }
212
+
213
+ files.map do |file|
214
+ relative_path = file.sub(%r{^#{Regexp.escape(expanded_root)}/}, '')
215
+ FileWrapper.new(expanded_root, relative_path)
216
+ end
217
+ end
218
+
219
+ # for Pipelines, this is every file, but it may be overridden
220
+ # by subclasses
221
+ alias eligible_input_files input_files
222
+
223
+ # @return [Rake::Application] The Rake::Application to install
224
+ # rake tasks onto. Defaults to Rake.application
225
+ def rake_application
226
+ @rake_application || Rake.application
227
+ end
228
+
229
+ # Set the rake_application on the pipeline and apply it to filters.
230
+ #
231
+ # @return [void]
232
+ def rake_application=(rake_application)
233
+ @rake_application = rake_application
234
+ @filters.each { |filter| filter.rake_application = rake_application }
235
+ @rake_tasks = nil
236
+ end
237
+
238
+ # Add one or more filters to the current pipeline.
239
+ #
240
+ # @param [Array<Filter>] filters a list of filters
241
+ # @return [void]
242
+ def add_filters(*filters)
243
+ filters.each { |filter| filter.rake_application = rake_application }
244
+ @filters.concat(filters)
245
+ end
246
+ alias add_filter add_filters
247
+
248
+ # Invoke the pipeline, processing the inputs into the output. If
249
+ # the pipeline has already been invoked, reinvoking will not
250
+ # pick up new input files added to the file system.
251
+ #
252
+ # @return [void]
253
+ def invoke
254
+ @mutex.synchronize do
255
+ self.rake_application = Rake::Application.new unless @rake_application
256
+
257
+ setup
258
+
259
+ @rake_tasks.each { |task| task.recursively_reenable(rake_application) }
260
+ @rake_tasks.each { |task| task.invoke }
261
+ end
262
+ end
263
+
264
+ # Pick up any new files added to the inputs and process them through
265
+ # the filters. Then call #invoke.
266
+ #
267
+ # @return [void]
268
+ def invoke_clean
269
+ @rake_tasks = @rake_application = nil
270
+ invoke
271
+ end
272
+
273
+ # Set up the filters and generate rake tasks. In general, this method
274
+ # is called by invoke.
275
+ #
276
+ # @return [void]
277
+ # @api private
278
+ def setup
279
+ setup_filters
280
+ generate_rake_tasks
281
+ end
282
+
283
+ # A list of the output files that invoking this pipeline will
284
+ # generate.
285
+ #
286
+ # @return [Array<FileWrapper>]
287
+ def output_files
288
+ @filters.last.output_files unless @filters.empty?
289
+ end
290
+
291
+ protected
292
+ # Generate a new temporary directory name.
293
+ #
294
+ # @return [String] a unique temporary directory name
295
+ def self.generate_tmpname
296
+ "rake-pipeline-tmp-#{@@tmp_id += 1}"
297
+ end
298
+
299
+ # Set up the filters. This will loop through all of the filters for
300
+ # the current pipeline and wire up their input_files and output_files.
301
+ #
302
+ # Because matchers implement the filter API, matchers will also be
303
+ # set up as part of this process.
304
+ #
305
+ # @return [void]
306
+ def setup_filters
307
+ last = @filters.last
308
+
309
+ @filters.inject(eligible_input_files) do |current_inputs, filter|
310
+ filter.input_files = current_inputs
311
+
312
+ # if filters are being reinvoked, they should keep their roots but
313
+ # get updated with new files.
314
+ filter.output_root ||= begin
315
+ output = if filter == last
316
+ output_root
317
+ else
318
+ generate_tmpdir
319
+ end
320
+
321
+ File.expand_path(output)
322
+ end
323
+
324
+ filter.setup_filters if filter.respond_to?(:setup_filters)
325
+
326
+ filter.output_files
327
+ end
328
+ end
329
+
330
+ # Generate a new temporary directory name under the main tmpdir.
331
+ #
332
+ # @return [void]
333
+ def generate_tmpdir
334
+ File.join(tmpdir, self.class.generate_tmpname)
335
+ end
336
+
337
+ # Generate all of the rake tasks for this pipeline.
338
+ #
339
+ # @return [void]
340
+ def generate_rake_tasks
341
+ @rake_tasks ||= begin
342
+ tasks = []
343
+
344
+ @filters.each do |filter|
345
+ # TODO: Don't generate rake tasks if we aren't
346
+ # creating a new Rake::Application
347
+ tasks = filter.generate_rake_tasks
348
+ end
349
+
350
+ tasks
351
+ end
352
+ end
353
+
354
+ # Assert that an input root and glob were both provided.
355
+ #
356
+ # @raise Rake::Pipeline::Error if input root or glob were missing.
357
+ # @return [void]
358
+ def assert_input_provided
359
+ if !input_root || !input_glob
360
+ raise Rake::Pipeline::Error, "You cannot get input files without " \
361
+ "first providing input files and an input root"
362
+ end
363
+ end
364
+ end
365
+ end