sitefuel 0.0.0a

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. data/README +86 -0
  2. data/RELEASE_NOTES +7 -0
  3. data/bin/sitefuel +162 -0
  4. data/lib/sitefuel/Configuration.rb +35 -0
  5. data/lib/sitefuel/SiteFuelLogger.rb +128 -0
  6. data/lib/sitefuel/SiteFuelRuntime.rb +293 -0
  7. data/lib/sitefuel/extensions/ArrayComparisons.rb +34 -0
  8. data/lib/sitefuel/extensions/DynamicClassMethods.rb +19 -0
  9. data/lib/sitefuel/extensions/FileComparison.rb +24 -0
  10. data/lib/sitefuel/extensions/Silently.rb +27 -0
  11. data/lib/sitefuel/extensions/StringFormatting.rb +75 -0
  12. data/lib/sitefuel/extensions/SymbolComparison.rb +13 -0
  13. data/lib/sitefuel/external/AbstractExternalProgram.rb +616 -0
  14. data/lib/sitefuel/external/ExternalProgramTestCase.rb +67 -0
  15. data/lib/sitefuel/external/GIT.rb +9 -0
  16. data/lib/sitefuel/external/JPEGTran.rb +68 -0
  17. data/lib/sitefuel/external/PNGCrush.rb +66 -0
  18. data/lib/sitefuel/processors/AbstractExternalProgramProcessor.rb +72 -0
  19. data/lib/sitefuel/processors/AbstractProcessor.rb +378 -0
  20. data/lib/sitefuel/processors/AbstractStringBasedProcessor.rb +88 -0
  21. data/lib/sitefuel/processors/CSSProcessor.rb +84 -0
  22. data/lib/sitefuel/processors/HAMLProcessor.rb +52 -0
  23. data/lib/sitefuel/processors/HTMLProcessor.rb +211 -0
  24. data/lib/sitefuel/processors/JavaScriptProcessor.rb +57 -0
  25. data/lib/sitefuel/processors/PHPProcessor.rb +32 -0
  26. data/lib/sitefuel/processors/PNGProcessor.rb +80 -0
  27. data/lib/sitefuel/processors/RHTMLProcessor.rb +25 -0
  28. data/lib/sitefuel/processors/SASSProcessor.rb +50 -0
  29. data/test/processor_listing.rb +28 -0
  30. data/test/test_AbstractExternalProgram.rb +186 -0
  31. data/test/test_AbstractProcessor.rb +237 -0
  32. data/test/test_AbstractStringBasedProcessor.rb +48 -0
  33. data/test/test_AllProcessors.rb +65 -0
  34. data/test/test_ArrayComparisons.rb +32 -0
  35. data/test/test_CSSProcessor.rb +120 -0
  36. data/test/test_FileComparisons.rb +42 -0
  37. data/test/test_HAMLProcessor.rb.rb +60 -0
  38. data/test/test_HTMLProcessor.rb +186 -0
  39. data/test/test_JPEGTran.rb +40 -0
  40. data/test/test_JavaScriptProcessor.rb +63 -0
  41. data/test/test_PHPProcessor.rb +51 -0
  42. data/test/test_PNGCrush.rb +58 -0
  43. data/test/test_PNGProcessor.rb +32 -0
  44. data/test/test_RHTMLProcessor.rb +62 -0
  45. data/test/test_SASSProcessor.rb +68 -0
  46. data/test/test_SiteFuelLogging.rb +79 -0
  47. data/test/test_SiteFuelRuntime.rb +96 -0
  48. data/test/test_StringFormatting.rb +51 -0
  49. data/test/test_SymbolComparison.rb +27 -0
  50. data/test/test_images/sample_jpg01.jpg +0 -0
  51. data/test/test_images/sample_png01.png +0 -0
  52. data/test/test_programs/versioning.rb +26 -0
  53. data/test/test_sites/simplehtml/deployment.yml +22 -0
  54. data/test/test_sites/simplehtml/index.html +66 -0
  55. data/test/test_sites/simplehtml/style.css +40 -0
  56. data/test/ts_all.rb +39 -0
  57. metadata +165 -0
@@ -0,0 +1,68 @@
1
+ #
2
+ # File:: JPEGTran.rb
3
+ # Author:: wkm
4
+ # Copyright:: 2009
5
+ # License:: GPL
6
+ #
7
+ # Wrapper around the jpegtran program.
8
+ #
9
+
10
+ module SiteFuel
11
+ module External
12
+
13
+ require 'sitefuel/external/AbstractExternalProgram'
14
+
15
+ class JPEGTran < AbstractExternalProgram
16
+
17
+ def self.program_name
18
+ 'jpegtran'
19
+ end
20
+
21
+ # the versioning scheme for jpegtran is a little weir and not all
22
+ # versions of jpegtran actually give a version number. So the best
23
+ # we can do is check if the program exists and hope for the best.
24
+ def self.compatible_versions
25
+ ['6']
26
+ end
27
+
28
+ # since jpegtran by default writes jpeg files to stdout it's
29
+ # a little obsessed about writing everything that isn't a jpeg
30
+ # to stderr.
31
+ #
32
+ # this is to circumvent that.
33
+ def self.capture_stderr
34
+ true
35
+ end
36
+
37
+ # if the program exists... hope for the best.
38
+ def self.test_version_number(compatible, version_number)
39
+ true
40
+ end
41
+
42
+ # this rarely actually gives the option...
43
+ def self.option_version
44
+ '--help'
45
+ end
46
+
47
+ option :copy, '-copy ${value}', 'none'
48
+ option :optimize, '-optimize'
49
+ option :perfect, '-perfect'
50
+
51
+ option :output, '-outfile ${value}'
52
+
53
+ # this option must always be the last one specified
54
+ option :input, '${value}'
55
+
56
+
57
+ def self.compress_losslessly(in_file, out_file)
58
+ self.execute :copy,
59
+ :optimize,
60
+ :perfect,
61
+ :output, out_file,
62
+ :input, in_file
63
+ end
64
+
65
+ end
66
+
67
+ end
68
+ end
@@ -0,0 +1,66 @@
1
+ #
2
+ # File:: PNGCrush.rb
3
+ # Author:: wkm
4
+ # Copyright:: 2009
5
+ # License:: GPL
6
+ #
7
+ # Wrapper around the pngcrush program.
8
+ #
9
+ #
10
+
11
+ module SiteFuel
12
+ module External
13
+
14
+ require 'sitefuel/external/AbstractExternalProgram'
15
+
16
+ # Defines a gentle wrapper around the pngcrush program. This wrapper is
17
+ # specifically intended for use with the -reduce and -brute options.
18
+ class PNGCrush < AbstractExternalProgram
19
+
20
+ def self.program_name
21
+ 'pngcrush'
22
+ end
23
+
24
+ # most likely earlier versions of pngcrush would work as well
25
+ # but we've only ever tested it with 1.5.10
26
+ def self.compatible_versions
27
+ ['> 1.5']
28
+ end
29
+
30
+ # define options
31
+ option :version, '-version'
32
+ option :brute, '-brute'
33
+ option :reduce, '-reduce'
34
+ option :method, '-method ${value}', '115'
35
+ option :rem, '-rem ${value}', 'alla'
36
+ option :z, '-z ${value}', '1'
37
+ option :input, '${value}'
38
+ option :output, '${value}'
39
+
40
+ # uses -brute with PNGCrush to find the smallest file size, but at the
41
+ # expense of taking quite a while to run.
42
+ def self.brute(in_file, out_file)
43
+ execute :brute,
44
+ :reduce,
45
+ :input, in_file,
46
+ :output, out_file
47
+ end
48
+
49
+ # quick uses the default png crush configuration to smash up PNGs
50
+ def self.quick(in_file, out_file)
51
+ execute :input, in_file,
52
+ :output, out_file
53
+ end
54
+
55
+ # strips out all data except the RGBA values (any copyrights, gamma, etc.)
56
+ def self.chainsaw (in_file, out_file)
57
+ execute :rem, 'alla',
58
+ :reduce,
59
+ :z, '1',
60
+ :input, in_file,
61
+ :output, out_file
62
+ end
63
+
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,72 @@
1
+ #
2
+ # File:: AbstractExternalProgramProcessor.rb
3
+ # Author:: wkm
4
+ # Copyright:: 2009
5
+ # License:: GPL
6
+ #
7
+ # TODO: this abstraction assumes only one filter will ever be run on a file,
8
+ # which is rather naive. Need to add support to process a file multiple times.
9
+ #
10
+
11
+
12
+ module SiteFuel
13
+ module Processor
14
+
15
+ require 'tempfile'
16
+ require 'sitefuel/processors/AbstractProcessor'
17
+
18
+
19
+ # Defines an abstract processor that offloads the work onto an external program.
20
+ # These are typically processors for handling binary files (eg. images)
21
+ #
22
+ # These processors spend a bunch of time ensuring the external program exists
23
+ # and of the appropriate version; each filter then will typically setup more
24
+ # parameters to pass to the program.
25
+ class AbstractExternalProgramProcessor < AbstractProcessor
26
+
27
+ def initialize
28
+ super
29
+ @output_filename = nil
30
+ end
31
+
32
+ def self.processor_type
33
+ 'External'
34
+ end
35
+
36
+ # processes a file using a given configuration
37
+ def self.process_file(filename, config = {})
38
+ proc = self.new()
39
+ proc.configure(config)
40
+ proc.set_file(filename)
41
+ proc.generate
42
+ end
43
+
44
+ # sets the file used by this processor
45
+ def set_file(filename)
46
+ self.resource_name = filename
47
+ self.original_size = File.size(filename)
48
+
49
+ return self
50
+ end
51
+
52
+ # gives the output filename for this processor; typically this will
53
+ # be a temporary file.
54
+ def output_filename
55
+ if @output_filename == nil
56
+ @output_filename = Tempfile.new(File.basename(resource_name)).path
57
+ end
58
+
59
+ @output_filename
60
+ end
61
+
62
+ # generates the new document using external programs
63
+ def generate
64
+ self.execute
65
+ self.processed_size = File.size(output_filename)
66
+ return self
67
+ end
68
+
69
+ end
70
+
71
+ end
72
+ end
@@ -0,0 +1,378 @@
1
+ #
2
+ # File:: AbstractProcessor.rb
3
+ # Author:: wkm
4
+ # Copyright:: 2009
5
+ #
6
+ # Defines an AbstractProcessor class that gives the interface implemented by
7
+ # specific processors.
8
+ #
9
+
10
+ module SiteFuel
11
+ module Processor
12
+
13
+ require 'sitefuel/SiteFuelLogger'
14
+
15
+ # raised when a method isn't implemented by a child class.
16
+ class NotImplemented < StandardError; end
17
+
18
+ # raised when attempting to run a filter that doesn't exist
19
+ class UnknownFilter < StandardError
20
+ attr_reader :processor, :name
21
+
22
+ def initialize(processor, name)
23
+ @processor = processor
24
+ @name = name
25
+ end
26
+
27
+ def to_s
28
+ "'%s' called for processor '%s'" %
29
+ [@name, @processor.class]
30
+ end
31
+ end
32
+
33
+ class UnknownFilterset < StandardError
34
+ attr_reader :processor, :name
35
+
36
+ def initialize(processor, name)
37
+ @processor = processor
38
+ @name = name
39
+ end
40
+
41
+ def to_s
42
+ "'%s' called for processor '%s'" %
43
+ [@name, @processor.class]
44
+ end
45
+ end
46
+
47
+ # raised when multiple processors trigger off of a single file
48
+ class MultipleApplicableProcessors < StandardError
49
+ attr_reader :filename, :processors, :chosen_processor
50
+
51
+ def initialize(filename, processors, chosen_processor)
52
+ @filename = filename
53
+ @resource_processors = processors
54
+ @chosen_processor = chosen_processor
55
+ end
56
+
57
+ def to_s
58
+ "File '%s' triggered processors: %s. Using %s" %
59
+ [@filename, @resource_processors.join(', '), @chosen_processor.class]
60
+ end
61
+ end
62
+
63
+ # defines the base functions every processor must implement to
64
+ # interface with the sitefuel architecture
65
+ class AbstractProcessor
66
+
67
+ include SiteFuel::Logging
68
+
69
+ # setup an AbstractProcessor
70
+ def initialize
71
+ self.logger = SiteFuelLogger.instance
72
+ @execution_list = []
73
+ @filters = []
74
+ end
75
+
76
+
77
+ # gives a list of processors that implement AbstractProcessor
78
+ def self.find_processors
79
+ procs = []
80
+ ObjectSpace.each_object(Class) do |cls|
81
+ if cls.ancestors.include?(self) and
82
+ cls.to_s =~ /^.*Processor$/ and
83
+ not cls.to_s =~ /^.*Abstract.*Processor$/
84
+ then
85
+ procs << cls
86
+ end
87
+ end
88
+
89
+ procs
90
+ end
91
+
92
+ # gives the type of the processor, usually implemented
93
+ # by the more specific abstract processors.
94
+ def self.processor_type
95
+ ''
96
+ end
97
+
98
+
99
+ #
100
+ # PROCESSOR INFORMATION
101
+ #
102
+
103
+ # gives the canonical name of the resource
104
+ attr_reader :resource_name
105
+
106
+ # gives the original size of a resource before being processed
107
+ attr_reader :original_size
108
+
109
+ # gives the size of the resouce now that it's been processed
110
+ attr_reader :processed_size
111
+
112
+ # gives the display name for the processor
113
+ def self.processor_name
114
+ self.to_s.sub(/.*\b(.*)Processor/, '\1')
115
+ end
116
+
117
+
118
+ # gives the file patterns that trigger the processor by default; this
119
+ # behavior can be over-ridden by configuration files.
120
+ #
121
+ # * strings are assumed to be extensions and are tested for a literal match
122
+ # * regexes are tested against the entire file name
123
+ #
124
+ def self.file_patterns
125
+ []
126
+ end
127
+
128
+ # gives +true+ if filename matches one of #file_patterns.
129
+ def self.file_pattern_match?(filename)
130
+ file_patterns.map { |patt|
131
+ case patt
132
+ when String
133
+ regex = Regexp.new("^.*"+Regexp.escape(patt)+"$", Regexp::IGNORECASE)
134
+ return true if filename.match(regex) != nil
135
+ when Regexp
136
+ return true if filename.match(patt) != nil
137
+ end
138
+ }
139
+
140
+ # if we got this far nothing matched
141
+ return false
142
+ end
143
+
144
+
145
+ # uses #file_pattern_match? to decide if the file can be processed
146
+ # eventually this may use other metrics (like mime types)
147
+ def self.processes_file?(filename)
148
+ file_pattern_match? filename
149
+ end
150
+
151
+
152
+
153
+
154
+ #
155
+ # FILTER SET SUPPORT
156
+ #
157
+
158
+ # gives the default filterset used
159
+ def self.default_filterset
160
+ nil
161
+ end
162
+
163
+ # lists all filtersets for this processor
164
+ def self.filtersets
165
+ names = methods
166
+ names = names.delete_if{|method| not method =~ /^filterset_.*$/ }
167
+ names.sort!
168
+
169
+ names.map { |filterset| filterset.sub(/^filterset_(.*)$/, '\1').to_sym }
170
+ end
171
+
172
+ # gives true if the given name is of a filter set for this processor
173
+ def self.filterset?(name)
174
+ respond_to?("filterset_" + name.to_s)
175
+ end
176
+
177
+ # the ignore filter set is used when configuring sitefuel to not process
178
+ # certain kinds of files.
179
+ def self.filterset_ignore
180
+ []
181
+ end
182
+
183
+ # returns the filters in the given filter set, [] if no such filters
184
+ # exist
185
+ def self.filters_in_filterset(name)
186
+ return [] unless self.filterset?(name)
187
+
188
+ filter_list = self.send("filterset_" + name.to_s)
189
+
190
+ if filter_list == nil
191
+ return []
192
+ else
193
+ return filter_list
194
+ end
195
+ end
196
+
197
+ # adds the filters in a filterset to the execution list
198
+ def add_filterset(filterset)
199
+ if self.class.filterset?(filterset)
200
+ # extract the filters in the filterset and add them to the list
201
+ filter_list = self.class.filters_in_filterset(filterset)
202
+ filter_list.each do |filter|
203
+ add_filter(filter)
204
+ end
205
+ @execution_list
206
+ else
207
+ raise UnknownFilterset.new(self, filterset)
208
+ end
209
+ end
210
+
211
+ # evaluate a filterset
212
+ def run_filterset(name)
213
+ if self.class.filter_set?("filterset_" + name.to_s)
214
+ self.send("filterset_" + name.to_s)
215
+ else
216
+ raise UnknownFilterset(self, name)
217
+ end
218
+ end
219
+
220
+
221
+ #
222
+ # FILTER SUPPORT
223
+ #
224
+
225
+ # lists all the filters implemented by a processor
226
+ def self.filters
227
+ names = instance_methods
228
+ names = names.delete_if{ |method| not method =~ /^filter_.*$/ }
229
+ names.sort!
230
+
231
+ names.map { |filter_name| filter_name.sub(/^filter_(.*)$/, '\1').to_sym }
232
+ end
233
+
234
+ # gives true if the given filter is known for this processor class
235
+ def self.filter?(name)
236
+ filters.include?(name.to_sym)
237
+ end
238
+
239
+ # gives true if the given filter is known for this processor instance
240
+ def filter?(filter)
241
+ respond_to?("filter_" + filter.to_s)
242
+ end
243
+
244
+ # array of filters to run
245
+ attr_reader :execution_list
246
+
247
+ # adds a filter or array of filters to the execution list
248
+ #
249
+ # add_filter(:minify)
250
+ # add_filter([:beautifytext, :minify])
251
+ def add_filter(filter)
252
+ case filter
253
+ when Array
254
+ filter.each do |f|
255
+ add_filter f
256
+ end
257
+ when Symbol, String
258
+ if filter?(filter)
259
+ @execution_list << filter
260
+ else
261
+ raise UnknownFilter.new(self, filter)
262
+ end
263
+ end
264
+ end
265
+
266
+ # clears all filters from the execution list
267
+ def clear_filters
268
+ @execution_list = []
269
+ end
270
+
271
+ # drops a filter from the execution list
272
+ def drop_filter(filter)
273
+ @execution_list.delete(filter)
274
+ @execution_list
275
+ end
276
+
277
+ # runs a particular filter
278
+ def run_filter(name)
279
+ if respond_to?("filter_" + name.to_s)
280
+ self.send("filter_"+name.to_s)
281
+ else
282
+ raise UnknownFilter.new(self, name)
283
+ end
284
+ end
285
+
286
+ # called in #execute _before_ running the execution list of filters; note
287
+ # that #setup_filters is only called _once_ before all of the filters are
288
+ # batch executed. It is not called before every filter executes.
289
+ def setup_filters; end
290
+
291
+ # called in #execute _after_ running the execution list of filters
292
+ def finish_filters; end
293
+
294
+ # runs all filters in the execution list
295
+ def execute
296
+ setup_filters
297
+ @execution_list.uniq.each do |filter|
298
+ run_filter(filter)
299
+ end
300
+ finish_filters
301
+ rescue => exception
302
+ error 'from %s:%s: %s' % [self.class, resource_name, exception]
303
+ end
304
+
305
+
306
+
307
+
308
+ def save(basepath)
309
+ File.open(File.join(basepath, resource_name), 'w') do |fhndl|
310
+ fhndl.write(document.to_s)
311
+ end
312
+ end
313
+
314
+
315
+ #
316
+ # CONFIGURATION SUPPORT
317
+ #
318
+ def configure(config)
319
+ @filters_cleared = false
320
+ unless config == nil or config == {}
321
+ config.each_pair do |k, v|
322
+ set_configuration(k, v)
323
+ end
324
+ end
325
+
326
+ if !@filters_cleared && execution_list.empty?
327
+ add_filterset(self.class.default_filterset)
328
+ end
329
+ @filters_cleared = false
330
+ end
331
+
332
+ private
333
+ def set_configuration(key, value)
334
+ case key
335
+ when :resource_name
336
+ @resource_name = value
337
+
338
+
339
+ when :filters
340
+ if not @filters_cleared
341
+ clear_filters
342
+ @filters_cleared = true
343
+ end
344
+
345
+ case value
346
+ when Array
347
+ value.each { |v| add_filter(v) }
348
+ when Symbol, String
349
+ add_filter(value)
350
+ end
351
+
352
+ when :filtersets
353
+ if not @filters_cleared
354
+ clear_filters
355
+ @filters_cleared = true
356
+ end
357
+
358
+ case value
359
+ when Array
360
+ value.each { |v| add_filterset(v) }
361
+ when Symbol, String
362
+ add_filterset(value)
363
+ end
364
+
365
+ else
366
+ raise UnknownConfigurationOption(self.class, key, value)
367
+ end
368
+ end
369
+
370
+ protected
371
+ # gives write-access to children classes
372
+ attr_writer :original_size
373
+ attr_writer :processed_size
374
+ attr_writer :resource_name
375
+
376
+ end
377
+ end
378
+ end
@@ -0,0 +1,88 @@
1
+ #
2
+ # File:: AbstractStringBasedProcessor.rb
3
+ # Author:: wkm
4
+ # Copyright:: 2009
5
+ # License:: GPL
6
+ #
7
+ # Defines an abstract processor that runs by loading an entire file into
8
+ # memory as a string. Since most files we're looking at are very small
9
+ # anyway (seeing as they're intended to be served millions of times) this
10
+ # is usually fine.
11
+ #
12
+
13
+ module SiteFuel
14
+ module Processor
15
+
16
+ require 'sitefuel/processors/AbstractProcessor'
17
+
18
+ class AbstractStringBasedProcessor < AbstractProcessor
19
+
20
+ def self.processor_type
21
+ 'String'
22
+ end
23
+
24
+ # lightweight wrapper for opening a resource and generating the file
25
+ def self.process_file(filename, config = {})
26
+ proc = self.new()
27
+ proc.configure(config)
28
+ proc.open_file(filename)
29
+ proc.generate
30
+ end
31
+
32
+ # opens a resource in-memory and returns the generated string
33
+ def self.process_string(string, config = {})
34
+ proc = self.new()
35
+ proc.configure(config)
36
+ proc.open_string(string)
37
+ proc.generate_string
38
+ end
39
+
40
+ # mostly intended for debugging; applies a single filter directly
41
+ # to a string
42
+ #
43
+ # filter can either be a single filter or an array of filters
44
+ def self.filter_string(filter, string)
45
+ proc = self.new()
46
+ proc.configure(:filters => filter)
47
+ proc.open_string(string)
48
+ proc.generate_string
49
+ end
50
+
51
+ # opens a resource from a file
52
+ def open_file(filename)
53
+ self.document = File.read(filename)
54
+ self.original_size = File.size(filename)
55
+ self.resource_name = filename
56
+
57
+ return self
58
+ end
59
+
60
+ # opens a resource directly from a string
61
+ def open_string(string)
62
+ self.document = string
63
+ self.original_size = string.length
64
+ self.resource_name = '<<in-memory string>>'
65
+ end
66
+
67
+ # generates the actual string
68
+ def generate_string
69
+ self.execute
70
+ self.processed_size = @document.length
71
+
72
+ document
73
+ end
74
+
75
+ # generates the string and shoves it into the deployment abstraction
76
+ def generate
77
+ generate_string
78
+ return self
79
+ end
80
+
81
+ attr_reader :document
82
+
83
+ protected
84
+ attr_writer :document
85
+ end
86
+
87
+ end
88
+ end