sitefuel 0.0.0a

data/lib/sitefuel/external/JPEGTran.rb

@@ -0,0 +1,68 @@
+#
+# File::      JPEGTran.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# Wrapper around the jpegtran program.
+#
+
+module SiteFuel
+  module External
+
+    require 'sitefuel/external/AbstractExternalProgram'
+
+    class JPEGTran < AbstractExternalProgram
+
+      def self.program_name
+        'jpegtran'
+      end
+
+      # the versioning scheme for jpegtran is a little weir and not all
+      # versions of jpegtran actually give a version number. So the best
+      # we can do is check if the program exists and hope for the best.
+      def self.compatible_versions
+        ['6']
+      end
+
+      # since jpegtran by default writes jpeg files to stdout it's
+      # a little obsessed about writing everything that isn't a jpeg
+      # to stderr.
+      #
+      # this is to circumvent that.
+      def self.capture_stderr
+        true
+      end
+
+      # if the program exists... hope for the best.
+      def self.test_version_number(compatible, version_number)
+        true
+      end
+
+      # this rarely actually gives the option...
+      def self.option_version
+        '--help'
+      end
+
+      option :copy, '-copy ${value}', 'none'
+      option :optimize, '-optimize'
+      option :perfect, '-perfect'
+
+      option :output, '-outfile ${value}'
+
+      # this option must always be the last one specified
+      option :input, '${value}'
+
+
+      def self.compress_losslessly(in_file, out_file)
+        self.execute :copy,
+                     :optimize,
+                     :perfect,
+                     :output, out_file,
+                     :input, in_file
+      end
+
+    end
+
+  end
+end

data/lib/sitefuel/external/PNGCrush.rb

@@ -0,0 +1,66 @@
+#
+# File::      PNGCrush.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# Wrapper around the pngcrush program.
+#
+#
+
+module SiteFuel
+  module External
+
+    require 'sitefuel/external/AbstractExternalProgram'
+
+    # Defines a gentle wrapper around the pngcrush program. This wrapper is
+    # specifically intended for use with the -reduce and -brute options.
+    class PNGCrush < AbstractExternalProgram
+
+      def self.program_name
+        'pngcrush'
+      end
+
+      # most likely earlier versions of pngcrush would work as well
+      # but we've only ever tested it with 1.5.10
+      def self.compatible_versions
+        ['> 1.5']
+      end
+
+      # define options
+      option :version, '-version'
+      option :brute,   '-brute'
+      option :reduce,  '-reduce'
+      option :method,  '-method ${value}', '115'
+      option :rem,     '-rem ${value}', 'alla'
+      option :z,       '-z ${value}', '1'
+      option :input,   '${value}'
+      option :output,  '${value}'
+
+      # uses -brute with PNGCrush to find the smallest file size, but at the
+      # expense of taking quite a while to run.
+      def self.brute(in_file, out_file)
+        execute :brute,
+                :reduce,
+                :input, in_file,
+                :output, out_file
+      end
+
+      # quick uses the default png crush configuration to smash up PNGs
+      def self.quick(in_file, out_file)
+        execute :input, in_file,
+                :output, out_file
+      end
+
+      # strips out all data except the RGBA values (any copyrights, gamma, etc.)
+      def self.chainsaw (in_file, out_file)
+        execute :rem, 'alla',
+                :reduce,
+                :z, '1',
+                :input, in_file,
+                :output, out_file
+      end
+
+    end
+  end
+end

data/lib/sitefuel/processors/AbstractExternalProgramProcessor.rb

@@ -0,0 +1,72 @@
+#
+# File::      AbstractExternalProgramProcessor.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# TODO: this abstraction assumes only one filter will ever be run on a file,
+# which is rather naive. Need to add support to process a file multiple times.
+#
+
+
+module SiteFuel
+  module Processor
+
+    require 'tempfile'
+    require 'sitefuel/processors/AbstractProcessor'
+
+    
+    # Defines an abstract processor that offloads the work onto an external program.
+    # These are typically processors for handling binary files (eg. images)
+    #
+    # These processors spend a bunch of time ensuring the external program exists
+    # and of the appropriate version; each filter then will typically setup more
+    # parameters to pass to the program.
+    class AbstractExternalProgramProcessor < AbstractProcessor
+
+      def initialize
+        super
+        @output_filename = nil
+      end
+
+      def self.processor_type
+        'External'
+      end
+
+      # processes a file using a given configuration
+      def self.process_file(filename, config = {})
+        proc = self.new()
+        proc.configure(config)
+        proc.set_file(filename)
+        proc.generate
+      end
+
+      # sets the file used by this processor
+      def set_file(filename)
+        self.resource_name = filename
+        self.original_size = File.size(filename)
+
+        return self
+      end
+
+      # gives the output filename for this processor; typically this will
+      # be a temporary file.
+      def output_filename
+        if @output_filename == nil
+          @output_filename = Tempfile.new(File.basename(resource_name)).path
+        end
+
+        @output_filename
+      end
+
+      # generates the new document using external programs
+      def generate
+        self.execute
+        self.processed_size = File.size(output_filename)
+        return self
+      end
+      
+    end
+
+  end
+end

data/lib/sitefuel/processors/AbstractProcessor.rb

@@ -0,0 +1,378 @@
+#
+# File::      AbstractProcessor.rb
+# Author::    wkm
+# Copyright:: 2009
+#
+# Defines an AbstractProcessor class that gives the interface implemented by
+# specific processors.
+#
+
+module SiteFuel
+  module Processor
+
+    require 'sitefuel/SiteFuelLogger'
+
+    # raised when a method isn't implemented by a child class.
+    class NotImplemented < StandardError; end
+
+    # raised when attempting to run a filter that doesn't exist
+    class UnknownFilter < StandardError
+      attr_reader :processor, :name
+
+      def initialize(processor, name)
+        @processor = processor
+        @name = name
+      end
+
+      def to_s
+        "'%s' called for processor '%s'" %
+        [@name, @processor.class]
+      end
+    end
+
+    class UnknownFilterset < StandardError
+      attr_reader :processor, :name
+
+      def initialize(processor, name)
+        @processor = processor
+        @name = name
+      end
+
+      def to_s
+        "'%s' called for processor '%s'" %
+        [@name, @processor.class]
+      end
+    end
+
+    # raised when multiple processors trigger off of a single file
+    class MultipleApplicableProcessors < StandardError
+      attr_reader :filename, :processors, :chosen_processor
+
+      def initialize(filename, processors, chosen_processor)
+        @filename = filename
+        @resource_processors = processors
+        @chosen_processor = chosen_processor
+      end
+
+      def to_s
+        "File '%s' triggered processors: %s. Using %s" %
+        [@filename, @resource_processors.join(', '), @chosen_processor.class]
+      end
+    end
+
+    # defines the base functions every processor must implement to
+    # interface with the sitefuel architecture
+    class AbstractProcessor
+
+      include SiteFuel::Logging
+
+      # setup an AbstractProcessor
+      def initialize
+        self.logger = SiteFuelLogger.instance
+        @execution_list = []
+        @filters = []
+      end
+
+
+      # gives a list of processors that implement AbstractProcessor
+      def self.find_processors
+        procs = []
+        ObjectSpace.each_object(Class) do |cls|
+          if cls.ancestors.include?(self) and
+             cls.to_s =~ /^.*Processor$/ and
+             not cls.to_s =~ /^.*Abstract.*Processor$/
+          then
+            procs << cls
+          end
+        end
+
+        procs
+      end
+
+      # gives the type of the processor, usually implemented
+      # by the more specific abstract processors.
+      def self.processor_type
+        ''
+      end
+
+
+      #
+      # PROCESSOR INFORMATION
+      #
+
+      # gives the canonical name of the resource
+      attr_reader :resource_name
+
+      # gives the original size of a resource before being processed
+      attr_reader :original_size
+
+      # gives the size of the resouce now that it's been processed
+      attr_reader :processed_size
+
+      # gives the display name for the processor
+      def self.processor_name
+        self.to_s.sub(/.*\b(.*)Processor/, '\1')
+      end
+
+      
+      # gives the file patterns that trigger the processor by default; this
+      # behavior can be over-ridden by configuration files.
+      #
+      # * strings are assumed to be extensions and are tested for a literal match
+      # * regexes are tested against the entire file name
+      #
+      def self.file_patterns
+        []
+      end
+
+      # gives +true+ if filename matches one of #file_patterns.
+      def self.file_pattern_match?(filename)
+        file_patterns.map { |patt|
+          case patt
+          when String
+              regex = Regexp.new("^.*"+Regexp.escape(patt)+"$", Regexp::IGNORECASE)
+              return true if filename.match(regex) != nil
+          when Regexp
+              return true if filename.match(patt) != nil
+          end
+        }
+        
+        # if we got this far nothing matched
+        return false
+      end
+
+      
+      # uses #file_pattern_match? to decide if the file can be processed
+      # eventually this may use other metrics (like mime types)
+      def self.processes_file?(filename)
+        file_pattern_match? filename
+      end
+
+
+      
+
+      #
+      # FILTER SET SUPPORT
+      #
+
+      # gives the default filterset used
+      def self.default_filterset
+        nil
+      end
+
+      # lists all filtersets for this processor
+      def self.filtersets
+        names = methods
+        names = names.delete_if{|method| not method =~ /^filterset_.*$/ }
+        names.sort!
+
+        names.map { |filterset| filterset.sub(/^filterset_(.*)$/, '\1').to_sym }
+      end
+
+      # gives true if the given name is of a filter set for this processor
+      def self.filterset?(name)
+        respond_to?("filterset_" + name.to_s)
+      end
+
+      # the ignore filter set is used when configuring sitefuel to not process
+      # certain kinds of files.
+      def self.filterset_ignore
+        []
+      end
+
+      # returns the filters in the given filter set, [] if no such filters
+      # exist
+      def self.filters_in_filterset(name)
+        return [] unless self.filterset?(name)
+
+        filter_list = self.send("filterset_" + name.to_s)
+        
+        if filter_list == nil
+          return []
+        else
+          return filter_list
+        end
+      end
+
+      # adds the filters in a filterset to the execution list
+      def add_filterset(filterset)
+        if self.class.filterset?(filterset)
+          # extract the filters in the filterset and add them to the list
+          filter_list = self.class.filters_in_filterset(filterset)
+          filter_list.each do |filter|
+            add_filter(filter)
+          end
+          @execution_list
+        else
+          raise UnknownFilterset.new(self, filterset)
+        end
+      end
+
+      # evaluate a filterset
+      def run_filterset(name)
+        if self.class.filter_set?("filterset_" + name.to_s)
+          self.send("filterset_" + name.to_s)
+        else
+          raise UnknownFilterset(self, name)
+        end
+      end
+
+
+      #
+      # FILTER SUPPORT
+      #
+
+      # lists all the filters implemented by a processor
+      def self.filters
+        names = instance_methods
+        names = names.delete_if{ |method| not method =~ /^filter_.*$/ }
+        names.sort!
+        
+        names.map { |filter_name| filter_name.sub(/^filter_(.*)$/, '\1').to_sym }
+      end
+
+      # gives true if the given filter is known for this processor class
+      def self.filter?(name)
+        filters.include?(name.to_sym)
+      end
+
+      # gives true if the given filter is known for this processor instance
+      def filter?(filter)
+        respond_to?("filter_" + filter.to_s)
+      end
+
+      # array of filters to run
+      attr_reader :execution_list
+
+      # adds a filter or array of filters to the execution list
+      #
+      #  add_filter(:minify)
+      #  add_filter([:beautifytext, :minify])
+      def add_filter(filter)
+        case filter
+        when Array
+          filter.each do |f|
+            add_filter f
+          end
+        when Symbol, String
+          if filter?(filter)
+            @execution_list << filter
+          else
+            raise UnknownFilter.new(self, filter)
+          end
+        end
+      end
+
+      # clears all filters from the execution list
+      def clear_filters
+        @execution_list = []
+      end
+
+      # drops a filter from the execution list
+      def drop_filter(filter)
+        @execution_list.delete(filter)
+        @execution_list
+      end
+
+      # runs a particular filter
+      def run_filter(name)
+        if respond_to?("filter_" + name.to_s)
+          self.send("filter_"+name.to_s)
+        else
+          raise UnknownFilter.new(self, name)
+        end
+      end
+
+      # called in #execute _before_ running the execution list of filters; note
+      # that #setup_filters is only called _once_ before all of the filters are
+      # batch executed. It is not called before every filter executes.
+      def setup_filters; end
+
+      # called in #execute _after_ running the execution list of filters
+      def finish_filters; end
+
+      # runs all filters in the execution list
+      def execute
+        setup_filters
+        @execution_list.uniq.each do |filter|
+          run_filter(filter)
+        end
+        finish_filters
+      rescue => exception
+        error 'from %s:%s: %s' % [self.class, resource_name, exception]
+      end
+
+
+
+
+      def save(basepath)
+        File.open(File.join(basepath, resource_name), 'w') do |fhndl|
+          fhndl.write(document.to_s)
+        end
+      end
+
+
+      #
+      # CONFIGURATION SUPPORT
+      #
+      def configure(config)
+        @filters_cleared = false
+        unless config == nil  or  config == {}
+          config.each_pair do |k, v|
+            set_configuration(k, v)
+          end
+        end
+
+        if !@filters_cleared && execution_list.empty?
+          add_filterset(self.class.default_filterset)
+        end
+        @filters_cleared = false
+      end
+
+    private
+      def set_configuration(key, value)
+        case key
+          when :resource_name
+            @resource_name = value
+
+
+          when :filters
+            if not @filters_cleared
+              clear_filters
+              @filters_cleared = true
+            end
+
+            case value
+              when Array
+                value.each { |v| add_filter(v) }
+              when Symbol, String
+                add_filter(value)
+            end
+
+          when :filtersets
+            if not @filters_cleared
+              clear_filters
+              @filters_cleared = true
+            end
+
+            case value
+              when Array
+                value.each { |v| add_filterset(v) }
+              when Symbol, String
+                add_filterset(value)
+            end
+
+          else
+            raise UnknownConfigurationOption(self.class, key, value)
+        end
+      end
+
+    protected
+      # gives write-access to children classes
+      attr_writer :original_size
+      attr_writer :processed_size
+      attr_writer :resource_name
+      
+    end
+  end
+end

data/lib/sitefuel/processors/AbstractStringBasedProcessor.rb

@@ -0,0 +1,88 @@
+#
+# File::      AbstractStringBasedProcessor.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# Defines an abstract processor that runs by loading an entire file into
+# memory as a string. Since most files we're looking at are very small
+# anyway (seeing as they're intended to be served millions of times) this
+# is usually fine.
+#
+
+module SiteFuel
+  module Processor
+
+    require 'sitefuel/processors/AbstractProcessor'
+
+    class AbstractStringBasedProcessor < AbstractProcessor
+      
+      def self.processor_type
+        'String'
+      end
+
+      # lightweight wrapper for opening a resource and generating the file
+      def self.process_file(filename, config = {})
+        proc = self.new()
+        proc.configure(config)
+        proc.open_file(filename)
+        proc.generate
+      end
+
+      # opens a resource in-memory and returns the generated string
+      def self.process_string(string, config = {})
+        proc = self.new()
+        proc.configure(config)
+        proc.open_string(string)
+        proc.generate_string
+      end
+
+      # mostly intended for debugging; applies a single filter directly
+      # to a string
+      #
+      # filter can either be a single filter or an array of filters
+      def self.filter_string(filter, string)
+        proc = self.new()
+        proc.configure(:filters => filter)
+        proc.open_string(string)
+        proc.generate_string
+      end
+
+      # opens a resource from a file
+      def open_file(filename)
+        self.document = File.read(filename)
+        self.original_size = File.size(filename)
+        self.resource_name = filename
+
+        return self
+      end
+
+      # opens a resource directly from a string
+      def open_string(string)
+        self.document = string
+        self.original_size = string.length
+        self.resource_name = '<<in-memory string>>'
+      end
+
+      # generates the actual string
+      def generate_string
+        self.execute
+        self.processed_size = @document.length
+
+        document
+      end
+
+      # generates the string and shoves it into the deployment abstraction
+      def generate
+        generate_string
+        return self
+      end
+      
+      attr_reader :document
+      
+      protected
+      attr_writer :document
+    end
+    
+  end
+end