sitefuel 0.0.0a

data/lib/sitefuel/processors/CSSProcessor.rb

@@ -0,0 +1,84 @@
+#
+# File::      CSSProcessor.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# Rather lightweight CSS processor; primarily intended to minify CSS, but has
+# basic support for beautifcation as well
+#
+
+module SiteFuel
+  module Processor
+
+    require 'cssmin'
+
+    require 'sitefuel/processors/AbstractStringBasedProcessor'
+
+    class CSSProcessor < AbstractStringBasedProcessor
+
+      # file patterns for CSS
+      def self.file_patterns
+        [".css"]
+      end
+
+      #
+      # FILTER SETS
+      #
+
+      # gives the default filterset to run
+      def self.default_filterset
+        :minify
+      end
+
+      # gives the minify filter to run
+      def self.filterset_minify
+        [:minify]
+      end
+
+      # beautifies the source
+      def self.filterset_beautify
+        [:beautify]
+      end
+
+
+
+      #
+      # FILTERS
+      #
+
+      # strips comments out of CSS using Regexp
+      def filter_strip_comments
+        @document.gsub!(/\/\/.*?$/m, '')
+        @document.gsub!(/\*(.*?)\*/m, '')
+      end
+
+      # lightweight minification by removing excess whitespace
+      def filter_clean_whitespace
+        @document.gsub!(/(\s\s+)/) do |text_block|
+          # if there's a newline we keep it. This is necessary for comments.
+          # in general, however, this filter should really be run after the
+          # strip_comments filter because it's silly to clip whitespace but
+          # not comments....
+          if text_block.count("\n") > 0
+            "\n"
+          else
+            text_block[0..0]
+          end
+        end
+      end
+
+      # lightweight beautifier that works through Regexp, adds whitespace above
+      # line comments, puts each declaration on it's own line. etc.
+      def filter_beautify
+        @document
+      end
+
+      # uses the CSSMin gem to minify a CSS document using regular expressions
+      def filter_minify
+        @document = CSSMin.minify(document)
+      end
+
+    end
+  end
+end

data/lib/sitefuel/processors/HAMLProcessor.rb

@@ -0,0 +1,52 @@
+#
+# File::       HAMLProcessor.rb
+# Author::     wkm
+# Copyright::  2009
+# License::    GPL
+#
+# processes .haml source files to generate the associated HTML
+#
+
+module SiteFuel
+  module Processor
+
+    require 'sitefuel/extensions/Silently'
+
+    # since the haml gem gives exec() warnings, we temporarily lower the verbosity
+    # (last tested with 2.2.14, this might not be needed with a future version)
+    silently { require 'haml' }
+
+    require 'sitefuel/processors/AbstractStringBasedProcessor'
+    require 'sitefuel/processors/HTMLProcessor'
+
+    class HAMLProcessor < AbstractStringBasedProcessor
+
+      def self.file_patterns
+        ['.haml']
+      end
+
+      def self.default_filterset
+        :generate
+      end
+
+      def self.filterset_generate
+        [:generate, :minify]
+      end
+
+      # generate the raw .html file from a .haml file
+      def filter_generate
+        # to silence instance variable not initialized warnings from haml
+        silently {
+          engine = Haml::Engine.new(document)
+          @document = engine.render
+        }
+      end
+
+      # run the HTMLProcessor's whitespace filter
+      def filter_minify
+        @document = HTMLProcessor.filter_string(:whitespace, document)
+      end
+
+    end
+  end
+end

data/lib/sitefuel/processors/HTMLProcessor.rb

@@ -0,0 +1,211 @@
+#
+# File::      HTMLProcessor.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# Defines an HTMLProcessor class that is used to compress and cleanup HTML
+# files for deployment with sitefuel. Uses hpricot to process the HTML.
+#
+
+module SiteFuel
+  module Processor
+
+    require 'hpricot'
+
+    require 'sitefuel/processors/AbstractStringBasedProcessor'
+    require 'sitefuel/processors/CSSProcessor'
+    require 'sitefuel/processors/JavaScriptProcessor'
+
+    class HTMLProcessor < AbstractStringBasedProcessor
+
+      #
+      # HTML ENTITIES
+      #
+
+      # quotes
+      SINGLE_QUOTE_OPEN        = '&#8216;'.freeze
+      SINGLE_QUOTE_CLOSE       = '&#8217;'.freeze
+      DOUBLE_QUOTE_OPEN        = '&#8220;'.freeze
+      DOUBLE_QUOTE_CLOSE       = '&#8221;'.freeze
+
+      # dashes
+      EN_DASH                  = '&#8211;'.freeze
+      EM_DASH                  = '&#8212;'.freeze
+
+      # signs
+      ELLIPSIS                 = '&#8230;'.freeze
+      COPYRIGHT                = '&#169;'.freeze
+      TRADEMARK                = '&#8482;'.freeze
+      REGISTERED               = '&#174;'.freeze
+
+      # arrows
+      ARROW_LEFTWARD           = '&#8592;'.freeze
+      ARROW_RIGHTWARD          = '&#8594;'.freeze
+      ARROW_LEFTRIGHT          = '&#8596;'.freeze
+      ARROW_DOUBLE_LEFTWARD    = '&#8656;'.freeze
+      ARROW_DOUBLE_RIGHTWARD   = '&#8658;'.freeze
+      ARROW_DOUBLE_LEFTRIGHT   = '&#8660;'.freeze
+
+      # math operators
+      MULTIPLICATION_SIGN      = '&#215;'.freeze
+
+
+      # list of tags which have proper text items inside them
+      TEXTUAL_TAGS             = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6',
+                                  'p', 'b', 'i', 'ul', 'a', 'li', 'td',
+                                  'th'].freeze
+      
+      # filter for use with XPath searches
+      TEXTUAL_TAGS_FILTER      = TEXTUAL_TAGS.join('|').freeze
+
+
+
+      # gives the file patterns which this processor will match
+      def self.file_patterns
+        [
+          # plain html
+          ".html", ".htm"
+        ]
+      end
+
+      def self.default_filterset
+        :minify
+      end
+
+      def self.filterset_minify
+        [:whitespace, :minify_javascript, :minify_styles]
+      end
+
+      def self.filterset_beautify
+        [:beautify_quotes, :beautify_dashes, :beautify_arrows, :beautify_symbols]
+      end
+
+
+      #
+      # FILTERS
+      #
+
+      # before any filters are run parse the document with hpricot
+      def setup_filters
+        @htmlstruc = Hpricot.parse(document)
+      end
+
+      # after all the filters are run dump the HTML as a string and do a
+      # tiny bit of post processing
+      def finish_filters
+        # do a last minute, ugly +br+ cleanup
+        @document = @htmlstruc.to_s.gsub('<br />', '<br>')
+      end
+
+      def traverse(patterns = TEXTUAL_TAGS_FILTER, &block)
+        (@htmlstruc/patterns).each do |tag|
+          tag.traverse_text do |txt|
+            block.call(tag.pathname, txt)
+          end
+        end
+      end
+
+      # strips excess whitespace in most HTML tags. Notably, +pre+ tags are
+      # left alone.
+      def filter_whitespace
+        @htmlstruc.traverse_text do |txt|
+          if /\A\s+\z/ =~ txt.content then
+            txt.content = ''
+          else
+            txt.content = txt.content.gsub(/\s+/m, ' ')
+          end
+        end
+      end
+
+      # minifies embedded JavaScript code using the JavaScriptProcessor
+      def filter_minify_javascript
+        # TODO check the language attribute to make sure it's javascript
+        traverse('script') do |tag,txt|
+          txt.content = JavaScriptProcessor.process_string(
+                  txt.content,
+                  {:resource_name => resource_name+'<embedded_JS>'}
+          )
+        end
+      end
+
+      # minifies embedded CSS styles using the CSSProcessor
+      def filter_minify_styles
+        traverse('style') do |tag,txt|
+          txt.content = CSSProcessor.process_string(
+                  txt.content,
+                  :resource_name => resource_name+'<embedded_CSS>'
+          )
+        end
+      end
+
+      # cleans up double and single quotes in textual objects
+      # <pre>"hello world"  =>  &#8220; hello world&#8221;</pre>
+      def filter_beautify_quotes
+        traverse do |tag,txt|
+          txt.content = txt.content.
+            # apostrophes
+            gsub(/(\S)'(s)/i,   '\1%s\2' % SINGLE_QUOTE_CLOSE).
+            gsub(/(\Ss)'(\s)/i, '\1%s\2'   % SINGLE_QUOTE_CLOSE).
+
+            # double quotes
+            gsub(/"(\S.*?\S)"/, '%s\1%s' % [DOUBLE_QUOTE_OPEN, DOUBLE_QUOTE_CLOSE]).
+
+            # single quotes
+            gsub(/'(\S.*?\S)'/, '%s\1%s' % [SINGLE_QUOTE_OPEN, SINGLE_QUOTE_CLOSE])
+        end
+      end
+
+      # cleans up the various dash forms:
+      # <pre>12--13  =>  12&#8211;13</pre>
+      # <pre>the car---it was red---was destroyed  =>  ...&#8212;it was red&#8212;...</pre>
+      def filter_beautify_dashes
+        traverse do |tag,txt|
+          txt.content = txt.content.
+            # between two numbers we have an en dash
+            # this would be a bit cleaner with (negative) lookbehind
+            gsub(/(\d)--(\d)/,        "\\1#{EN_DASH}\\2").
+
+            # we can also have multiple en-dashes
+            gsub(/\b(--(--)+)(\b|\z|\s)/) do ||
+              EN_DASH * ($1.length / 2) + $3
+            end.
+
+            # three dashes in general are an em dash
+            gsub(/(\s|\b)---(\s|\b)/, "\\1#{EM_DASH}\\2")
+        end
+      end
+
+      # convert basic arrow forms to unicode characters
+      def filter_beautify_arrows
+        traverse do |tag,txt|
+          txt.content = txt.content.
+            gsub(/(\s|\b)-->(\s|\b)/, "\\1#{ARROW_RIGHTWARD}\\2").
+            gsub(/(\s|\b)<--(\s|\b)/, "\\1#{ARROW_LEFTWARD}\\2").
+            gsub(/(\s|\b)<->(\s|\b)/, "\\1#{ARROW_LEFTRIGHT}\\2").
+            gsub(/(\s|\b)==>(\s|\b)/, "\\1#{ARROW_DOUBLE_RIGHTWARD}\\2").
+            gsub(/(\s|\b)<==(\s|\b)/, "\\1#{ARROW_DOUBLE_LEFTWARD}\\2").
+            gsub(/(\s|\b)<=>(\s|\b)/, "\\1#{ARROW_DOUBLE_LEFTRIGHT}\\2")
+        end
+      end
+
+      # converts 'x' signs between numbers into the unicode symbol
+      def filter_beautify_math
+
+      end
+
+      # convert a few shorthands like (c), (tm) to their unicode symbols
+      def filter_beautify_symbols
+        traverse do |tag,txt|
+          txt.content = txt.content.
+            gsub(/\(tm\)/i, TRADEMARK).
+            gsub(/\(c\)/i,  COPYRIGHT).
+            gsub(/\(r\)/i,  REGISTERED).
+            gsub(/(\b| )\.\.\.(\.)?/, "\\1#{ELLIPSIS}\\2")
+
+        end
+      end
+
+    end
+  end
+end

data/lib/sitefuel/processors/JavaScriptProcessor.rb

@@ -0,0 +1,57 @@
+#
+# File::      JavaScriptProcessor.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+
+module SiteFuel
+  module Processor
+    
+    require 'jsmin'
+
+    require 'sitefuel/processors/AbstractStringBasedProcessor'
+
+    class JavaScriptProcessor < AbstractStringBasedProcessor
+
+      def self.file_patterns
+        ['.js']
+      end
+      
+      # override AbstractProcessor#processor_name so output shows up as +JS+
+      # instead of +JavaScript+.
+      def self.processor_name
+        "JS"
+      end
+
+      def self.default_filterset
+        :minify
+      end
+
+      def self.filterset_minify
+        [:minify]
+      end
+
+      CDATA_START = '//<![CDATA['
+      CDATA_END   = '//]]>'
+      def filter_minify
+        return nil if @document == nil
+        return nil if @document.length == 0
+
+        # JSMin doesn't like having files without any newlines
+        @document << "\n"
+
+        # put in CDATA placeholders
+        @document.gsub!(CDATA_START, '[[CDATA_START]]')
+        @document.gsub!(CDATA_END,   '[[CDATA_END]]')
+
+        # run the minification
+        @document = JSMin.minify(@document).strip
+
+        # put back CDATA
+        @document.gsub!('[[CDATA_START]]', CDATA_START)
+        @document.gsub!('[[CDATA_END]]',   CDATA_END)
+      end
+    end
+  end
+end

data/lib/sitefuel/processors/PHPProcessor.rb

@@ -0,0 +1,32 @@
+#
+# File::      PHPProcessor.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# Defines a PHPProcessor class; but it's really a very lightweight alias
+# around the HTMLProcessor
+#
+
+
+module SiteFuel
+  module Processor
+
+    require 'sitefuel/processors/HTMLProcessor'
+
+    # An alias for the HTMLProcessor; there is nothing PHP specific
+    # sitefuel can do at the moment.
+    class PHPProcessor < HTMLProcessor
+
+      # PHP specific file patterns
+      def self.file_patterns
+        # it doesn't really make sense to handle phps files, so we're leaving
+        # them alone. It might make sense to have a dummy empty processor for
+        # the files so it's clear we recognize them.
+        ['.php', '.phtml', '.php5']
+      end
+
+    end
+
+  end
+end

data/lib/sitefuel/processors/PNGProcessor.rb

@@ -0,0 +1,80 @@
+#
+# File::      PNGProcessor.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# Interfaces with the rbcrush library that's wrapped around pngcrush
+#
+
+module SiteFuel
+  module Processor
+
+    require 'sitefuel/processors/AbstractExternalProgramProcessor'
+    require 'sitefuel/external/PNGCrush'
+
+    include External
+
+    # processor for handling Portable Network Graphics images
+    # currently operates as a lightweight wrapper around 'pngcrush'
+    class PNGProcessor < AbstractExternalProgramProcessor
+      def self.file_patterns
+        ['.png']
+      end
+
+      #
+      # External Program Handling
+      #
+
+      # gives the name of the pngcrush binary
+      def self.program_binary
+        'pngcrush'
+      end
+
+      def self.program_version_option
+        '-version'
+      end
+
+      # we've only tested 1.5.10; but we're not using pngcrush
+      # in any special way
+      def self.appropriate_program_versions
+        "> 1.5.0"
+      end
+
+
+      #
+      # FILTERS AND FILTERSETS
+      #
+
+      def self.default_filterset
+        :max
+      end
+
+      def self.filterset_quick
+        [:quick]
+      end
+
+      def self.filterset_max
+        [:brute_chainsaw]
+      end
+
+
+      #
+      # FILTERS
+      #
+      
+      def filter_brute
+        SiteFuel::External::PNGCrush.brute(resource_name, output_filename)
+      end
+
+      def filter_quick
+        SiteFuel::External::PNGCrush.quick(resource_name, output_filename)
+      end
+
+      def filter_brute_chainsaw
+        SiteFuel::External::PNGCrush.chainsaw(resource_name, output_filename)
+      end
+    end
+
+  end
+end

data/lib/sitefuel/processors/RHTMLProcessor.rb

@@ -0,0 +1,25 @@
+#
+# File::      RHTMLProcessor.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# Defines the RHTMLProcessor class. This is a lightweight wrapper
+# around the HTMLProcessor class
+#
+
+module SiteFuel
+  module Processor
+    require 'sitefuel/processors/HTMLProcessor'
+
+    class RHTMLProcessor < HTMLProcessor
+
+      def self.file_patterns
+        ['.rhtml', '.erb']
+      end
+
+
+    end
+
+  end
+end

data/lib/sitefuel/processors/SASSProcessor.rb

@@ -0,0 +1,50 @@
+#
+# File::       SASSProcessor.rb
+# Author::     wkm
+# Copyright::  2009
+# License::    GPL
+#
+# processes .sass stylesheets to generate the associated CSS
+#
+
+module SiteFuel
+  module Processor
+    require 'sitefuel/extensions/Silently'
+
+    # since the haml gem gives exec() warnings, we temporarily lower the verbosity
+    # (last tested with 2.2.14, this might not be needed with a future version)
+    silently { require 'sass' }
+
+    require 'sitefuel/processors/AbstractStringBasedProcessor'
+    require 'sitefuel/processors/CSSProcessor'
+
+    class SASSProcessor < AbstractStringBasedProcessor
+
+      def self.file_patterns
+        ['.sass']
+      end
+      
+      def self.default_filterset
+        :generate
+      end
+
+      def self.filterset_generate
+        [:generate, :minify]
+      end
+
+      # generates the raw .css file from a .sass file
+      #
+      # TODO it's very important that generate be the first filter run
+      def filter_generate
+        engine = Sass::Engine.new(document)
+        @document = engine.render
+      end
+
+      # runs the CSSProcessor's minify filter
+      def filter_minify
+        @document = CSSProcessor.filter_string(:minify, document)
+      end
+
+    end
+  end
+end

data/test/processor_listing.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+#
+# File::      processor_listing.rb
+# Author::    wkm
+# Copyright:: 2009
+# License::   GPL
+#
+# Lists all the processors in SiteFuel with their file patterns
+#
+
+$:.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'sitefuel/SiteFuelRuntime'
+include SiteFuel
+
+SiteFuelRuntime.load_processors
+processors = SiteFuelRuntime.find_processors
+processors = processors.delete_if do |proc| 
+  proc.processor_name =~ /Abstract.*/ 
+end
+processors.each do |proc|
+  puts ' | %s | %s | %s | '%
+          [
+                  proc.processor_name.ljust(10),
+                  proc.file_patterns.join(', ').ljust(30),
+                  proc.processor_type.ljust(8)
+          ]
+end