awestruct 0.5.6.beta8 → 0.5.6.beta9

data/lib/awestruct/context_helper.rb

@@ -1,8 +1,8 @@
-require 'rexml/document'
+require 'oga'
+require 'awestruct/util/exception_helper'
 
 module Awestruct
   module ContextHelper
-    include REXML
 
     def html_to_text(str)
       str.gsub( /<[^>]+>/, '' ).gsub( /&nbsp;/, ' ' )
@@ -39,27 +39,31 @@ module Awestruct
     end
 
     def fully_qualify_urls(base_url, text)
-      doc = Document.new text
-      doc.context[:attribute_quote] = :quote  # Set double-quote as the attribute value delimiter
+      begin
+        doc = Oga.parse_html text
 
-      XPath.each(doc, "//a") do |a|
-        a.attributes['href'] = fix_url( base_url, a.attributes['href'] ) if a.attributes['href']
-      end
-
-      XPath.each(doc, "//link") do |link|
-        link.attributes['href'] = fix_url( base_url, link.attributes['href'] )
-      end
-
-      XPath.each(doc, "//img") do |img|
-        img.attributes['src'] = fix_url( base_url, img.attributes['src'] )
-      end
+        doc.each_node do |elem|
+          if elem.is_a?(Oga::XML::Element) && elem.html?
+            case elem.name
+            when 'a'
+              elem.set 'href', fix_url(base_url, elem.get('href')) if elem.get('href')
+            when 'link'
+              elem.set 'href', fix_url(base_url, elem.get('href')) if elem.get('href')
+            when 'img'
+              elem.set 'src', fix_url(base_url, elem.get('src')) if elem.get('src')
+            end
+          end
+        end
 
-      if RUBY_VERSION.start_with? '1.8'
-        doc.to_s
-      else
-        doc.to_s.tap do |d| 
-          d.force_encoding(text.encoding) if d.encoding != text.encoding 
-        end 
+        doc.to_xml.tap do |d|
+          d.force_encoding(text.encoding) if d.encoding != text.encoding
+        end
+      rescue => e
+        Awestruct::ExceptionHelper.log_error e
+        $LOG.info %Q(If the error has to do with 'end of input' ensure none of the following tags have a closing tag:
+#{Oga::XML::HTML_VOID_ELEMENTS.to_a.collect {|a| a.downcase}.uniq.join(', ')}) if $LOG.info?
+        $LOG.warn "Text being parsed:\n#{text}" if $LOG.warn?
+        text # returning the bad text, which hopefully will help find the cause
       end
     end
 

data/lib/awestruct/deploy/s3_deploy.rb

@@ -1,5 +1,4 @@
 require 'awestruct/deploy/base_deploy'
-require 'ruby-s3cmd'
 
 module Awestruct
   module Deploy
@@ -7,14 +6,53 @@ module Awestruct
       def initialize( site_config, deploy_config )
         super
         @bucket = deploy_config['bucket']
+        @metadata = deploy_config['metadata']
       end
 
       def publish_site
         $LOG.info "Syncing #{@site_path} to bucket #{@bucket}" if $LOG.info?
-        s3cmd = RubyS3Cmd::S3Cmd.new
-        s3cmd.sync("#{@site_path}/", @bucket)
+        if @metadata and !@metadata.empty?
+          @metadata.each do |fileType, headers|
+            # Build the add-header command because the s3cmd-dsl gem doesn't support multi-headers
+            headerCmd = ""
+            if headers && !headers.empty?
+              headers.each do |key, value|
+                headerCmd << add_header(key, value)
+              end
+            end
+            # If gzip is enabled, add 'Content-Encoding: gzip' on js, css and html files
+            if @gzip and ['js', 'css', 'html'].include? fileType
+              headerCmd << add_header("Content-Encoding", "gzip")
+            end
+            # Sync files of current type with specified headers
+            s3_sync(@site_path, @bucket, "*", "*.#{fileType}", headerCmd)
+          end
+        end
+        # If gzip is enabled, add 'Content-Encoding: gzip' on not processed js, css and html files
+        if @gzip
+          remainingFileType = ['js', 'css', 'html'].find_all { |fileType| !@metadata.keys.include? fileType }
+          remainingFileType.each do |fileType|
+            headerCmd = add_header("Content-Encoding", "gzip")
+            s3_sync(@site_path, @bucket, "*", "*.#{fileType}", headerCmd)
+          end
+        end
+        # Finally, sync others files
+        s3_sync(@site_path, @bucket)
         $LOG.info "DONE" if $LOG.info?
       end
+
+      def add_header(key, value)
+        " --add-header '#{key}:#{value}'"
+      end
+
+      def s3_sync(site_path, bucket, exclude = nil, include = nil, headersCmd = nil)
+        cmd="s3cmd sync '#{site_path}' '#{bucket}'"
+        cmd << " --exclude '#{exclude}'" if exclude
+        cmd << " --include '#{include}'" if include
+        cmd << " #{headersCmd}" if headersCmd
+        $LOG.info "Execute #{cmd}"
+        `#{cmd}`
+      end
     end
   end
 end

data/lib/awestruct/engine.rb

@@ -13,6 +13,7 @@ require 'awestruct/extensions/pipeline'
 require 'fileutils'
 require 'set'
 require 'date'
+require 'erb'
 
 require 'compass'
 require 'parallel'
@@ -52,7 +53,7 @@ module Awestruct
       site.config
     end
 
-    def run(profile, base_url, default_base_url, force=false)
+    def run(profile, base_url, default_base_url, force=false, generate=true)
       $LOG.debug 'adjust_load_path' if $LOG.debug?
       adjust_load_path
       $LOG.debug 'load_default_site_yaml' if $LOG.debug?
@@ -69,27 +70,32 @@ module Awestruct
       load_pages
       $LOG.debug 'execute_pipeline' if $LOG.debug?
       $LOG.info 'Excecuting pipeline...' if $LOG.info?
-      execute_pipeline
+      execute_pipeline(false)
       $LOG.debug 'configure_compass' if $LOG.debug?
       configure_compass
       $LOG.debug 'set_urls' if $LOG.debug?
       set_urls( site.pages )
       $LOG.debug 'build_page_index' if $LOG.debug?
       build_page_index
-      $LOG.debug 'generate_output' if $LOG.debug?
-      $LOG.info 'Generating pages...' if $LOG.info?
-      generate_output
-      return 0
+
+      if ( generate )
+        $LOG.debug 'generate_output' if $LOG.debug?
+        $LOG.info 'Generating pages...' if $LOG.info?
+        generate_output
+      end
+      return Awestruct::ExceptionHelper::EXITCODES[:success]
     end
 
     def build_page_index
       site.pages_by_relative_source_path = {}
+      site.pages_by_output_path = {}
       site.pages.each do |p|
         # Add the layout to the set of dependencies
         p.dependencies.add_dependency(site.layouts.find_matching(p.layout, p.output_extension))
         if ( p.relative_source_path )
           site.pages_by_relative_source_path[ p.relative_source_path ] = p
         end
+        site.pages_by_output_path[ p.output_path ] = p
       end
       site.layouts.each do |p|
         # Add the layout to the set of dependencies
@@ -137,7 +143,7 @@ module Awestruct
     def load_site_yaml(yaml_path, profile = nil)
       if ( File.exist?( yaml_path ) )
         begin
-          data = YAML.load( File.read( yaml_path, :encoding => 'bom|utf-8' ) )
+          data = YAML.load( ERB.new(File.read( yaml_path, :encoding => 'bom|utf-8' )).result )
           if ( profile )
             # JP: Interpolation now turned off by default, turn it per page if needed
             site.interpolate = false
@@ -167,7 +173,7 @@ module Awestruct
 
     def load_yaml(yaml_path)
       begin
-        data = YAML.load( File.read( yaml_path ) )
+        data = YAML.load( ERB.new(File.read( yaml_path )).result )
       rescue Exception => e
         ExceptionHelper.log_building_error e, yaml_path
         ExceptionHelper.mark_failed
@@ -241,22 +247,31 @@ module Awestruct
       pipeline_file = File.join( ext_dir, 'pipeline.rb' )
       if ( File.exists?( pipeline_file ) )
         p = eval(File.read( pipeline_file ), nil, pipeline_file, 1)
+        p.before_all_extensions.each do |e|
+          pipeline.add_before_extension( e )
+        end
         p.extensions.each do |e|
           pipeline.extension( e )
         end
+        p.after_all_extensions.each do |e|
+          pipeline.add_after_extension( e )
+        end
         p.helpers.each do |h|
           pipeline.helper( h )
         end
         p.transformers.each do |t|
           pipeline.transformer( t )
         end
+        p.after_generation_extensions.each do |e|
+          pipeline.add_after_generation_extension( e )
+        end
       end
     end
 
-    def execute_pipeline
+    def execute_pipeline(on_reload = false)
       FileUtils.mkdir_p( site.config.output_dir )
       FileUtils.mkdir_p( site.config.tmp_dir )
-      pipeline.execute( site )
+      pipeline.execute( site, on_reload )
     end
 
     def configure_compass 
@@ -343,27 +358,36 @@ module Awestruct
 
     def generate_output
       FileUtils.mkdir_p( site.config.output_dir )
-      Parallel.each(@site.pages, :in_processes => Parallel.processor_count) do |page|
-        generated_path = File.join( site.config.output_dir, page.output_path )
-        if ( page.stale_output?( generated_path ) )
-          generate_page( page, generated_path )
-        else
-          generate_page( page, generated_path, false )
+      return_value = [Awestruct::ExceptionHelper::EXITCODES[:success]]
+      begin
+        return_value = Parallel.map(@site.pages, site.generation) do |page|
+          generate_page( page )
         end
+      rescue Exception => e
+        return_value = [Awestruct::ExceptionHelper::EXITCODES[:generation_error]]
+      end
+
+      if return_value.nil? || return_value.include?(Awestruct::ExceptionHelper::EXITCODES[:generation_error])
+        $LOG.error 'An error occurred during output generation, all pages may not have completed during generation'
+        exit Awestruct::ExceptionHelper::EXITCODES[:generation_error]
       end
+      site.engine.pipeline.execute_after_generation(site)
     end
 
-    def generate_page(page, generated_path, produce_output=true)
+    def generate_page(page, produce_output=true)
       if ( produce_output )
         $LOG.debug "Generating: #{generated_path}" if $LOG.debug? && config.verbose
-        FileUtils.mkdir_p( File.dirname( generated_path ) )
 
         c = page.rendered_content
         c = site.engine.pipeline.apply_transformers( site, page, c )
 
+        generated_path = File.join( site.config.output_dir, page.output_path )
+        FileUtils.mkdir_p( File.dirname( generated_path ) )
+
         File.open( generated_path, 'wb' ) do |file|
           file << c
         end
+        return Awestruct::ExceptionHelper::EXITCODES[:generation_error] if c.include? 'Backtrace:'
       elsif ( site.config.track_dependencies )
         if page.dependencies.load!
           $LOG.debug "Cached:     #{generated_path}" if $LOG.debug?
@@ -371,6 +395,7 @@ module Awestruct
           $LOG.debug "Analyzing:  #{generated_path}" if $LOG.debug?
           page.rendered_content
         end
+        return Awestruct::ExceptionHelper::EXITCODES[:success]
       end
     end
 
@@ -395,6 +420,19 @@ module Awestruct
         generate_page_internal(page)
       end
 
+      regen_pages = page_dependencies( page )
+
+      $LOG.debug "Starting regeneration of content dependent pages:" if regen_pages.size > 0 && $LOG.debug?
+
+      regen_pages.each do |p|
+        $LOG.info "Regenerating page #{p.output_path}" if $LOG.info?
+        generate_page_internal(p)
+      end
+
+      regen_pages
+    end
+
+    def page_dependencies(page)
       regen_pages = Set.new [ page ] 
       regen_pages.merge page.dependencies.dependents
 
@@ -411,35 +449,28 @@ module Awestruct
       end
 
       regen_pages.merge temp_set
-
-      $LOG.debug "Starting regeneration of content dependent pages:" if regen_pages.size > 0 && $LOG.debug?
-
-      regen_pages.each do |p|
-        $LOG.info "Regenerating page #{p.output_path}" if $LOG.info?
-        generate_page_internal(p)
-      end
-
-      regen_pages
     end
 
-    def run_auto_for_non_page(file)
+    def run_auto_for_non_page(file, generate = true)
       if File.extname(file) == '.rb'
         load file
       end
       @pipeline = Pipeline.new
       load_yamls
       load_pipeline
-      execute_pipeline
-      site.pages.each do |p|
-        generate_page_internal(p)
+      execute_pipeline(true)
+
+      if ( generate )
+        site.pages.each do |p|
+          generate_page_internal(p)
+        end
       end
       site.pages
     end
 
     def generate_page_internal(p)
       unless ( p.output_path.nil? || p.__is_layout || !p.stale_output?(p.output_path) )
-        generated_path = File.join( site.config.output_dir, p.output_path )
-        generate_page( p, generated_path )
+        generate_page( p )
       end
     end
 

data/lib/awestruct/extensions/disqus.rb

@@ -24,7 +24,7 @@ module Awestruct
             var disqus_identifier = #{identifier};
             (function() {
               var dsq = document.createElement("script"); dsq.type = "text/javascript"; dsq.async = true;
-              dsq.src = "http://#{site.disqus}.disqus.com/embed.js";
+              dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
               (document.getElementsByTagName("head")[0] || document.getElementsByTagName("body")[0]).appendChild(dsq);
             })();
             </script>
@@ -46,7 +46,7 @@ module Awestruct
             var disqus_shortname = '#{site.disqus}';
             (function () {
               var s = document.createElement('script'); s.async = true;
-              s.src = "http://disqus.com/forums/#{site.disqus}/count.js";
+              s.src = '//' + disqus_shortname + '.disqus.com/count.js';
               (document.getElementsByTagName('HEAD')[0] || document.getElementsByTagName('BODY')[0]).appendChild(s);
             }());
             </script>

data/lib/awestruct/extensions/flattr.rb

@@ -12,7 +12,7 @@ module Awestruct
           html += %Q|var s = document.createElement('script'), t = document.getElementsByTagName('script')[0];|
           html += %Q|s.type = 'text/javascript';\n|
           html += %Q|s.async = true;\n|
-          html += %Q|s.src = 'http://api.flattr.com/js/0.6/load.js?mode=auto&uid=#{site.flattr_username}&category=text';\n|
+          html += %Q|s.src = '//api.flattr.com/js/0.6/load.js?mode=auto&uid=#{site.flattr_username}&category=text';\n|
           html += %Q|t.parentNode.insertBefore(s, t);\n|
           html += %Q|})(); /* ]]> */ </script>|
           html

data/lib/awestruct/extensions/pipeline.rb

@@ -11,29 +11,47 @@ end
 
 module Awestruct
   module Extensions
+    # Public. Extension declaration class, initialized by the end user to
+    # declare their extensions, helpers, transformers, etc.
     class Pipeline
 
-      attr_reader :before_extensions
+      attr_reader :before_all_extensions
       attr_reader :extensions
-      attr_reader :after_extensions
+      attr_reader :after_all_extensions
       attr_reader :helpers
       attr_reader :transformers
+      attr_reader :after_generation_extensions
 
       def initialize(&block)
-        @extensions   = []
-        @helpers      = []
-        @transformers = []
+        @before_all_extensions       = []
+        @extensions                  = []
+        @helpers                     = []
+        @transformers                = []
+        @after_all_extensions        = []
+        @after_generation_extensions = []
         begin
-          instance_eval &block if block
+          instance_eval(&block) if block
         rescue Exception => e
           abort("Failed to initialize pipeline: #{e}")
         end
       end
 
+      def before_extensions(ext)
+        @before_all_extensions << ext
+      end
+
       def extension(ext)
-        @extensions << ext
+        @extensions << ext if ext.respond_to?(:execute)
         # TC: why? transformer and extension?
-        ext.transform(@transformers) if ext.respond_to?('transform')
+        transformer(ext) if ext.respond_to?(:transform)
+
+        @before_all_extensions << ext if ext.respond_to?(:before_extensions)
+        @after_all_extensions << ext if ext.respond_to?(:after_extensions)
+        @after_generation_extensions << ext if ext.respond_to?(:after_generation)
+      end
+
+      def after_extensions(ext)
+        @after_all_extensions << ext
       end
 
       def helper(helper)
@@ -44,6 +62,10 @@ module Awestruct
         @transformers << transformer
       end
 
+      def after_generation(ext)
+        @after_generation_extensions << ext
+      end
+
       def execute(site)
         extensions.each do |ext|
           ext.execute( site )

data/lib/awestruct/extensions/relative.rb

@@ -11,7 +11,9 @@ module Awestruct
           if href.start_with?("http://") || href.start_with?("https://")
             result = href
           else
-            result = Pathname.new(href).relative_path_from(Pathname.new(File.dirname(p.output_path))).to_s
+            pathname = Pathname.new(href).relative_path_from(Pathname.new(File.dirname(p.output_path)))
+            result = pathname.to_s
+            result << '/' if pathname.extname.empty?
           end
           result
         rescue Exception => e