terrimporter 0.6.4 → 0.7.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    terrimporter (0.6.4)
+    terrimporter (0.6.6)
       kwalify (>= 0.7.2)
 
 GEM

data/config/schema.yml

@@ -29,7 +29,7 @@ mapping:
     type: map
     required: yes
     mapping:
-      "relative_destination_path":
+      "destination_path":
         type: str
         required: yes
       "styles":
@@ -51,13 +51,13 @@ mapping:
     type: map
     required: yes
     mapping:
-      "relative_destination_path":
+      "destination_path":
         type: str
         required: yes
       "dynamic_libraries":
         type: str
         required: no
-      "libraries_relative_destination_path":
+      "libraries_destination_path":
         type: str
         required: no
   "images":
@@ -69,9 +69,32 @@ mapping:
           "server_path":
             type: str
             required: yes
-          "relative_destination_path":
+          "destination_path":
             type: str
             required: yes
           "file_types":
             type: str
-            required: no
+            required: no
+  "modules":
+    type: seq
+    required: no
+    sequence:
+      - type: map
+        mapping:
+          "name":
+            type: str
+            required: yes
+          "skin":
+            type: str
+            required: no
+            default: ""
+          "destination_path":
+            type: str
+            required: yes
+          "module_template":
+            type: str
+            required: yes
+          "template_only":
+            type: bool
+            default: false
+            required: no

data/config/{terrimporter.config.yml → terrimporter.yml}

@@ -1,13 +1,23 @@
 #-------------------------------------------------------------------------------
-# terrimporter configuration, adjust as needed. Don't mess with indentation!
+# terrimporter configuration, adjust as needed.
 #-------------------------------------------------------------------------------
+# Important notes READ THEM:
+# * Don't mess with indentation!
+# * All local paths are relative to the running directory
+# * multiple entries have the following format (stylesheet replacement example):
+# - what: /replacethis/
+#   with: /withthis/
+# - what: /another/
+#   with: /replacement/
+
+# APPLICATION
 # url: Terrific base url
 application_url: http://terrific.project.url
 
 # STYLESHEETS
 stylesheets:
 # The destination directory, relative from the current running directory
-  relative_destination_path: public/stylesheets/
+  destination_path: public/stylesheets/
 # Additional stylesheets, you can but don't have to add the .css extension
 #  styles: ie, ie6, ie7.css
 # replace_strings: Define string replacements in css files 
@@ -19,18 +29,26 @@ stylesheets:
 # JAVASCRIPTS
 javascripts:
 # Destination path of the javascript base.js file
-  relative_destination_path: public/javascripts/
+  destination_path: public/javascripts/
 # Dynamic libraries definition, you can but don't have to add the .js extension
 #  dynamic_libraries: foo_lib.js bar_lib.js
-# Destination path of the dynamic libraries relative 
-#  libraries_relative_destination_path:  public/javascripts/libraries/
+# Destination path of the dynamic libraries (relative)
+#  libraries_destination_path:  public/javascripts/libraries/
 
 # IMAGES - multiple entries allowed
 images:
 #  - server_path: / # The server path is appended to the image_server_path
-#    relative_destination_path: public/images/  # The relative destination path
+#    destination_path: public/images/  # The relative destination path
 #    file_types: jpg, png, gif  # Image file endings to be downloaded
 
+# MODULES - multiple entries allowed
+modules:
+#  - name:                 #Module name
+#    skin:                 #Skin name, if used
+#    module_template:      #The module template used, required
+#    template_only: false  #Only import template without surrounding divs, default: false
+#    destination_path: #Destination path of
+
 #-------------------------------------------------------------------------------
 # ADVANCED EDITING - Only edit the following if you know what you're doing...
 # ------------------------------------------------------------------------------

data/lib/terrimporter/app_logger.rb

@@ -1,43 +1,52 @@
-module Logging
-  require 'logger'
-
-  #logformatter monkeypatch
-  class LogFormatter < Logger::Formatter
-    CUSTOM_FORMAT = "[%s] %5s %s: %s\n"
-
-    def call(severity, time, progname, message)
-      CUSTOM_FORMAT % [format_datetime(time), severity, progname, msg2str(message)]
-    end
-
-    def format_datetime(time)
-      time.strftime("%Y-%m-%d %H:%M:%S")
-    end
+class Logger
+  attr_accessor :level
+
+  LOG_LEVELS = {:debug => 0, :info => 1, :warn => 2, :error=> 3, :fatal => 4}
+  LOG_COLORS = {:debug =>'33', :info =>'32', :warn =>'33', :error=>'31', :fatal =>'31'}
+
+  # more infos: https://wiki.archlinux.org/index.php/Color_Bash_Prompt
+  #\033[0m      Text reset
+  #\033[0;37m   White
+  #\033[032m    Green
+  #\033[033m    Yellow
+  #\033[031m    Red
+  #\033[037m    White
+
+  # %s => [datetime], %s => color, %-5s => severity, %s => message
+  LOG_FORMAT = "\033[0;37m %s \033[0m[\033[%sm%-5s\033[0m]: %s \n"
+  TIME_FORMAT = "%H:%M:%S"
+
+  def initialize
+    self.level = :debug
   end
 
-
-# @param hash [:info => "message", :debug => "message", :error => "message"]
-  def log(hash)
-    Logging.log.debug hash[:debug] unless hash[:debug].nil?
-    Logging.log.info hash[:info] unless hash[:info].nil?
-    Logging.log.warn hash[:warn] unless hash[:warn].nil?
-    Logging.log.error hash[:error] unless hash[:error].nil?
-    Logging.log.fatal hash[:fatal] unless hash[:fatal].nil?
+  def error(message)
+    log(:error, message)
   end
 
-  def self.log
-    @logger ||= Logger.new $stdout
-    @logger.formatter = LogFormatter.new
-    @logger
+  def info(message)
+    log(:info, message)
   end
 
-  def verbose?
-    !self.options.nil? && self.options[:verbose] = true
+  def debug(message)
+    log(:debug, message)
   end
 
-  def self.initialize_logger
+  def log(severity, message)
+    return if LOG_LEVELS[severity] < LOG_LEVELS[self.level]
 
+    color = LOG_COLORS[severity]
+    if LOG_LEVELS[severity] >= LOG_LEVELS[:error]
+      $stderr.puts(LOG_FORMAT % [format_datetime(Time.now), color, severity.to_s.upcase, message])
+    else
+      $stdout.puts(LOG_FORMAT % [format_datetime(Time.now), color, severity.to_s.upcase, message])
+    end
   end
 
-end
+  def format_datetime(time)
+    time.strftime(TIME_FORMAT)
+  end
 
+end
 
+LOG = Logger.new

data/lib/terrimporter/configuration.rb

@@ -4,7 +4,7 @@
 module TerrImporter
   class Application
     class Configuration < Hash
-      include ConfigHelper
+      include ConfigurationHelper
 
       attr_accessor :validations, :config_file
 
@@ -15,7 +15,7 @@ module TerrImporter
 
       def load_configuration
         config_file_path = determine_config_file_path
-        puts "Configuration file located, load from #{config_file_path}"
+        LOG.debug "Configuration file located, load from #{config_file_path}"
         validate_and_load_config(config_file_path)
       end
 
@@ -42,9 +42,8 @@ module TerrImporter
         ]
       end
 
-      #todo split!
       def validate_and_load_config(file)
-        puts "Validating configuration..."
+        LOG.debug "Validating configuration..."
 
         parser = Kwalify::Yaml::Parser.new(load_validator)
         document = parser.parse_file(file)
@@ -59,12 +58,12 @@ module TerrImporter
       end
 
       def load_validator
-        puts "Loading validator from #{schema_file_path}"
+        LOG.debug "Loading validator from #{schema_file_path}"
         schema = Kwalify::Yaml.load_file(schema_file_path)
         Kwalify::Validator.new(schema)
       end
 
-      def required_present?
+      def mandatory_present?
         if self['export_path'].nil? or self['export_settings']['application'].nil? or self['application_url'].nil?
           false
         else
@@ -72,12 +71,38 @@ module TerrImporter
         end
       end
 
+      def determine_configuration_values_from_html(raw_html)
+        css_result, js_result = raw_html.scan(/(\/terrific\/base\/(.*?)\/public\/.*base.(css|js).php)\?.*application=(.*?)(&amp;|&)/)
+
+        raise ConfigurationError, "Unable to extract css information from application url, content is: #{raw_html}" if css_result.nil? or css_result.size < 5
+        raise ConfigurationError, "Unable to extract javascript information from application url, content is: #{raw_html}" if js_result.nil? or js_result.size < 5
+
+        css_export_path = css_result[0]
+        js_export_path = js_result[0]
+        terrific_version = css_result[1]
+        application = css_result[3]
+
+        raise ConfigurationError, "Unable to determine css export path" if css_export_path.nil?
+        raise ConfigurationError, "Unable to determine js export path " if js_export_path.nil?
+        raise ConfigurationError, "Unable to determine terrific version" if terrific_version.nil?
+        raise ConfigurationError, "Unable to determine application path" if application.nil?
+
+        LOG.info "Determined the following configuration values from #{self['application_url']}:\n" +
+                     "terrific version: #{terrific_version} \n" +
+                     "application path: #{application}"
+
+        self['version'] = terrific_version
+        self['export_settings'] ||= {}
+        self['export_settings']['application'] = application
+        self['export_path'] = {'css' => css_export_path, 'js' => js_export_path}
+      end
+
       def stylesheets
         stylesheets = ["base.css"]
         if additional_stylesheets?
           stylesheets = stylesheets + self['stylesheets']['styles'].to_s.robust_split
         else
-          puts "No additional stylesheets defined."
+          LOG.debug "No additional stylesheets defined."
         end
         stylesheets.add_if_missing!('.css')
       end
@@ -92,10 +117,10 @@ module TerrImporter
       end
 
       def libraries_destination_path
-        if !self['javascripts']['libraries_relative_destination_path'].nil?
-          File.join(self['javascripts']['libraries_relative_destination_path'])
+        if !self['javascripts']['libraries_destination_path'].nil?
+          File.join(self['javascripts']['libraries_destination_path'])
         else
-          File.join(self['javascripts']['relative_destination_path'])
+          File.join(self['javascripts']['destination_path'])
         end
       end
 
@@ -111,6 +136,10 @@ module TerrImporter
         !self['images'].nil?
       end
 
+      def modules?
+        !self['modules'].nil?
+      end
+
     end
   end
 end

data/lib/terrimporter/{config_helper.rb → configuration_helper.rb}

@@ -1,7 +1,7 @@
-module ConfigHelper
+module ConfigurationHelper
 
   def config_default_name
-    'terrimporter.config.yml'
+    'terrimporter.yml'
   end
 
   def schema_default_name
@@ -25,13 +25,13 @@ module ConfigHelper
   end
 
   def create_config_file(backup_or_replace = nil, application_url = nil)
-    puts "Creating configuration file..."
+    LOG.info "Creating configuration file..."
     case backup_or_replace
       when :backup
-        puts "Backing up old configuration file to #{config_working_directory_path}.bak"
+        LOG.debug "Backing up old configuration file to #{config_working_directory_path}.bak"
         FileUtils.mv(config_working_directory_path, config_working_directory_path + '.bak')
       when :replace
-        puts "Replacing old configuration file"
+        LOG.debug "Replacing old configuration file"
         FileUtils.rm_f(config_working_directory_path) if File.exists? config_working_directory_path
     end
     FileUtils.cp(config_example_path, config_working_directory_path)
@@ -42,7 +42,7 @@ module ConfigHelper
       File.open(config_working_directory_path, 'w') { |f| f.write(configuration) }
     end
 
-    puts "done! You should take a look an edit it to your needs..."
+    LOG.info "done! You should take a look an edit it to your needs..."
   end
 
   private

data/lib/terrimporter/download_helper.rb

@@ -0,0 +1,8 @@
+module DownloadHelper
+  def create_dir_path(dir)
+    unless File.directory?(dir) and File.file?(dir)
+      LOG.debug "Creating directory: #{dir}"
+      FileUtils.mkpath(dir)
+    end
+  end
+end

data/lib/terrimporter/downloader.rb

@@ -4,39 +4,66 @@ require 'uri'
 module TerrImporter
   class Application
     class Downloader
+      include DownloadHelper
 
       def initialize(base_uri)
         @base_uri = base_uri
-        puts "Downloader initialized to uri: #{base_uri}"
+        LOG.debug "Downloader initialized to uri: #{@base_uri}"
       end
 
-      def download(remote_path, local_path=nil)
-        absolute_uri = absolute_path(remote_path)
+      def download(remote_path, local_path = nil)
+        remote_url = url(remote_path)
         begin
           if local_path.nil? #download to buffer
+            LOG.debug "Download #{remote_url} to buffer"
             data = StringIO.new
-
-            puts "Downloading #{absolute_uri} to buffer"
-
-            absolute_uri.open { |io| data = io.read }
+            remote_url.open { |io| data = io.read }
             data.to_s
           else
-            puts "Downloading #{absolute_uri} to local path #{local_path}"
-
-            open(local_path, "wb") { |file|
-              file.write(absolute_uri.open.read)
-            }
+            LOG.info "Download #{remote_url} to local path #{local_path}"
+            create_dir_path File.dirname(local_path)
+            open(local_path, "wb") { |file| file.write(remote_url.open.read) }
           end
         rescue SocketError => e
-          raise DefaultError, "Error opening url #{absolute_uri}: \n #{e.message}"
+          raise DefaultError, "Error opening url #{remote_url}: \n #{e.message}"
+        end
+      end
+
+      def batch_download(remote_path, local_path, type_filter = "")
+        source_path = url(remote_path)
+        create_dir_path local_path
+        LOG.debug "Download multiple files from #{source_path} to #{local_path} #{"allowed extensions: " + type_filter unless type_filter.empty?}"
+
+        files = html_directory_list(source_path)
+
+        unless type_filter.empty?
+          LOG.debug "Apply type filter: #{type_filter}"
+          files = files.find_all { |file| file =~ Regexp.new(".*" + type_filter.robust_split.join("|") + "$") }
+        end
+
+        LOG.info "Download #{files.size} files..."
+        files.each do |file|
+          local_file_path = File.join(local_path.to_s, file)
+          self.download(File.join(source_path.to_s, file), local_file_path)
         end
       end
 
       private
-      def absolute_path(relative_path)
-        URI.join(@base_uri, relative_path)
+      def html_directory_list(remote_path)
+        LOG.debug "Get html directory list"
+        output = self.download(remote_path)
+        files = []
+
+        output.scan(/<a\shref=\"([^\"]+)\"/) do |res|
+          files << res[0] if not res[0] =~ /^\?/ and not res[0] =~ /.*\/$/ and res[0].size > 1
+        end
+        LOG.debug "Found #{files.size} files"
+        files
       end
 
+      def url(relative_path)
+        URI.join(@base_uri, relative_path)
+      end
     end
   end
 end