terrimporter 0.6.0 → 0.6.4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    terrimporter (0.5.4)
+    terrimporter (0.6.4)
       kwalify (>= 0.7.2)
 
 GEM

data/README.rdoc

@@ -18,10 +18,14 @@ terrimporter has the following features:
 
 === Gem Installation
 
+You need sudo to install terrimporter if you're not using a ruby version manager like rvm for example.
+It is strongly encouraged to use such a ruby version management tool in order to install the gem without sudo rights.
 Download and install terrimporter with the following.
 
    gem install terrimporter
 
+
+
 == Usage
 
 === Simple Example

data/Rakefile

@@ -57,7 +57,7 @@ namespace :version do
       when :major
         version[:major] = version[:major].to_i + 1
       when :minor
-        version[:minor] = version[:minor].to_1 + 1
+        version[:minor] = version[:minor].to_i + 1
       when :patch
         version[:patch] = version[:patch].to_i + 1
     end
@@ -91,8 +91,6 @@ end
 }
     File.open(version_file_path, 'w') { |f| f.write(version_rb) }
   end
-
-
 end
 
 

data/config/schema.yml

@@ -1,30 +1,20 @@
 type: map
 mapping:
-  "version":
-    type: text
-    required: yes
-  "url":
-    type: str
-    required: yes
-    pattern: /^(http|https):\/\/[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$/
-  "app_path":
-    type: str
-    required: yes
-    pattern:  /\/.*/
-  "export_path":
+  "application_url":
     type: str
     required: yes
-    pattern: /^\/terrific.*\%2\$s.*\%1\$s.*\%1\$s.*\.php$/
-  "libraries_source_path":
+    pattern: /^(http|https):\/\/.*$/
+  "libraries_server_path":
     type: str
     required: yes
     pattern:  /\/.*/
-  "image_base_path":
+  "image_server_path":
       type: str
       required: yes
       pattern:  /\/.*/
   "export_settings":
     type: map
+    required: yes
     mapping:
       "layout":
         type: str
@@ -37,15 +27,15 @@ mapping:
         default: false
   "stylesheets":
     type: map
+    required: yes
     mapping:
-      "dest":
+      "relative_destination_path":
         type: str
         required: yes
       "styles":
          type: str
-         name: css
          required: no
-      "replace":
+      "replace_strings":
         type: seq
         required: no
         sequence:
@@ -59,28 +49,29 @@ mapping:
                 required: yes
   "javascripts":
     type: map
+    required: yes
     mapping:
-      "dest":
+      "relative_destination_path":
         type: str
         required: yes
       "dynamic_libraries":
         type: str
-        name: js
         required: no
-      "libraries_dest":
+      "libraries_relative_destination_path":
         type: str
-        required: yes
+        required: no
   "images":
     type: seq
+    required: no
     sequence:
       - type: map
         mapping:
-          "src":
+          "server_path":
             type: str
             required: yes
-          "dest":
+          "relative_destination_path":
             type: str
             required: yes
-          "types":
+          "file_types":
             type: str
             required: no

data/config/terrimporter.config.yml

@@ -1,66 +1,47 @@
-########################################################################################################################
-# Terrific import configuration file, adjust as needed.
-#
-# version:          Define the terrific version in use
-# url:              Terrific base url
-# app_path          Terrific application path (must start with '/')
-# export_path       You most probably won't have to change this
-# image_base_path:  The Terrific image base path. You most probably won't have to change this
-# libraries_source_path:  The library source path. You most probably won't have to change this
-#
-# export_settings:  The settings below are added as query parameters for base.js and base.css
-#   layout:         You most probably won't have to change this
-#   debug:          False compacts the base.js and base.css files
-#   cache:          You most probably won't have to change this
-#
-# stylesheets:      Define the stylesheets in the following form:
-#   dest:           The destination directory, relative from the current running directory
-#   styles:         Name of the stylesheets, space separated without the .css extension. ex: ie ie6 ie7 ie8
-#                   the main style base.css is included by default and MUST NOT BE SPECIFIED
-#   replace:        Define path replacements after the stylesheets have been downloaded
-#                   you can define multiple replacements by repeating the '-what: with:' statements
-#     - what:       What to replace, default is string. if you want to use regex, use the format r/[regex]/
-#                   for help on regular expressions see http://www.rubular.com/ (an online editor/tester)
-#       with:       Replace with this (string)
-#
-# javascripts:      Define the javascript files in the following form
-#   dest:           Destination of the javascript base.js file
-#   dynamic_libraries: Dynamic libraries definition, space separated without the .js extension. ex: warning pngfix
-#   libraries_dest: Destination of the dynamic libraries relative to the 'dest' path. '/' means they're included in 'dest'
-#
-# images:           Define the images to be downloaded, the terrific base path is always "img", so the src root is "img/"
-#                   each of the entries describes one path for images to be downloaded
-#   -src:           The source path, "/" means "img/"
-#   dest:           The destination directory, relative from the current running directory
-#   types:          The image endings to be downloaded, specified as a space separated string
-########################################################################################################################
-    version:      0.5
-    url:          http://terrific.url
-    app_path:     /terrific/webapp/path
-    export_path:  /terrific/base/%2$s/public/%1$s/base/base.%1$s.php
-    libraries_source_path: /js/libraries/dynamic
-    image_base_path: /img
-    export_settings:
-      layout: project
-      debug: false
-      cache: false
-    stylesheets:
-      dest: public/stylesheets/
-      styles: ie
-      replace:
-        - what: /img/
-          with: /images/
-    javascripts:
-      dest: public/javascripts/
-      dynamic_libraries: dynlib
-      libraries_dest: /
-    images:
-      - src: /
-        dest: public/images/
-        types: jpg png gif
-      - src: /
-        dest: public/
-        types: ico
-      - src: /backgrounds/
-        dest: public/images/backgrounds/
-        types: jpg
+#-------------------------------------------------------------------------------
+# terrimporter configuration, adjust as needed. Don't mess with indentation!
+#-------------------------------------------------------------------------------
+# url: Terrific base url
+application_url: http://terrific.project.url
+
+# STYLESHEETS
+stylesheets:
+# The destination directory, relative from the current running directory
+  relative_destination_path: public/stylesheets/
+# Additional stylesheets, you can but don't have to add the .css extension
+#  styles: ie, ie6, ie7.css
+# replace_strings: Define string replacements in css files 
+# (use multiple replacements by repeating the '-what: with:' statements)
+  replace_strings:
+#    - what: /img/     # What to replace, format: [string] or [r/regex/]
+#      with: /images/  # Replace with [string]
+
+# JAVASCRIPTS
+javascripts:
+# Destination path of the javascript base.js file
+  relative_destination_path: public/javascripts/
+# Dynamic libraries definition, you can but don't have to add the .js extension
+#  dynamic_libraries: foo_lib.js bar_lib.js
+# Destination path of the dynamic libraries relative 
+#  libraries_relative_destination_path:  public/javascripts/libraries/
+
+# IMAGES - multiple entries allowed
+images:
+#  - server_path: / # The server path is appended to the image_server_path
+#    relative_destination_path: public/images/  # The relative destination path
+#    file_types: jpg, png, gif  # Image file endings to be downloaded
+
+#-------------------------------------------------------------------------------
+# ADVANCED EDITING - Only edit the following if you know what you're doing...
+# ------------------------------------------------------------------------------
+# The settings below are added as query string parameters to the export path
+export_settings:
+  debug: false      # False compacts the base.js and base.css files
+  cache: false      # You most probably won't have to change this
+  layout: project   # You most probably won't have to change this
+
+# The server side library source path for the dynamic libraries
+libraries_server_path: /js/libraries/dynamic
+
+# The server side image base path
+image_server_path: /img

data/lib/terrimporter.rb

@@ -4,9 +4,10 @@ require 'terrimporter/app_logger'
 require 'terrimporter/options'
 require 'terrimporter/importer'
 require 'terrimporter/config_helper'
-require 'terrimporter/config_validator'
 require 'terrimporter/configuration'
 require 'terrimporter/downloader'
+require 'terrimporter/string_monkeypatch'
+require 'terrimporter/array_monkeypatch'
 require 'etc'
 require 'kwalify'
 require 'fileutils'
@@ -23,11 +24,11 @@ module TerrImporter
         options = build_options(arguments)
 
         begin
-          unless options[:init].nil?
+          if !options[:init].nil?
             if config_working_directory_exists? and options[:init] != :backup and options[:init] != :replace
               raise TerrImporter::ConfigurationError, "Configuration already exists, use the override or backup option"
             end
-            create_config_file
+            create_config_file(options[:init], options[:application_url])
             return 0
           end
 

data/lib/terrimporter/array_monkeypatch.rb

@@ -0,0 +1,11 @@
+class Array
+  def add_if_missing!(ending)
+    self.collect! do |item|
+      unless item =~ /#{ending}$/
+        item + ending
+      else
+        item
+      end
+    end
+  end
+end

data/lib/terrimporter/config_helper.rb

@@ -1,4 +1,3 @@
-
 module ConfigHelper
 
   def config_default_name
@@ -25,7 +24,7 @@ module ConfigHelper
     File.join(base_config_path, schema_default_name)
   end
 
-  def create_config_file(backup_or_replace = nil)
+  def create_config_file(backup_or_replace = nil, application_url = nil)
     puts "Creating configuration file..."
     case backup_or_replace
       when :backup
@@ -36,6 +35,13 @@ module ConfigHelper
         FileUtils.rm_f(config_working_directory_path) if File.exists? config_working_directory_path
     end
     FileUtils.cp(config_example_path, config_working_directory_path)
+
+    unless application_url.nil?
+      configuration = File.read(config_working_directory_path)
+      configuration.gsub!(/application_url:.*$/, "application_url: #{application_url}")
+      File.open(config_working_directory_path, 'w') { |f| f.write(configuration) }
+    end
+
     puts "done! You should take a look an edit it to your needs..."
   end
 

data/lib/terrimporter/configuration.rb

@@ -61,7 +61,54 @@ module TerrImporter
       def load_validator
         puts "Loading validator from #{schema_file_path}"
         schema = Kwalify::Yaml.load_file(schema_file_path)
-        ConfigValidator.new(schema)
+        Kwalify::Validator.new(schema)
+      end
+
+      def required_present?
+        if self['export_path'].nil? or self['export_settings']['application'].nil? or self['application_url'].nil?
+          false
+        else
+          true
+        end
+      end
+
+      def stylesheets
+        stylesheets = ["base.css"]
+        if additional_stylesheets?
+          stylesheets = stylesheets + self['stylesheets']['styles'].to_s.robust_split
+        else
+          puts "No additional stylesheets defined."
+        end
+        stylesheets.add_if_missing!('.css')
+      end
+
+      def dynamic_libraries
+        libraries = self['javascripts']['dynamic_libraries'].robust_split
+        libraries.add_if_missing!('.js')
+      end
+
+      def replace_style_strings?
+        !self['stylesheets'].nil? and !self['stylesheets']['replace_strings'].nil?
+      end
+
+      def libraries_destination_path
+        if !self['javascripts']['libraries_relative_destination_path'].nil?
+          File.join(self['javascripts']['libraries_relative_destination_path'])
+        else
+          File.join(self['javascripts']['relative_destination_path'])
+        end
+      end
+
+      def additional_stylesheets?
+        !self['stylesheets'].nil? and !self['stylesheets']['styles'].nil?
+      end
+
+      def additional_dynamic_javascripts?
+        !self['javascripts'].nil? and !self['javascripts']['dynamic_libraries'].nil?
+      end
+
+      def images?
+        !self['images'].nil?
       end
 
     end

data/lib/terrimporter/downloader.rb

@@ -12,19 +12,23 @@ module TerrImporter
 
       def download(remote_path, local_path=nil)
         absolute_uri = absolute_path(remote_path)
-        if local_path.nil? #download to buffer
-          data = StringIO.new
-
-          puts "Downloading #{absolute_uri} to buffer"
-
-          absolute_uri.open { |io| data = io.read }
-          data.to_s
-        else
-          puts "Downloading #{absolute_uri} to local path #{local_path}"
-
-          open(local_path, "wb") { |file|
-            file.write(absolute_uri.open.read)
-          }
+        begin
+          if local_path.nil? #download to buffer
+            data = StringIO.new
+
+            puts "Downloading #{absolute_uri} to buffer"
+
+            absolute_uri.open { |io| data = io.read }
+            data.to_s
+          else
+            puts "Downloading #{absolute_uri} to local path #{local_path}"
+
+            open(local_path, "wb") { |file|
+              file.write(absolute_uri.open.read)
+            }
+          end
+        rescue SocketError => e
+          raise DefaultError, "Error opening url #{absolute_uri}: \n #{e.message}"
         end
       end
 

data/lib/terrimporter/importer.rb

@@ -1,5 +1,3 @@
-
-
 module TerrImporter
 
   class DefaultError < StandardError
@@ -21,10 +19,12 @@ module TerrImporter
         self.options = options
         self.config = Configuration.new options[:config_file]
         self.config.load_configuration
-        @downloader = Downloader.new config['url']
+        @downloader = Downloader.new config['application_url']
       end
 
       def run
+
+
         if options[:all] != nil and options[:all] == true
           puts "Import of everything started"
           import_js
@@ -40,65 +40,116 @@ module TerrImporter
         end
       end
 
+      def determine_configuration_values_from_uri
+        result = @downloader.download('')
+        #result =~ /\/terrific\/base\/(.*?)\/public\/.*application=(.*?)(&amp;|&)/
+        #result =~ /(\/terrific\/base\/(.*?)\/public\/.*base.(css|js).php).*application=(.*?)(&amp;|&)/
+
+        css_result, js_result = result.scan(/(\/terrific\/base\/(.*?)\/public\/.*base.(css|js).php)\?.*application=(.*?)(&amp;|&)/)
+
+
+        if css_result.nil? or css_result.size < 5
+          raise ConfigurationError, "Unable to extract css information from application url, content is: #{result}"
+        end
+        if js_result.nil? or js_result.size < 5
+          raise ConfigurationError, "Unable to extract javascript information from application url, content is: #{result}"
+        end
+
+        css_export_path = css_result[0]
+        js_export_path = js_result[0]
+        terrific_version = css_result[1]
+        application = css_result[3]
+
+        raise ConfigurationError, "Unable to determine css export path from application url" if css_export_path.nil?
+        raise ConfigurationError, "Unable to determine js export path from application url" if js_export_path.nil?
+
+        puts "Determined the following configuration values from #{config['application_url']}:\n" +
+                 "terrific version: #{terrific_version} \n" +
+                 "application path: #{application}"
+
+        config['version'] = terrific_version
+        config['export_settings']['application'] = application
+        config['export_path'] = {'css' => css_export_path, 'js' => js_export_path}
+      end
+
       def import_css
+        check_and_complete_config!
         unclean_suffix = "_unclean"
 
-        check_and_create_dir config['stylesheets']['dest']
+        check_and_create_dir config['stylesheets']['relative_destination_path']
 
-        #create stylesheet array and add base.css
-        styles = config['stylesheets']['styles'].split(" ")
-        styles << "base"
+        styles = config.stylesheets
 
         styles.each do |css|
-          destination_path = File.join(config['stylesheets']['dest'], css + ".css")
+          relative_destination_path = File.join(config['stylesheets']['relative_destination_path'], css)
           options = {}
-          options[:suffix] = css if css.include?('ie') #add ie option if in array
+          options[:suffix] = $1 if css =~ /(ie.*).css$/ #add ie option if in array
 
           source_url = construct_export_path(:css, options)
 
-          @downloader.download(source_url, destination_path + unclean_suffix)
+          @downloader.download(source_url, relative_destination_path + unclean_suffix)
 
-          #do line replacement
-          puts "Start css line replacements"
-          File.open(destination_path, 'w') do |d|
-            File.open(destination_path + unclean_suffix, 'r') do |s|
-              lines = s.readlines
-              lines.each do |line|
-                d.print stylesheet_replace_strings!(line)
+          if config.replace_style_strings?
+            puts "Start css line replacements..."
+            File.open(relative_destination_path, 'w') do |d|
+              File.open(relative_destination_path + unclean_suffix, 'r') do |s|
+                lines = s.readlines
+                lines.each do |line|
+                  d.print stylesheet_replace_strings!(line)
+                end
               end
             end
+          else
+            puts "No css line replacements defined; skipping..."
           end
           puts "Deleting unclean css files"
-          FileUtils.remove destination_path + unclean_suffix
+          FileUtils.remove relative_destination_path + unclean_suffix
         end
       end
 
       def import_js
-        check_and_create_dir config['javascripts']['dest']
-        destination_path = File.join(config['javascripts']['dest'], "base.js")
+        check_and_complete_config!
+        check_and_create_dir config['javascripts']['relative_destination_path']
+        relative_destination_path = File.join(config['javascripts']['relative_destination_path'], "base.js")
         js_source_url = construct_export_path :js
 
-        puts "Importing base.js from #{js_source_url} to #{destination_path}"
+        puts "Importing base.js from #{js_source_url} to #{relative_destination_path}"
+
+        @downloader.download(js_source_url, relative_destination_path)
+
 
-        @downloader.download(js_source_url, destination_path)
+        if config.additional_dynamic_javascripts?
 
+          libraries_destination_path = config.libraries_destination_path
+          check_and_create_dir libraries_destination_path
+          js_libraries = config.dynamic_libraries
 
-        libraries_destination_path = File.join(config['javascripts']['dest'], config['javascripts']['libraries_dest'])
-        check_and_create_dir libraries_destination_path
-        js_libraries = config['javascripts']['dynamic_libraries'].split(" ")
+          puts "Importing libraries from #{config['libraries_server_path']} to #{libraries_destination_path}"
 
-        puts "Importing libraries from #{config['libraries_source_path']} to #{libraries_destination_path}"
+          if config['libraries_server_path'].nil?
+            puts "Please define 'libraries_server_path' in configuration file"
+          else
+            js_libraries.each do |lib|
+              @downloader.download(File.join(config['libraries_server_path'], lib), File.join(libraries_destination_path, lib))
+            end
+
+          end
 
-        js_libraries.each do |lib|
-          @downloader.download(File.join(config['libraries_source_path'], lib+ ".js"), File.join(libraries_destination_path, lib + ".js"))
         end
       end
 
       def import_images
-        config['images'].each do |image|
-          check_and_create_dir image['dest']
-          image_source_path = File.join(config['image_base_path'], image['src'])
-          batch_download(image_source_path, image['dest'], image['types'])
+        check_and_complete_config!
+        if config.images?
+          puts "Start importing images..."
+
+          config['images'].each do |image|
+            check_and_create_dir image['relative_destination_path']
+            image_source_path = File.join(config['image_server_path'], image['server_path'])
+            batch_download(image_source_path, image['relative_destination_path'], image['file_types'])
+          end
+        else
+          puts "No image configuration found, skipping image import..."
         end
       end
 
@@ -107,19 +158,19 @@ module TerrImporter
       def batch_download(relative_source_path, relative_dest_path, type_filter = "")
         source_path = relative_source_path
 
-        puts "Downloading multiple files from #{config['url']}#{source_path} to #{relative_dest_path} #{"allowed extensions: " + type_filter unless type_filter.empty?}"
+        puts "Downloading multiple files from #{config['application_url']}#{source_path} to #{relative_dest_path} #{"allowed extensions: " + type_filter unless type_filter.empty?}"
 
         files = html_directory_content_list(source_path)
 
         unless type_filter.empty?
           puts "Appling type filter: #{type_filter}"
-          files = files.find_all { |file| file =~ Regexp.new(".*" + type_filter.strip.gsub(" ", "|") + "$") }
+          files = files.find_all { |file| file =~ Regexp.new(".*" + type_filter.robust_split.join("|") + "$") }
         end
 
         puts "Downloading #{files.size} files..."
         files.each do |file|
-          destination_path = File.join(relative_dest_path, file)
-          @downloader.download(File.join(source_path,file), destination_path)
+          relative_destination_path = File.join(relative_dest_path, file)
+          @downloader.download(File.join(source_path, file), relative_destination_path)
         end
       end
 
@@ -139,15 +190,10 @@ module TerrImporter
         raise DefaultError, "Specify js or css url" unless for_what == :js or for_what == :css
         export_settings = config['export_settings'].clone
 
-        export_settings['application'] = config['app_path']
         export_settings.merge!(options)
         export_settings['appbaseurl'] = "" if for_what == :css
 
-        #glue everything together
-        export_path = config['export_path']
-        export_path.insert(0, "/") unless export_path.match(/^\//)
-
-        export_path = export_path % [for_what.to_s, config['version']] #replace placeholders
+        export_path = config['export_path'][for_what.to_s].clone
         export_path << '?' << export_settings.map { |k, v| "#{URI.escape(k.to_s)}=#{URI.escape(v.to_s)}" }.join("&")
         export_path
       end
@@ -166,8 +212,9 @@ module TerrImporter
         created_or_exists
       end
 
+      #todo refactor config access away
       def stylesheet_replace_strings!(line)
-        config['stylesheets']['replace'].each do |replace|
+        config['stylesheets']['replace_strings'].each do |replace|
           what = replace['what']
           with = replace['with']
           what = Regexp.new "#{$1}" if what.match(/^r\/(.*)\//)
@@ -178,6 +225,12 @@ module TerrImporter
         end
         line
       end
+
+      def check_and_complete_config!
+        unless config.required_present?
+          determine_configuration_values_from_uri
+        end
+      end
     end
   end
 end