terrimporter 0.6.4 → 0.7.0

data/lib/terrimporter/importer.rb

@@ -11,8 +11,7 @@ module TerrImporter
 
   class Application
     class Importer
-      include Logging
-
+      include ImporterHelper
       attr_accessor :options, :config
 
       def initialize(options = {})
@@ -23,173 +22,126 @@ module TerrImporter
       end
 
       def run
-
-
         if options[:all] != nil and options[:all] == true
-          puts "Import of everything started"
+          LOG.info "Import everything"
           import_js
           import_css
           import_images
+          import_modules
         else
           options.each do |option, value|
             if option.to_s =~ /^import_/ and value == true
-              puts "Import of #{option.to_s.split('_').last} started"
+              LOG.info "Import of #{option.to_s.split('_').last} started"
               self.send option.to_s
             end
           end
         end
       end
 
-      def determine_configuration_values_from_uri
-        result = @downloader.download('')
-        #result =~ /\/terrific\/base\/(.*?)\/public\/.*application=(.*?)(&|&)/
-        #result =~ /(\/terrific\/base\/(.*?)\/public\/.*base.(css|js).php).*application=(.*?)(&|&)/
-
-        css_result, js_result = result.scan(/(\/terrific\/base\/(.*?)\/public\/.*base.(css|js).php)\?.*application=(.*?)(&|&)/)
-
-
-        if css_result.nil? or css_result.size < 5
-          raise ConfigurationError, "Unable to extract css information from application url, content is: #{result}"
-        end
-        if js_result.nil? or js_result.size < 5
-          raise ConfigurationError, "Unable to extract javascript information from application url, content is: #{result}"
-        end
-
-        css_export_path = css_result[0]
-        js_export_path = js_result[0]
-        terrific_version = css_result[1]
-        application = css_result[3]
-
-        raise ConfigurationError, "Unable to determine css export path from application url" if css_export_path.nil?
-        raise ConfigurationError, "Unable to determine js export path from application url" if js_export_path.nil?
-
-        puts "Determined the following configuration values from #{config['application_url']}:\n" +
-                 "terrific version: #{terrific_version} \n" +
-                 "application path: #{application}"
-
-        config['version'] = terrific_version
-        config['export_settings']['application'] = application
-        config['export_path'] = {'css' => css_export_path, 'js' => js_export_path}
-      end
-
       def import_css
-        check_and_complete_config!
-        unclean_suffix = "_unclean"
+        LOG.info("Importing stylesheets")
+        complete_config!
 
-        check_and_create_dir config['stylesheets']['relative_destination_path']
-
-        styles = config.stylesheets
+        unclean_suffix = "_unclean"
+        stylesheets = config.stylesheets
 
-        styles.each do |css|
-          relative_destination_path = File.join(config['stylesheets']['relative_destination_path'], css)
+        stylesheets.each do |css|
+          file_path = File.join(config['stylesheets']['destination_path'], css)
           options = {}
           options[:suffix] = $1 if css =~ /(ie.*).css$/ #add ie option if in array
-
-          source_url = construct_export_path(:css, options)
-
-          @downloader.download(source_url, relative_destination_path + unclean_suffix)
+          source_url = export_path(:css, options)
+          @downloader.download(source_url, file_path + unclean_suffix)
 
           if config.replace_style_strings?
-            puts "Start css line replacements..."
-            File.open(relative_destination_path, 'w') do |d|
-              File.open(relative_destination_path + unclean_suffix, 'r') do |s|
+            LOG.debug "CSS line replacements"
+            File.open(file_path, 'w') do |d|
+              File.open(file_path + unclean_suffix, 'r') do |s|
                 lines = s.readlines
                 lines.each do |line|
-                  d.print stylesheet_replace_strings!(line)
+                  d.print replace_stylesheet_lines!(line)
                 end
               end
             end
           else
-            puts "No css line replacements defined; skipping..."
+            LOG.debug "Skipping css line replacements"
           end
-          puts "Deleting unclean css files"
-          FileUtils.remove relative_destination_path + unclean_suffix
+          LOG.debug "Deleting unclean css files"
+          FileUtils.remove file_path + unclean_suffix
         end
       end
 
       def import_js
-        check_and_complete_config!
-        check_and_create_dir config['javascripts']['relative_destination_path']
-        relative_destination_path = File.join(config['javascripts']['relative_destination_path'], "base.js")
-        js_source_url = construct_export_path :js
-
-        puts "Importing base.js from #{js_source_url} to #{relative_destination_path}"
-
-        @downloader.download(js_source_url, relative_destination_path)
-
+        LOG.info("Importing javascripts")
+        complete_config!
+        file_path = File.join(config['javascripts']['destination_path'], "base.js")
+        js_source_url = export_path(:js)
+        LOG.debug "Import base.js from #{js_source_url} to #{file_path}"
+        @downloader.download(js_source_url, file_path)
 
         if config.additional_dynamic_javascripts?
-
-          libraries_destination_path = config.libraries_destination_path
-          check_and_create_dir libraries_destination_path
-          js_libraries = config.dynamic_libraries
-
-          puts "Importing libraries from #{config['libraries_server_path']} to #{libraries_destination_path}"
-
           if config['libraries_server_path'].nil?
-            puts "Please define 'libraries_server_path' in configuration file"
+            LOG.info "Define 'libraries_server_path' in configuration file"
           else
+            libraries_file_path = config.libraries_destination_path
+            LOG.info "Import libraries from #{config['libraries_server_path']} to #{libraries_file_path}"
+            js_libraries = config.dynamic_libraries
             js_libraries.each do |lib|
-              @downloader.download(File.join(config['libraries_server_path'], lib), File.join(libraries_destination_path, lib))
+              @downloader.download(File.join(config['libraries_server_path'], lib), File.join(libraries_file_path, lib))
             end
-
           end
-
         end
       end
 
       def import_images
-        check_and_complete_config!
+        complete_config!
         if config.images?
-          puts "Start importing images..."
-
+          LOG.info "Import images"
           config['images'].each do |image|
-            check_and_create_dir image['relative_destination_path']
             image_source_path = File.join(config['image_server_path'], image['server_path'])
-            batch_download(image_source_path, image['relative_destination_path'], image['file_types'])
+            @downloader.batch_download(image_source_path, image['destination_path'], image['file_types'])
           end
         else
-          puts "No image configuration found, skipping image import..."
+          LOG.debug "Skipping image import"
         end
       end
 
-      private
-
-      def batch_download(relative_source_path, relative_dest_path, type_filter = "")
-        source_path = relative_source_path
-
-        puts "Downloading multiple files from #{config['application_url']}#{source_path} to #{relative_dest_path} #{"allowed extensions: " + type_filter unless type_filter.empty?}"
-
-        files = html_directory_content_list(source_path)
-
-        unless type_filter.empty?
-          puts "Appling type filter: #{type_filter}"
-          files = files.find_all { |file| file =~ Regexp.new(".*" + type_filter.robust_split.join("|") + "$") }
+      def complete_config!
+        unless config.mandatory_present?
+          config.determine_configuration_values_from_html @downloader.download('')
         end
+      end
 
-        puts "Downloading #{files.size} files..."
-        files.each do |file|
-          relative_destination_path = File.join(relative_dest_path, file)
-          @downloader.download(File.join(source_path, file), relative_destination_path)
+      def import_modules
+        complete_config!
+        if config.modules?
+          LOG.info "Module import"
+          config['modules'].each do |mod|
+            name = mod['name']
+            skin = mod['skin']
+            module_source_url = module_path(name, mod['module_template'], skin, mod['template_only'])
+            filename = name.clone
+            filename << "_#{skin}" unless skin.to_s.strip.length == 0
+            @downloader.download(module_source_url, File.join(mod['destination_path'], filename + '.html'))
+          end
+        else
+          LOG.debug "Skipping module import"
         end
       end
 
-      def html_directory_content_list(source_path)
-        puts "Getting html directory list"
-        output = @downloader.download(source_path)
-        files = []
-
-        output.scan(/<a\shref=\"([^\"]+)\"/) do |res|
-          files << res[0] if not res[0] =~ /^\?/ and not res[0] =~ /.*\/$/ and res[0].size > 1
-        end
-        puts "Found #{files.size} files"
-        files
+      def module_path(name, module_template, skin = nil, template = nil)
+        skin = '' if skin.nil?
+        raise ConfigurationError, "Name cannot be empty for template" if name.nil?
+        raise ConfigurationError, "Module template missing in configuration for template #{name}" if module_template.nil?
+        export_path = config['application_url'].clone
+        export_path << "/terrific/module/details/#{name}/#{module_template}/#{skin}/format/module#{"content" if template}"
+        export_path
       end
 
-      def construct_export_path(for_what = :js, options={})
+      private
+
+      def export_path(for_what = :js, options={})
         raise DefaultError, "Specify js or css url" unless for_what == :js or for_what == :css
         export_settings = config['export_settings'].clone
-
         export_settings.merge!(options)
         export_settings['appbaseurl'] = "" if for_what == :css
 
@@ -198,39 +150,12 @@ module TerrImporter
         export_path
       end
 
-      def check_and_create_dir(dir, create = true)
-        created_or_exists = false
-        unless File.directory?(dir)
-          puts "Directory #{dir} does not exists... it will #{"not" unless create} be created"
-          if create
-            FileUtils.mkpath(dir)
-            created_or_exists = true
-          end
-        else
-          created_or_exists = true
-        end
-        created_or_exists
-      end
-
-      #todo refactor config access away
-      def stylesheet_replace_strings!(line)
+      def replace_stylesheet_lines!(line)
         config['stylesheets']['replace_strings'].each do |replace|
-          what = replace['what']
-          with = replace['with']
-          what = Regexp.new "#{$1}" if what.match(/^r\/(.*)\//)
-
-          puts "Replacing #{what.to_s} with #{with}"
-
-          line.gsub! what, with
+          replace_line!(line, replace['what'], replace['with'])
         end
         line
       end
-
-      def check_and_complete_config!
-        unless config.required_present?
-          determine_configuration_values_from_uri
-        end
-      end
     end
   end
 end

data/lib/terrimporter/importer_helper.rb

@@ -0,0 +1,9 @@
+module ImporterHelper
+
+  def replace_line!(line, what, with)
+    what = Regexp.new "#{$1}" if what.match(/^r\/(.*)\//)
+    LOG.info "Replacing #{what.to_s} with #{with}"
+    line.gsub! what, with
+  end
+
+end

data/lib/terrimporter/options.rb

@@ -5,51 +5,32 @@ module TerrImporter
 
       def initialize(args)
         super()
-
         @orig_args = args.clone
-
-        self[:verbose] = true
+        self[:verbose] = false
         self[:show_help] = false
 
         require 'optparse'
         @opts = OptionParser.new do |o|
           o.banner = "Usage: #{File.basename($0)} [options] \n" +
           "Use #{File.basename($0)} [application_url] --init to initialize importer before first usage."
-
           o.separator ''
           o.separator 'Common options:'
-
-          o.on('-a', '--all', 'export everything configured; javascripts, css files and images') do
+          o.on('-a', '--all', 'import everything configured; javascripts, css files and images') do
             self[:import_css] = true
             self[:import_js] = true
             self[:import_images] = true
+            self[:import_modules] = true
           end
-
-          o.on('-c', '--css', 'export configured css files') { self[:import_css] = true }
-
-          o.on('-i', '--img', 'export configured image files') { self[:import_images] = true }
-
-          o.on('-j', '--js', 'export configured javascript files') { self[:import_js] = true }
-
-          o.on('--init [CONFIG_EXISTS]', [:backup, :replace], 'create configuration file in current working directory. use optional argument to force file replacement (backup, replace)') do |init|
-            self[:init] = init || true
-          end
-
-          o.on('-f', '--config CONFIG_FILE', 'use alternative configuration file') do |config_file|
-            self[:config_file] = config_file
-          end
-
+          o.on('-c', '--css', 'import configured css files') { self[:import_css] = true }
+          o.on('-i', '--img', 'import configured image files') { self[:import_images] = true }
+          o.on('-j', '--js', 'import configured javascript files') { self[:import_js] = true }
+          o.on('-m', '--module', 'import configured module files') { self[:import_modules] = true }
+          o.on('--init [CONFIG_EXISTS]', [:backup, :replace], 'create configuration file in current working directory. use optional argument to force file replacement (backup, replace)') { |init| self[:init] = init || true }
+          o.on('-f', '--config CONFIG_FILE', 'use alternative configuration file') { |config_file| self[:config_file] = config_file }
           o.separator ''
           o.separator 'Additional configuration:'
-
-          o.on('-v', '--[no-]verbose', 'run verbosely') do |v|
-            self[:verbose] = v
-          end
-
-          o.on('--version', 'Show version') do
-            self[:show_version] = true
-          end
-
+          o.on('-v', '--[no-]verbose', 'run verbosely') { |v| self[:verbose] = v }
+          o.on('--version', 'Show version') { self[:show_version] = true }
           o.on_tail('-h', '--help', 'display this help and exit') { self[:show_help] = true }
         end
 
@@ -57,6 +38,8 @@ module TerrImporter
           @opts.parse!(args)
           self[:application_url] = args.shift
         rescue OptionParser::InvalidOption => e
+          self[:invalid_option] = e.message
+        rescue OptionParser::InvalidArgument => e
           self[:invalid_argument] = e.message
         end
       end

data/lib/terrimporter/version.rb

@@ -1,4 +1,4 @@
-#Generated by rake task, last bump: minor6patch4major0
+#Generated by rake task, last bump: minor6patch6major0
 module TerrImporter
-  VERSION = "0.6.4"
+  VERSION = "0.7.0"
 end

data/lib/terrimporter.rb

@@ -1,9 +1,11 @@
 require 'shellwords'
 require 'terrimporter/version'
+require 'terrimporter/download_helper'
 require 'terrimporter/app_logger'
 require 'terrimporter/options'
+require 'terrimporter/importer_helper'
 require 'terrimporter/importer'
-require 'terrimporter/config_helper'
+require 'terrimporter/configuration_helper'
 require 'terrimporter/configuration'
 require 'terrimporter/downloader'
 require 'terrimporter/string_monkeypatch'
@@ -18,7 +20,7 @@ module TerrImporter
   class Application
     class << self
       include Shellwords
-      include ConfigHelper
+      include ConfigurationHelper
 
       def run!(*arguments)
         options = build_options(arguments)
@@ -32,11 +34,24 @@ module TerrImporter
             return 0
           end
 
+          case options[:verbose]
+            when true
+              LOG.level = :debug
+            when false
+              LOG.level = :info
+          end
+
+
           if options[:invalid_argument]
             $stderr.puts options[:invalid_argument]
             options[:show_help] = true
           end
 
+          if options[:invalid_option]
+            $stderr.puts options[:invalid_option]
+            options[:show_help] = true
+          end
+
           if options[:show_help]
             $stderr.puts options.opts
             return 1

data/test/fixtures/html/application_root.html

@@ -1,4 +1,5 @@
 <!DOCTYPE html>
+<html>
 <head>
     <title>Some title</title>
     <meta http-equiv="content-type" content="text/html; charset=utf-8"/>

data/test/fixtures/html/application_root_css_error.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Some title</title>
+    <meta http-equiv="content-type" content="text/html; charset=utf-8"/>
+    <meta name="language" content="de-ch, de"/>
+    <link href="/img/favicon.png" rel="apple-touch-icon"/>
+    <link href="/img/favicon.ico" rel="shortcut icon"/>
+</head>
+<body>
+    <p>this is not as is should be...</p>
+</body>
+</html>

data/test/fixtures/html/application_root_js_error.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Some title</title>
+    <meta http-equiv="content-type" content="text/html; charset=utf-8"/>
+    <meta name="language" content="de-ch, de"/>
+    <link href="/img/favicon.png" rel="apple-touch-icon"/>
+    <link href="/img/favicon.ico" rel="shortcut icon"/>
+    <link href="/terrific/base/0.5/public/css/base/base.css.php?appbaseurl=&amp;application=/terrific/webapp/path&amp;layout=project&amp;debug=true&amp;cache=false&amp;t=1314606536" media="all" rel="stylesheet" type="text/css"/>
+</head>
+<body>
+    <p>this is not as is should be...</p>
+</body>
+</html>

data/test/fixtures/html/module.html

@@ -0,0 +1,9 @@
+<div class="mod modList"><div class="inner">
+<!-- tpl List-sublist -->
+<div class="class1">
+	<div class="class2">
+        <p>this is a terrific module</p>
+	</div>
+</div>
+<!-- /tpl List-sublist -->
+</div></div><!-- /mod List -->

data/test/fixtures/html/modulecontent.html

@@ -0,0 +1,7 @@
+<!-- tpl List-sublist -->
+<div class="class1">
+	<div class="class2">
+        <p>this is a terrific module content</p>
+	</div>
+</div>
+<!-- /tpl List-sublist -->

data/test/fixtures/invalid.config.yml

@@ -14,7 +14,7 @@
     javascripts:
       dest: test/tmp/public/javascripts/
       dynamic_libraries: dynlib.js
-      libraries_relative_destination_path: /lib
+      libraries_destination_path: /lib
     images:
       - server_path: /
         dest: test/tmp/public/images/
@@ -22,3 +22,5 @@
       - server_path: /backgrounds
         dest: test/tmp/public/images/backgrounds/
         file_types: jpg
+    modules:
+      - name:

data/test/fixtures/minimal.test.config.yml

@@ -6,8 +6,9 @@ export_settings:
   debug: false
   cache: false
 stylesheets:
-  relative_destination_path: test/tmp/public/stylesheets/
+  destination_path: test/tmp/public/stylesheets/
 javascripts:
-  relative_destination_path: test/tmp/public/javascripts/
+  destination_path: test/tmp/public/javascripts/
 images:
+modules:
 

data/test/fixtures/test.config.yml

@@ -6,20 +6,29 @@
       debug: false
       cache: false
     stylesheets:
-      relative_destination_path: test/tmp/public/stylesheets/
+      destination_path: test/tmp/public/stylesheets/
       styles: ie.css
       replace_strings:
         - what: /img/
           with: /images/
     javascripts:
-      relative_destination_path: test/tmp/public/javascripts/
+      destination_path: test/tmp/public/javascripts/
       dynamic_libraries: dynlib
-      libraries_relative_destination_path: test/tmp/public/javascripts/lib
+      libraries_destination_path: test/tmp/public/javascripts/lib
     images:
       - server_path: /
-        relative_destination_path: test/tmp/public/images/
+        destination_path: test/tmp/public/images/
         file_types: jpg, png, gif
       - server_path: /backgrounds
-        relative_destination_path: test/tmp/public/images/backgrounds/
+        destination_path: test/tmp/public/images/backgrounds/
         file_types: jpg
+    modules:
+      - name: moduleName
+        skin: moduleSkin
+        destination_path: test/tmp/modules/
+        module_template: moduleTemplate
+      - name: moduleName
+        destination_path: test/tmp/modules/
+        module_template: moduleTemplate
+        template_only: true
 

data/test/test_helper.rb

@@ -17,6 +17,27 @@ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib', 'terrimporter'
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'terrimporter'
 
+require 'stringio'
+module Kernel
+  def capture_stdout
+    out = StringIO.new
+    $stdout = out
+    yield
+    return out
+  ensure
+    $stdout = STDOUT
+  end
+
+  def capture_stderr
+    out = StringIO.new
+    $stderr = out
+    yield
+    return out
+  ensure
+    $stderr = STDERR
+  end
+end
+
 class Test::Unit::TestCase
   def schema_file_path
     File.join(File.dirname(__FILE__), '..', 'config', schema_default_name)
@@ -46,6 +67,8 @@ class Test::Unit::TestCase
     FileUtils.rm_rf tmp_test_directory
   end
 
-
+  def exists_in_tmp?(name)
+    File.exists? File.join(tmp_test_directory, name)
+  end
 
 end

data/test/test_terrimporter.rb

@@ -1,7 +1,7 @@
 require 'test_helper'
 
 class TestTerrimporter < Test::Unit::TestCase
-  include ConfigHelper
+  include ConfigurationHelper
 
   def setup
     ENV['TERRIMPORTER_OPTS'] = nil
@@ -9,6 +9,7 @@ class TestTerrimporter < Test::Unit::TestCase
 
   def teardown
     File.delete config_file if File.exists? config_file
+    File.delete config_file + ".bak" if File.exists? config_file + ".bak"
   end
 
   should 'build options as a combination form argument options and environment options' do
@@ -19,7 +20,7 @@ class TestTerrimporter < Test::Unit::TestCase
     assert merged_options.include?(:import_css)
   end
 
-    should 'merge environment and argument options' do
+  should 'merge environment and argument options' do
     ENV['TERRIMPORTER_OPTS'] = '-j -c'
     merged_options = TerrImporter::Application.build_options([''] + ['-i', '--verbose'])
     expected_options = {:application_url => "",
@@ -45,7 +46,7 @@ class TestTerrimporter < Test::Unit::TestCase
 
   should 'run the importer with the init command and a non existing configuration file' do
     TerrImporter::Application.run!(["test"], '--init')
-    TerrImporter::Application.run!(["test"], '--init','backup')
+    TerrImporter::Application.run!(["test"], '--init', 'backup')
     assert File.exists? config_file
   end
 
@@ -55,8 +56,13 @@ class TestTerrimporter < Test::Unit::TestCase
     assert return_code == 1
   end
 
+  should 'run the importer with an invalid option, display help and return error code' do
+    return_code = TerrImporter::Application.run!(["test"], '--invalidoption')
+    assert return_code == 1
+  end
+
   should 'run the importer with an invalid argument, display help and return error code' do
-    return_code = TerrImporter::Application.run!(["test"], '--invalid')
+    return_code = TerrImporter::Application.run!(["test"], '--init', 'INVALID')
     assert return_code == 1
   end
 
@@ -68,6 +74,17 @@ class TestTerrimporter < Test::Unit::TestCase
   should 'run the importer show version and return' do
     return_code = TerrImporter::Application.run!(["test"], '--version')
     assert return_code == 0
+    end
+
+  should 'run the importer in verbose mode' do
+    return_code = TerrImporter::Application.run!(["test"], '-v')
+    assert return_code == 1 #configuration missing because not initialized
+  end
+
+  should 'run the importer with the init command and a non existing configuration file' do
+    TerrImporter::Application.run!(["http://terrific.url"], '--init')
+    return_code = TerrImporter::Application.run!(["http://terrific.url"], '-a') #full run
+    assert return_code == 0
   end
 
   def config_file