terrimporter 0.4.0 → 0.5.4

data/.gitignore

@@ -6,3 +6,4 @@ doc
 .bundle
 Gemfile.lock
 pkg/*
+test/tmp

data/Gemfile.lock

@@ -1,20 +1,16 @@
 PATH
   remote: .
   specs:
-    terrimporter (0.4.0)
-      kwalify
+    terrimporter (0.5.3)
+      kwalify (>= 0.7.2)
 
 GEM
   remote: http://rubygems.org/
   specs:
-    git (1.2.5)
-    jeweler (1.6.4)
-      bundler (~> 1.0)
-      git (>= 1.2.5)
-      rake
+    fakeweb (1.3.0)
     kwalify (0.7.2)
     rake (0.9.2)
-    rcov (0.9.9)
+    rcov (0.9.10)
     shoulda (2.11.3)
 
 PLATFORMS
@@ -22,7 +18,8 @@ PLATFORMS
 
 DEPENDENCIES
   bundler (~> 1.0.0)
-  jeweler (~> 1.6.4)
+  fakeweb
+  rake
   rcov
   shoulda
   terrimporter!

data/Rakefile

@@ -1,4 +1,5 @@
 require 'bundler'
+require 'rake'
 Bundler::GemHelper.install_tasks
 
 require 'rake/testtask'
@@ -18,6 +19,7 @@ end
 
 task :default => :test
 
+#todo this is wrong!
 require 'rake/rdoctask'
 Rake::RDocTask.new do |rdoc|
   version = File.exist?('VERSION') ? File.read('VERSION') : ""
@@ -27,3 +29,70 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
+
+
+namespace :version do
+
+  namespace :bump do
+    desc "Bump major version"
+    task :major do
+      bump_version :major
+    end
+
+    desc "Bump minor version"
+    task :minor do
+      bump_version :minor
+    end
+
+    desc "Bump patch version"
+    task :patch do
+      bump_version :patch
+    end
+  end
+
+  def bump_version(version_to_bump)
+    puts "Pumping version"
+    version = version?
+    case version_to_bump
+      when :major
+        version[:major] = version[:major].to_i + 1
+      when :minor
+        version[:minor] = version[:minor].to_1 + 1
+      when :patch
+        version[:patch] = version[:patch].to_i + 1
+    end
+    puts "New version; " + version_string(version)
+    write_version(version)
+  end
+
+  def version_file_path
+    File.join(File.dirname(__FILE__), 'lib', 'terrimporter', 'version.rb')
+  end
+
+  def version?
+    pattern = /(\d+).(\d+).(\d+)/
+    version = nil
+    version_file = File.read(version_file_path)
+    version_file.scan(pattern) do |match|
+      version = {:major => match[0], :minor => match[1], :patch => match[2]}
+    end
+    version
+  end
+
+  def version_string(version)
+    "#{version[:major]}.#{version[:minor]}.#{version[:patch]}"
+  end
+
+  def write_version(version={})
+    version_rb = %Q{#Generated by rake task, last bump: #{version.to_s}
+module TerrImporter
+  VERSION = "#{version_string version}"
+end
+}
+    File.open(version_file_path, 'w') { |f| f.write(version_rb) }
+  end
+
+
+end
+
+

data/bin/terrimporter

@@ -1,6 +1,10 @@
 #!/usr/bin/env ruby
+###
+### $Rev$
+### $Release: 0.4.0 $
+###
+
 $LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
-require 'rubygems'
 require 'terrimporter'
 
 exit TerrImporter::Application.run!(*ARGV)

data/config/schema.yml

@@ -23,9 +23,6 @@ mapping:
       type: str
       required: yes
       pattern:  /\/.*/
-  "downloader":
-       type: str
-       enum: [curl, wget]
   "export_settings":
     type: map
     mapping:

data/config/terrimporter.config.yml

@@ -7,7 +7,6 @@
 # export_path       You most probably won't have to change this
 # image_base_path:  The Terrific image base path. You most probably won't have to change this
 # libraries_source_path:  The library source path. You most probably won't have to change this
-# downloader:       The downloader to use, supported are 'curl' and 'wget'
 #
 # export_settings:  The settings below are added as query parameters for base.js and base.css
 #   layout:         You most probably won't have to change this
@@ -19,7 +18,7 @@
 #   styles:         Name of the stylesheets, space separated without the .css extension. ex: ie ie6 ie7 ie8
 #                   the main style base.css is included by default and MUST NOT BE SPECIFIED
 #   replace:        Define path replacements after the stylesheets have been downloaded
-#                   you can define multiple replacements by repeating the '-what: with:' pattern:
+#                   you can define multiple replacements by repeating the '-what: with:' statements
 #     - what:       What to replace, default is string. if you want to use regex, use the format r/[regex]/
 #                   for help on regular expressions see http://www.rubular.com/ (an online editor/tester)
 #       with:       Replace with this (string)
@@ -37,12 +36,10 @@
 ########################################################################################################################
     version:      0.5
     url:          http://terrific.url
-    app_path:     /Users/danielkummer/_NAMICS/projekte/mobilezone/mobilezone-abo_optimizer/public/terrific
+    app_path:     /terrific/webapp/path
     export_path:  /terrific/base/%2$s/public/%1$s/base/base.%1$s.php
-    libraries_source_path: /js/libraries/dynamic/
-    #todo remove backslash
+    libraries_source_path: /js/libraries/dynamic
     image_base_path: /img
-    downloader: curl
     export_settings:
       layout: project
       debug: false
@@ -55,7 +52,7 @@
           with: /images/
     javascripts:
       dest: public/javascripts/
-      dynamic_libraries: warning pngfix
+      dynamic_libraries: dynlib
       libraries_dest: /
     images:
       - src: /

data/lib/terrimporter/config_helper.rb

@@ -0,0 +1,46 @@
+
+module ConfigHelper
+
+  def config_default_name
+    'terrimporter.config.yml'
+  end
+
+  def schema_default_name
+    'schema.yml'
+  end
+
+  def config_working_directory_path
+    File.expand_path config_default_name
+  end
+
+  def config_working_directory_exists?
+    File.exists? config_working_directory_path
+  end
+
+  def config_example_path
+    File.join(base_config_path, config_default_name)
+  end
+
+  def schema_file_path
+    File.join(base_config_path, schema_default_name)
+  end
+
+  def create_config_file(backup_or_replace = nil)
+    puts "Creating configuration file"
+    case backup_or_replace
+      when :backup
+        puts "Backing up old configuration file to #{config_working_directory_path}.bak"
+        FileUtils.mv(config_working_directory_path, config_working_directory_path + '.bak')
+      when :replace
+        puts "Replacing old configuration file"
+        FileUtils.rm_f(config_working_directory_path) if File.exists? config_working_directory_path
+    end
+    FileUtils.cp(config_example_path, config_working_directory_path)
+  end
+
+  private
+  def base_config_path
+    File.join(File.dirname(__FILE__), '..', '..', 'config')
+  end
+
+end

data/lib/terrimporter/configuration.rb

@@ -0,0 +1,72 @@
+require 'etc'
+require 'kwalify'
+require 'config_validator'
+require 'config_helper'
+require 'configuration'
+
+module TerrImporter
+  class Application
+    class Configuration < Hash
+      include ConfigHelper
+
+      attr_accessor :validations, :config_file
+
+      def initialize(config_file = nil)
+        self.config_file = config_file unless config_file.nil?
+
+      end
+
+      def load_configuration
+        config_file_path = determine_config_file_path
+        puts "Configuration file located, load from #{config_file_path}"
+        validate_and_load_config(config_file_path)
+      end
+
+      def determine_config_file_path
+        unless self.config_file.nil?
+          return self.config_file
+        end
+
+        valid_config_paths.each do |path|
+          file_path = File.join path, config_default_name
+          return file_path if File.exists?(file_path) and not file_path.include?(File.join('terrimporter', 'config')) #default config NOT valid
+        end
+
+        raise ConfigurationError, %Q{config file #{config_default_name} not found in search paths. Search paths are:
+        #{valid_config_paths.join "\n"} \n If this is a new project, run with the option --init}
+      end
+
+      def valid_config_paths
+        [
+            Dir.pwd,
+            File.join(Dir.pwd, 'config'),
+            File.join(Dir.pwd, '.config'),
+            Etc.getpwuid.dir
+        ]
+      end
+
+      #todo split!
+      def validate_and_load_config(file)
+        puts "Validating configuration..."
+
+        parser = Kwalify::Yaml::Parser.new(load_validator)
+        document = parser.parse_file(file)
+
+        errors = parser.errors()
+        if errors && !errors.empty?
+          error_message = errors.inject("") { |result, e| result << "#{e.linenum}:#{e.column} [#{e.path}] #{e.message}\n" }
+          raise ConfigurationError, error_message
+        end
+        self.merge! document
+
+      end
+
+      def load_validator
+        puts "Loading validator from #{schema_file_path}"
+        schema = Kwalify::Yaml.load_file(schema_file_path)
+        ConfigValidator.new(schema)
+      end
+
+    end
+  end
+end

data/lib/terrimporter/downloader.rb

@@ -0,0 +1,38 @@
+require 'open-uri'
+require 'uri'
+
+module TerrImporter
+  class Application
+    class Downloader
+
+      def initialize(base_uri)
+        @base_uri = base_uri
+        puts "Downloader initialized to uri: #{base_uri}"
+      end
+
+      def download(remote_path, local_path=nil)
+        absolute_uri = absolute_path(remote_path)
+        if local_path.nil? #download to buffer
+          data = StringIO.new
+
+          puts "Downloading #{absolute_uri} to buffer"
+
+          absolute_uri.open { |io| data = io.read }
+          data.to_s
+        else
+          puts "Downloading #{absolute_uri} to local path #{local_path}"
+
+          open(local_path, "wb") { |file|
+            file.write(absolute_uri.open.read)
+          }
+        end
+      end
+
+      private
+      def absolute_path(relative_path)
+        URI.join(@base_uri, relative_path)
+      end
+
+    end
+  end
+end

data/lib/terrimporter/importer.rb

@@ -0,0 +1,185 @@
+require 'fileutils'
+require 'yaml'
+require 'uri'
+
+module TerrImporter
+
+  class DefaultError < StandardError
+  end
+
+  class ConfigurationError < StandardError
+  end
+
+  class ConfigurationMissingError < StandardError
+  end
+
+  class Application
+    class Importer
+      include Logging
+
+      attr_accessor :options, :config
+
+      def initialize(options = {})
+        self.options = options
+        self.config = Configuration.new options[:config_file]
+        self.config.load_configuration
+        @downloader = Downloader.new config['url']
+      end
+
+      def run
+        if options[:all] != nil and options[:all] == true
+          puts "Import of everything started"
+          import_js
+          import_css
+          import_images
+        else
+          options.each do |option, value|
+            if option.to_s =~ /^import_/ and value == true
+              puts "Import of #{option.to_s.split('_').last} started"
+              self.send option.to_s
+            end
+          end
+        end
+      end
+
+      def import_css
+        unclean_suffix = "_unclean"
+
+        check_and_create_dir config['stylesheets']['dest']
+
+        #create stylesheet array and add base.css
+        styles = config['stylesheets']['styles'].split(" ")
+        styles << "base"
+
+        styles.each do |css|
+          destination_path = File.join(config['stylesheets']['dest'], css + ".css")
+          options = {}
+          options[:suffix] = css if css.include?('ie') #add ie option if in array
+
+          source_url = construct_export_path(:css, options)
+
+          @downloader.download(source_url, destination_path + unclean_suffix)
+
+          #do line replacement
+          puts "Start css line replacements"
+          File.open(destination_path, 'w') do |d|
+            File.open(destination_path + unclean_suffix, 'r') do |s|
+              lines = s.readlines
+              lines.each do |line|
+                d.print stylesheet_replace_strings!(line)
+              end
+            end
+          end
+          puts "Deleting unclean css files"
+          FileUtils.remove destination_path + unclean_suffix
+        end
+      end
+
+      def import_js
+        check_and_create_dir config['javascripts']['dest']
+        destination_path = File.join(config['javascripts']['dest'], "base.js")
+        js_source_url = construct_export_path :js
+
+        puts "Importing base.js from #{js_source_url} to #{destination_path}"
+
+        @downloader.download(js_source_url, destination_path)
+
+
+        libraries_destination_path = File.join(config['javascripts']['dest'], config['javascripts']['libraries_dest'])
+        check_and_create_dir libraries_destination_path
+        js_libraries = config['javascripts']['dynamic_libraries'].split(" ")
+
+        puts "Importing libraries from #{config['libraries_source_path']} to #{libraries_destination_path}"
+
+        js_libraries.each do |lib|
+          @downloader.download(File.join(config['libraries_source_path'], lib+ ".js"), File.join(libraries_destination_path, lib + ".js"))
+        end
+      end
+
+      def import_images
+        config['images'].each do |image|
+          check_and_create_dir image['dest']
+          image_source_path = File.join(config['image_base_path'], image['src'])
+          batch_download(image_source_path, image['dest'], image['types'])
+        end
+      end
+
+      private
+
+      def batch_download(relative_source_path, relative_dest_path, type_filter = "")
+        source_path = relative_source_path
+
+        puts "Downloading multiple files from #{config['url']}#{source_path} to #{relative_dest_path} #{"allowed extensions: " + type_filter unless type_filter.empty?}"
+
+        files = html_directory_content_list(source_path)
+
+        unless type_filter.empty?
+          puts "Appling type filter: #{type_filter}"
+          files = files.find_all { |file| file =~ Regexp.new(".*" + type_filter.strip.gsub(" ", "|") + "$") }
+        end
+
+        puts "Downloading #{files.size} files..."
+        files.each do |file|
+          destination_path = File.join(relative_dest_path, file)
+          @downloader.download(File.join(source_path,file), destination_path)
+        end
+      end
+
+      def html_directory_content_list(source_path)
+        puts "Getting html directory list"
+        output = @downloader.download(source_path)
+        files = []
+
+        output.scan(/<a\shref=\"([^\"]+)\"/) do |res|
+          files << res[0] if not res[0] =~ /^\?/ and not res[0] =~ /.*\/$/ and res[0].size > 1
+        end
+        puts "Found #{files.size} files"
+        files
+      end
+
+      def construct_export_path(for_what = :js, options={})
+        raise DefaultError, "Specify js or css url" unless for_what == :js or for_what == :css
+        export_settings = config['export_settings'].clone
+
+        export_settings['application'] = config['app_path']
+        export_settings.merge!(options)
+        export_settings['appbaseurl'] = "" if for_what == :css
+
+        #glue everything together
+        export_path = config['export_path']
+        export_path.insert(0, "/") unless export_path.match(/^\//)
+
+        export_path = export_path % [for_what.to_s, config['version']] #replace placeholders
+        export_path << '?' << export_settings.map { |k, v| "#{URI.escape(k.to_s)}=#{URI.escape(v.to_s)}" }.join("&")
+        export_path
+      end
+
+      def check_and_create_dir(dir, create = true)
+        created_or_exists = false
+        unless File.directory?(dir)
+          puts "Directory #{dir} does not exists... it will #{"not" unless create} be created"
+          if create
+            FileUtils.mkpath(dir)
+            created_or_exists = true
+          end
+        else
+          created_or_exists = true
+        end
+        created_or_exists
+      end
+
+      def stylesheet_replace_strings!(line)
+        config['stylesheets']['replace'].each do |replace|
+          what = replace['what']
+          with = replace['with']
+          what = Regexp.new "#{$1}" if what.match(/^r\/(.*)\//)
+
+          puts "Replacing #{what.to_s} with #{with}"
+
+          line.gsub! what, with
+        end
+        line
+      end
+    end
+  end
+end

data/lib/{options.rb → terrimporter/options.rb}

@@ -9,6 +9,7 @@ module TerrImporter
         @orig_args = args.clone
 
         self[:verbose] = true
+        self[:show_help] = false
 
         require 'optparse'
         @opts = OptionParser.new do |o|
@@ -24,10 +25,15 @@ module TerrImporter
           end
 
           o.on('-c', '--css', 'export configured css files') { self[:import_css] = true }
+
           o.on('-i', '--img', 'export configured image files') { self[:import_images] = true }
+
           o.on('-j', '--js', 'export configured javascript files') { self[:import_js] = true }
-          o.on('--init', 'create configuration file in current working directory') { self[:init] = true }
-          #todo add force option to init
+
+          o.on('--init [CONFIG_EXISTS]', [:backup, :replace], 'create configuration file in current working directory. use optional argument to force file replacement (backup, replace)') do |init|
+            self[:init] = init || true
+          end
+
           o.on('-f', '--config CONFIG_FILE', 'use alternative configuration file') do |config_file|
             self[:config_file] = config_file
           end
@@ -35,14 +41,12 @@ module TerrImporter
           o.separator ''
           o.separator 'Additional configuration:'
 
-          #o.on('-v', '--verbose [on, off], default [on]', [true, false], 'Verbose mode') do |v|
           o.on('-v', '--[no-]verbose', 'run verbosely') do |v|
             self[:verbose] = v
           end
 
           o.on('--version', 'Show version') do
-            puts ::File.open(::File.join(File.dirname(__FILE__), "..", "VERSION"), 'r').gets
-            exit
+            self[:show_version] = true
           end
 
           o.on_tail('-h', '--help', 'display this help and exit') { self[:show_help] = true }
@@ -51,7 +55,7 @@ module TerrImporter
         begin
           @opts.parse!(args)
           show_help_on_no_options
-          self[:input_file] = args.shift
+          #self[:input_file] = args.shift #todo remove if really not necessary
         rescue OptionParser::InvalidOption => e
           self[:invalid_argument] = e.message
         end
@@ -62,9 +66,13 @@ module TerrImporter
       end
 
       def show_help_on_no_options
-        self[:show_help] = true unless self[:import_css] or self[:import_js] or self[:import_images] or self[:init] or self[:version]
+        unless self[:import_css] or self[:import_js] or self[:import_images] or self[:init] or self[:version]
+          puts "None of the default options selected, showing help"
+          self[:show_help] = true
+        else
+          self[:show_help] = false
+        end
       end
-
     end
   end
 end

data/lib/terrimporter/version.rb

@@ -1,3 +1,4 @@
-module Terrimporter
-  VERSION = "0.4.0"
+#Generated by rake task, last bump: patch3major1minor5
+module TerrImporter
+  VERSION = "0.5.4"
 end