rack2aws 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6b4761ec5789bc29a4d5fc2d56799a32b149be32
+  data.tar.gz: 63c2dadc08a92138f0e249732aa870d250c41636
+SHA512:
+  metadata.gz: acb2647fd5e88e139bce7e1eff15aac44333ac6753b555ffa607702b90975a151f00e502301557bfc1a2596578416cdd15d6825286674cb0bbca545f550e7b5f
+  data.tar.gz: c3d38eea9c6ab2591022869c828e791c63666c5e8d5d35a06ed46f04c7834c8f12026c2915113db4766425ab7bcd819365bab7f18e34bfa392700efdc5fd151b

data/bin/rack2aws

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'rack2aws'
+
+Rack2Aws::CLI.new.run

data/lib/rack2aws.rb

@@ -0,0 +1,130 @@
+require 'fog'
+require 'commander'
+require 'rack2aws/config'
+require 'rack2aws/version'
+
+
+module Rack2Aws
+  class FileCopy
+    include Rack2Aws::Configuration
+
+    attr_reader :per_page, :rackspace, :aws, :rackspace_directory, :aws_directory, :verbose_mode, :files, :total
+
+    def initialize(options={})
+      options = default_options.merge(options)
+      @per_page = options[:per_page]
+      @rackspace = Fog::Storage.new(RackspaceConfig.load())
+      @aws = Fog::Storage.new(AWSConfig.load())
+
+      @rackspace_directory = rackspace.directories.get(options[:rackspace_container])
+      @aws_directory = aws.directories.get(options[:aws_bucket])
+      @verbose_mode = options[:verbose]
+      @files = []
+      @total = 0
+    end
+
+    def default_options
+      { :per_page => 10000 }
+    end
+
+    def copy
+      time = Time.now
+      pages = rackspace_directory.count / per_page + 1
+      marker = ''
+
+      # get Rackspace files
+      pages.times do |i|
+        puts "! Getting page #{i+1}..."
+        files = rackspace_directory.files.all(:limit => per_page, :marker => marker).to_a
+        puts "! #{files.size} files in page #{i+1}, forking..." if verbose_mode
+        pid = fork do
+          copy_files(i, files)
+        end
+        puts "! Process #{pid} forked to copy files" if verbose_mode
+        marker = files.last.key
+        @total += files.size
+      end
+
+      pages.times do
+        Process.wait
+      end
+
+      puts "--------------------------------------------------"
+      puts "! #{total} files copied in #{Time.now - time}secs."
+      puts "--------------------------------------------------\n\n"
+    end
+
+    def copy_files(page, files)
+      puts "  [#{Process.pid}] Page #{page+1}: Copying #{files.size} files..." if verbose_mode
+      total = files.size
+      max_processes = 4
+      process_pids = {}
+      time = Time.now
+
+      while !files.empty? or !process_pids.empty?
+        while process_pids.size < max_processes and files.any? do
+          file = files.pop
+          pid = Process.fork do
+            copy_file(file)
+          end
+          process_pids[pid] = { :file => file }
+        end
+
+        if pid_done = Process.wait
+          if job_finished = process_pids.delete(pid_done)
+            puts "    [#{Process.pid}] Page #{page+1}: Copied #{job_finished[:file].key}." if verbose_mode
+          end
+        end
+      end
+
+      puts "  [#{Process.pid}] ** Page #{page+1}: Copied #{total} files in #{Time.now - time}secs" if verbose_mode
+    end
+
+    def copy_file(file)
+      aws_directory.files.create(:key          => file.key,
+                                 :body         => file.body,
+                                 :content_type => file,
+                                 :public       => false)
+    end
+
+    private :copy_files, :copy_file
+  end
+
+  class CLI
+    include Commander::Methods
+
+    def run
+      program :name, 'rack2aws'
+      program :version, Rack2Aws::VERSION
+      program :description, 'Bridge from Rackspace Cloud Files to AWS S3'
+      program :help, 'Author', 'Faissal Elamraoui <amr.faissal@gmail.com>'
+
+      global_option('--verbose', 'Explain what is being done') { $verbose = true }
+
+      command :port do |cmd|
+        cmd.syntax = 'rack2aws port [options]'
+        cmd.description = 'Port files from Rackspace Cloud Files(tm) to AWS S3'
+
+        cmd.option '--container CONTAINER_NAME', String, 'Rackspace Cloud Files container name'
+        cmd.option '--bucket BUCKET_NAME', String, 'AWS S3 bucket name'
+        cmd.action do |args, options|
+          if options.container.nil?
+            options.container = ask('Rackspace Cloud Files container: ')
+          end
+
+          if options.bucket.nil?
+            options.bucket = ask('AWS S3 bucket: ')
+          end
+
+          FileCopy.new({
+                         :rackspace_container => options.container,
+                         :aws_bucket => options.bucket,
+                         :verbose => $verbose
+                       }).copy
+        end
+      end
+      run!
+    end
+
+  end
+end

data/lib/rack2aws/config.rb

@@ -0,0 +1,187 @@
+require 'rack2aws/props_reader'
+require 'rack2aws/errors'
+
+
+# Class to parse configuration files in the format of "param = value".
+class KVConfigParser
+  attr_accessor :config_file, :params, :groups
+
+  # Initialize the class with the path to the 'config_fil'
+  # The class objects are dynamically generated by the
+  # name of the 'param' in the config file.  Therefore, if
+  # the config file is 'param = value' then the itializer
+  # will eval "@param = value"
+  #
+  def initialize(config_file=nil, separator = '=')
+    @config_file = config_file
+    @params = {}
+    @groups = []
+    @splitRegex = '\s*' + separator + '\s*'
+
+    if(self.config_file)
+      self.validate_config()
+      self.import_config()
+    end
+  end
+
+  # Validate the config file, and contents
+  def validate_config()
+    unless File.readable?(self.config_file)
+      raise Errno::EACCES, "#{self.config_file} is not readable"
+    end
+    # FIX ME: need to validate contents/structure?
+  end
+
+  # Import data from the config to our config object.
+  def import_config()
+    # The config is top down.. anything after a [group] gets added as part
+    # of that group until a new [group] is found.
+    group = nil
+    open(self.config_file) {
+      |f|
+      f.each_with_index do |line, i|
+        line.strip!
+        # force_encoding not available in all versions of ruby
+        begin
+          if i.eql? 0 and line.include?("\xef\xbb\xbf".force_encoding("UTF-8"))
+            line.delete!("\xef\xbb\xbf".force_encoding("UTF-8"))
+          end
+        rescue NoMethodError
+        end
+
+        unless (/^\#/.match(line))
+          if(/#{@splitRegex}/.match(line))
+            param, value = line.split(/#{@splitRegex}/, 2)
+            var_name = "#{param}".chomp.strip
+            value = value.chomp.strip
+            new_value = ''
+            if (value)
+              if value =~ /^['"](.*)['"]$/
+                new_value = $1
+              else
+                new_value = value
+              end
+            else
+              new_value = ''
+            end
+
+            if group
+              self.add_to_group(group, var_name, new_value)
+            else
+              self.add(var_name, new_value)
+            end
+
+          elsif(/^\[(.+)\]$/.match(line).to_a != [])
+            group = /^\[(.+)\]$/.match(line).to_a[1]
+            self.add(group, {})
+          end
+        end
+      end
+    }
+  end
+
+  # This method will provide the value held by the object "@param"
+  # where "@param" is actually the name of the param in the config
+  # file.
+  #
+  # DEPRECATED - will be removed in future versions
+  #
+  def get_value(param)
+    puts "ParseConfig Deprecation Warning: get_value() is deprecated. Use " + \
+         "config['param'] or config['group']['param'] instead."
+    return self.params[param]
+  end
+
+  # This method is a shortcut to accessing the @params variable
+  def [](param)
+    return self.params[param]
+  end
+
+  # This method returns all parameters/groups defined in a config file.
+  def get_params()
+    return self.params.keys
+  end
+
+  # List available sub-groups of the config.
+  def get_groups()
+    return self.groups
+  end
+
+  # Adds an element to the config object
+  def add(param_name, value, override = false)
+    if value.class == Hash
+      if self.params.has_key?(param_name)
+        if self.params[param_name].class == Hash
+          if override
+            self.params[param_name] = value
+          else
+            self.params[param_name].merge!(value)
+          end
+        elsif self.params.has_key?(param_name)
+          if self.params[param_name].class != value.class
+            raise ArgumentError, "#{param_name} already exists, and is of different type!"
+          end
+        end
+      else
+        self.params[param_name] = value
+      end
+      if ! self.groups.include?(param_name)
+        self.groups.push(param_name)
+      end
+    else
+      self.params[param_name] = value
+    end
+  end
+
+  # Add parameters to a group. Parameters with the same name
+  # could be placed in different groups
+  def add_to_group(group, param_name, value)
+    if ! self.groups.include?(group)
+      self.add(group, {})
+    end
+    self.params[group][param_name] = value
+  end
+end
+
+
+module Rack2Aws
+  module Configuration
+
+    class RackspaceConfig
+      def self.load()
+        config_path = "#{ENV['HOME']}/.rack/config"
+
+        if !File.exist?(config_path)
+          raise FileNotFoundError, "Rackspace configuration file not found"
+        end
+
+        props_reader = PropertiesReader.new(config_path)
+        return {
+          :provider => 'Rackspace',
+          :rackspace_api_key => props_reader.get("api-key"),
+          :rackspace_username => props_reader.get("username"),
+          :rackspace_region => props_reader.get("region")
+        }
+      end
+    end
+
+    class AWSConfig
+      def self.load()
+        config_path = "#{ENV['HOME']}/.aws/credentials"
+
+        if !File.exist?(config_path)
+          raise FileNotFoundError, "Rackspace configuration file not found".bold.red
+        end
+
+        credentials = KVConfigParser.new(config_path)
+        return {
+          :provider => 'AWS',
+          :region => credentials['default']['region'],
+          :aws_access_key_id => credentials['default']['aws_access_key_id'],
+          :aws_secret_access_key => credentials['default']['aws_secret_access_key']
+        }
+      end
+    end
+
+  end
+end

data/lib/rack2aws/errors.rb

@@ -0,0 +1,2 @@
+class FileNotFoundError < StandardError
+end

data/lib/rack2aws/props_reader.rb

@@ -0,0 +1,19 @@
+class PropertiesReader
+  def initialize(file)
+    @file = file
+    @properties = {}
+    IO.foreach(file) do |line|
+      @properties[$1.strip] = $2 if line =~ /([^=]*)=(.*)\/\/(.*)/ || line =~ /([^=]*)=(.*)/
+    end
+  end
+
+  def to_s
+    output = "File Name #{@file} \n"
+    @properties.each {|key,value| output += "#{key}= #{value} \n"}
+    output
+  end
+
+  def get(key)
+    @properties[key].strip
+  end
+end

data/lib/rack2aws/version.rb

@@ -0,0 +1,3 @@
+module Rack2Aws
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,107 @@
+--- !ruby/object:Gem::Specification
+name: rack2aws
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Faissal Elamraoui
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-03-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: commander
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.4.0
+- !ruby/object:Gem::Dependency
+  name: fog
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.37.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.37.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: 
+email:
+- amr.faissal@gmail.com
+executables:
+- rack2aws
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/rack2aws
+- lib/rack2aws.rb
+- lib/rack2aws/config.rb
+- lib/rack2aws/errors.rb
+- lib/rack2aws/props_reader.rb
+- lib/rack2aws/version.rb
+homepage: https://amrfaissal.github.io/rack2aws
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Bridge from Rackspace Cloud Files to AWS S3
+test_files: []