contentful-importer 0.0.2 → 0.1.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: contentful-importer
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Contentful GmbH (Andreas Tiefenthaler)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-30 00:00:00.000000000 Z
+date: 2015-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: http
@@ -229,16 +229,16 @@ files:
 - example_settings/contentful_structure.json
 - example_settings/files_meaning.txt
 - example_settings/settings.yml
-- lib/cli.rb
-- lib/configuration.rb
-- lib/converters/content_types_structure_creator.rb
-- lib/converters/contentful_model_to_json.rb
-- lib/importer/data_organizer.rb
-- lib/importer/mime_content_type.rb
-- lib/importer/parallel_importer.rb
-- lib/json_schema_validator.rb
-- lib/migrator.rb
-- lib/version.rb
+- lib/contentful/importer/cli.rb
+- lib/contentful/importer/configuration.rb
+- lib/contentful/importer/converters/content_types_structure_creator.rb
+- lib/contentful/importer/converters/contentful_model_to_json.rb
+- lib/contentful/importer/data_organizer.rb
+- lib/contentful/importer/json_schema_validator.rb
+- lib/contentful/importer/migrator.rb
+- lib/contentful/importer/mime_content_type.rb
+- lib/contentful/importer/parallel_importer.rb
+- lib/contentful/importer/version.rb
 - spec/fixtures/import_files/assets/image/image_1.json
 - spec/fixtures/import_files/assets/image/image_2.json
 - spec/fixtures/import_files/assets/image/image_3.json
@@ -434,3 +434,4 @@ test_files:
 - spec/support/db_rows_json.rb
 - spec/support/shared_configuration.rb
 - spec/support/vcr.rb
+has_rdoc: 

data/lib/cli.rb

@@ -1,13 +0,0 @@
-require_relative 'migrator'
-require 'yaml'
-
-module Command
-  class CLI < Escort::ActionCommand::Base
-
-    def execute
-      setting_file = YAML.load_file(global_options[:file])
-      Migrator.new(setting_file).run(command_name, command_options)
-    end
-
-  end
-end

data/lib/configuration.rb

@@ -1,36 +0,0 @@
-require 'active_support/core_ext/hash'
-module Contentful
-  class Configuration
-    attr_reader :space_id,
-                :config,
-                :data_dir,
-                :collections_dir,
-                :entries_dir,
-                :assets_dir,
-                :log_files_dir,
-                :threads_dir,
-                :imported_entries,
-                :published_entries,
-                :published_assets,
-                :space_id
-
-    def initialize(settings)
-      @config = settings
-      validate_required_parameters
-      @data_dir = settings['data_dir']
-      @collections_dir = "#{data_dir}/collections"
-      @entries_dir = "#{data_dir}/entries"
-      @assets_dir = "#{data_dir}/assets"
-      @log_files_dir = "#{data_dir}/logs"
-      @threads_dir = "#{data_dir}/threads"
-      @imported_entries = []
-      @published_entries = []
-      @published_assets = []
-      @space_id = settings['space_id']
-    end
-
-    def validate_required_parameters
-      fail ArgumentError, 'Set PATH to data_dir. Folder where all data will be stored. View README' if config['data_dir'].nil?
-    end
-  end
-end

data/lib/importer/data_organizer.rb

@@ -1,117 +0,0 @@
-require 'fileutils'
-require 'thread'
-require_relative 'parallel_importer'
-
-module Contentful
-  class DataOrganizer
-
-    attr_reader :config, :split_params, :logger
-
-    def initialize(settings)
-      @config = settings
-      @split_params = {object_index: 0, current_thread: 0}
-      @logger = Logger.new(STDOUT)
-    end
-
-    def execute(threads_count)
-      create_threads_subdirectories(threads_count, true)
-      split_entries(threads_count)
-    end
-
-    def split_assets_to_threads(threads_count)
-      create_threads_subdirectories(threads_count, false, {assets: 'assets/'})
-      split_assets(threads_count)
-    end
-
-    def split_entries(threads_count)
-      entries_per_thread_count = total_entries_count / threads_count
-      Dir.glob("#{config.entries_dir}/*") do |dir_path|
-        collection_name = File.basename(dir_path)
-        if has_contentful_structure?(collection_name)
-          content_type_id = content_type_id_from_file(collection_name)
-          process_collection_files(content_type_id, dir_path, entries_per_thread_count, threads_count)
-        end
-      end
-    end
-
-    def split_assets(threads_count)
-      asset_per_thread = total_assets_count / threads_count
-      Dir.glob("#{config.assets_dir}/**/*json") do |asset_path|
-        copy_asset(asset_path)
-        split_params[:object_index] += 1
-        count_index_files(asset_per_thread, threads_count)
-      end
-    end
-
-    def process_collection_files(content_type_id, dir_path, entries_per_thread_count, threads_count)
-      logger.info "Processing collection: #{content_type_id}"
-      Dir.glob("#{dir_path}/*.json") do |entry_path|
-        copy_entry(entry_path, split_params[:current_thread], content_type_id)
-        split_params[:object_index] += 1
-        count_index_files(entries_per_thread_count, threads_count)
-      end
-    end
-
-    def count_index_files(objects_per_thread_count, threads_count)
-      if split_params[:object_index] == objects_per_thread_count
-        split_params[:object_index] = 0
-        set_current_thread(threads_count)
-      end
-    end
-
-    def set_current_thread(threads_count)
-      split_params[:current_thread] += 1
-      split_params[:current_thread] = 0 if split_params[:current_thread] == threads_count
-    end
-
-    def has_contentful_structure?(collection_file)
-      File.exist?("#{config.collections_dir}/#{collection_file}.json")
-    end
-
-    def content_type_id_from_file(collection_file)
-      JSON.parse(File.read("#{config.collections_dir}/#{collection_file}.json"))['id']
-    end
-
-    def new_entry_name(content_type_id, entry_path)
-      "#{content_type_id}_#{File.basename(entry_path, '.*').match(/(\d+)/)[0]}.json"
-    end
-
-    def copy_entry(entry_path, current_thread, content_type_id)
-      FileUtils.cp entry_path, "#{config.threads_dir}/#{current_thread}/#{new_entry_name(content_type_id, entry_path)}"
-    end
-
-    def copy_asset(asset_path)
-      FileUtils.cp asset_path, "#{config.threads_dir}/assets/#{split_params[:current_thread]}/#{File.basename(asset_path)}"
-    end
-
-    def create_threads_subdirectories(threads_count, validate, dir = {})
-      validate_collections_files if validate
-      create_directory(config.threads_dir)
-      threads_count.times do |thread_id|
-        create_directory("#{config.threads_dir}/#{dir[:assets]}#{thread_id}")
-      end
-    end
-
-    def create_directory(path)
-      FileUtils.mkdir_p(path) unless File.directory?(path)
-    end
-
-    def total_entries_count
-      total_number = 0
-      Dir.glob("#{config.entries_dir}/*") do |dir_path|
-        collection_name = File.basename(dir_path)
-        total_number += Dir.glob("#{config.entries_dir}/#{collection_name}/*").count if has_contentful_structure?(collection_name)
-      end
-      total_number
-    end
-
-    def total_assets_count
-      Dir.glob("#{config.assets_dir}/**/*json").count
-    end
-
-    def validate_collections_files
-      fail ArgumentError, "Make sure the #{config.collections_dir} directory exists and the content structure resides within it. View README" unless Dir.exist?(config.collections_dir)
-      fail ArgumentError, 'Collections directory is empty! Create content types JSON files. View README' if Dir.glob("#{config.collections_dir}/*").empty?
-    end
-  end
-end