contentful-importer 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b73728474014c30ccd464ed1d5cf924709ab305c
-  data.tar.gz: 09ce5c3def7d50dd13817c754f0e5e8819ca10a5
+  metadata.gz: 7bbaffd5d3e9186ca222dc18419f7ed4fa489115
+  data.tar.gz: c3d3d73c0700ebc485e27d4f6a078768c5ba6ef1
 SHA512:
-  metadata.gz: 1c863bfd32bec39e6c64b437bf84053fa0bd6505e20f42d39a7777a997a7ac575c6ebccf3b7988da8b2b58d692cb4b770af3346c822bdb4222cae2183c9ffed2
-  data.tar.gz: a76e0c297f18bd96796e859470412a08dbd041f006725c7da4146ceee43cb89d0023c09ac36a109d5d31e6d240b5f9c9937b99249d2b123b1b977ef9e127c135
+  metadata.gz: fc7e161a2a8ba9305c8c9a291dbc82d1efa41a8cb65a91cf0191e90f495f238f1b733aba09d79029fbd4591c0901938f9232aee2c60379bc19d5aa3fc3e52630
+  data.tar.gz: 5550652c932b12155254e6ef460b657ad76f3ee03daadc6eed800771357f12878bacf3065aa3f8abfaa450c438cdbff10ad179fb7b412d365fd2dfa99d671578

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Change Log
 
+## 0.1.0
+### Added
+* Log publish errors similar to import errors
+
+### Fixed
+* Introduce proper namespacing of all classes
+
 ## 0.0.2
 ### Added
 * Read contentType and description from asset JSON files, falling back to contentType inference from the asset URL.

data/README.md

@@ -63,8 +63,8 @@ contentful-importer --config-file settings.yml --action
     contentful_structure_dir: PATH_TO_CONTENTFUL_STRUCTURE_JSON_FILE
 
     ## CONVERT CONTENTFUL MODEL TO CONTENTFUL IMPORT STRUCTURE
-    content_model_json:
-    converted_model_dir:
+    content_model_json: PATH_TO_CONTENTFUL_MODEL_JSON_FILE ## for input
+    converted_model_dir: PATH_TO_CONTENTFUL_MODEL_JSON_FILE ## for output
     ```
 
 2. Create the contentful_structure.json. First you need to create a content model using the [Contentful web application](www.contentful.com). Then you can download the content model using the content management api and use the content model for the import:
@@ -140,6 +140,7 @@ To display all actions use the `-h` option:
 ```bash
 contentful-importer -h
 ```
+
 #### --convert-content-model-to-json
 
 If you already have an existing content model for a space it can be downloaded and used for the import:
@@ -150,7 +151,6 @@ If you already have an existing content model for a space it can be downloaded a
       'https://api.contentful.com/spaces/SPACE_ID/content_types' > contentful_model.json
 ```
 
-
 In the **settings.yml** specify the PATH to **contentful_model.json**.
 
 ```yaml
@@ -222,6 +222,7 @@ To publish all entries:
 ```bash
 contentful-importer --config-file settings.yml --publish-entries
 ```
+
 Number of threads that are used in the publishing of entries is dependent on `--threads` argument, which you specified when import data.
 
 #### --publish-assets ARGS
@@ -231,6 +232,7 @@ You can publish all assets with single Thread:
 ```bash
 contentful-importer --config-file settings.yml --publish-assets
 ```
+
 or add ```--threads``` argument to use multiple Threads:
 
 ```bash
@@ -310,7 +312,7 @@ Before you start importing the content make sure you read [how to use it](https:
 
 #### Space ID
 
-After [importing the content types](https://github.com/contentful/generic-importer.rb#--import-content-types-args) to the Space, you need to specify the `space_id` parameter in the settings.
+After [importing the content types](https://github.com/contentful/generic-importer.rb#--import-content-types-args) to the Space, you need to specify the `space_id` parameter in the settings. Please note that the content-type import only accepts the space ID as a commandline option.
 
 
 Example:

data/bin/contentful-importer

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 
 require 'escort'
-require_relative '../lib/cli'
+require_relative '../lib/contentful/importer/cli'
 
 fail ArgumentError, 'Set the path to the configuration file and define an action. More information can be found in the README.' if ARGF.argv.empty?
 fail ArgumentError, "Missing '--config-file' argument. Usage: 'contentful-importer --config-file PATH_TO_CONFIGURATION_FILE --action'." unless ARGV.include?('--config-file')
@@ -15,13 +15,13 @@ Escort::App.create do |app|
     app.command '--create-contentful-model-from-json' do |command|
       command.summary 'Create content types json files, based on contentful_structure.json file.'
       command.action do |options, arguments|
-        Command::CLI.new(options, arguments).execute
+        Contentful::Importer::CLI.new(options, arguments).execute
       end
     end
     app.command '--convert-content-model-to-json' do |command|
       command.summary 'Transform contentful model to contentful structure ready to import. View README'
       command.action do |options, arguments|
-        Command::CLI.new(options, arguments).execute
+        Contentful::Importer::CLI.new(options, arguments).execute
       end
     end
     app.command '--import-content-types' do |command|
@@ -32,7 +32,7 @@ Escort::App.create do |app|
         opts.opt :space_name, 'space_name', short: :none, long: '--space_name', type: :string
       end
       command.action do |options, arguments|
-        Command::CLI.new(options, arguments).execute
+        Contentful::Importer::CLI.new(options, arguments).execute
       end
     end
     app.command '--import' do |command|
@@ -42,26 +42,26 @@ Escort::App.create do |app|
         opts.validate(:threads, 'argument must be set to 1 or 2 exclusive. ') { |option| option > 0 && option <= 2 }
       end
       command.action do |options, arguments|
-        Command::CLI.new(options, arguments).execute
+        Contentful::Importer::CLI.new(options, arguments).execute
       end
     end
     app.command '--test-credentials' do |command|
       command.summary 'Check your Contentful-management credentials'
       command.action do |options, arguments|
-        Command::CLI.new(options, arguments).execute
+        Contentful::Importer::CLI.new(options, arguments).execute
       end
     end
     app.command '--import-assets' do |command|
       command.summary 'Import only assets'
       command.action do |options, arguments|
-        Command::CLI.new(options, arguments).execute
+        Contentful::Importer::CLI.new(options, arguments).execute
       end
     end
 
     app.command '--publish-entries' do |command|
       command.summary 'Publish all entries'
       command.action do |options, arguments|
-        Command::CLI.new(options, arguments).execute
+        Contentful::Importer::CLI.new(options, arguments).execute
       end
     end
 
@@ -72,13 +72,13 @@ Escort::App.create do |app|
         opts.validate(:threads, 'argument must be set to 1 or 2 exclusive. ') { |option| option > 0 && option <= 2 }
       end
       command.action do |options, arguments|
-        Command::CLI.new(options, arguments).execute
+        Contentful::Importer::CLI.new(options, arguments).execute
       end
     end
     app.command '--validate-schema' do |command|
       command.summary 'Validate entries JSON schema'
       command.action do |options, arguments|
-        Command::CLI.new(options, arguments).execute
+        Contentful::Importer::CLI.new(options, arguments).execute
       end
     end
   end

data/contentful_importer.gemspec

@@ -2,11 +2,11 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
-require File.expand_path('../lib/version', __FILE__)
+require File.expand_path('../lib/contentful/importer/version', __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = 'contentful-importer'
-  spec.version       = Version::VERSION
+  spec.version       = Contentful::Importer::VERSION
   spec.authors       = ['Contentful GmbH (Andreas Tiefenthaler)']
   spec.email         = ['rubygems@contentful.com']
   spec.description   = 'Generic importer for contentful.com'

data/lib/contentful/importer/cli.rb

@@ -0,0 +1,15 @@
+require_relative 'migrator'
+require 'yaml'
+
+module Contentful
+	module Importer
+	  class CLI < Escort::ActionCommand::Base
+
+	    def execute
+	      setting_file = YAML.load_file(global_options[:file])
+	      Migrator.new(setting_file).run(command_name, command_options)
+	    end
+
+	  end
+	end
+end

data/lib/contentful/importer/configuration.rb

@@ -0,0 +1,38 @@
+require 'active_support/core_ext/hash'
+module Contentful
+  module Importer
+    class Configuration
+      attr_reader :space_id,
+                  :config,
+                  :data_dir,
+                  :collections_dir,
+                  :entries_dir,
+                  :assets_dir,
+                  :log_files_dir,
+                  :threads_dir,
+                  :imported_entries,
+                  :published_entries,
+                  :published_assets,
+                  :space_id
+
+      def initialize(settings)
+        @config = settings
+        validate_required_parameters
+        @data_dir = settings['data_dir']
+        @collections_dir = "#{data_dir}/collections"
+        @entries_dir = "#{data_dir}/entries"
+        @assets_dir = "#{data_dir}/assets"
+        @log_files_dir = "#{data_dir}/logs"
+        @threads_dir = "#{data_dir}/threads"
+        @imported_entries = []
+        @published_entries = []
+        @published_assets = []
+        @space_id = settings['space_id']
+      end
+
+      def validate_required_parameters
+        fail ArgumentError, 'Set PATH to data_dir. Folder where all data will be stored. View README' if config['data_dir'].nil?
+      end
+    end
+  end
+end

data/lib/{converters → contentful/importer/converters}/content_types_structure_creator.rb

@@ -1,5 +1,5 @@
 module Contentful
-  module Converter
+  module Importer
     class ContentTypesStructureCreator
 
       attr_reader :config, :logger

data/lib/{converters → contentful/importer/converters}/contentful_model_to_json.rb

@@ -1,7 +1,7 @@
 require_relative 'content_types_structure_creator'
 
 module Contentful
-  module Converter
+  module Importer
     class ContentfulModelToJson
       attr_reader :config, :logger, :converted_model_dir, :content_types
 

data/lib/contentful/importer/data_organizer.rb

@@ -0,0 +1,119 @@
+require 'fileutils'
+require 'thread'
+require_relative 'parallel_importer'
+
+module Contentful
+  module Importer
+    class DataOrganizer
+
+      attr_reader :config, :split_params, :logger
+
+      def initialize(settings)
+        @config = settings
+        @split_params = {object_index: 0, current_thread: 0}
+        @logger = Logger.new(STDOUT)
+      end
+
+      def execute(threads_count)
+        create_threads_subdirectories(threads_count, true)
+        split_entries(threads_count)
+      end
+
+      def split_assets_to_threads(threads_count)
+        create_threads_subdirectories(threads_count, false, {assets: 'assets/'})
+        split_assets(threads_count)
+      end
+
+      def split_entries(threads_count)
+        entries_per_thread_count = total_entries_count / threads_count
+        Dir.glob("#{config.entries_dir}/*") do |dir_path|
+          collection_name = File.basename(dir_path)
+          if has_contentful_structure?(collection_name)
+            content_type_id = content_type_id_from_file(collection_name)
+            process_collection_files(content_type_id, dir_path, entries_per_thread_count, threads_count)
+          end
+        end
+      end
+
+      def split_assets(threads_count)
+        asset_per_thread = total_assets_count / threads_count
+        Dir.glob("#{config.assets_dir}/**/*json") do |asset_path|
+          copy_asset(asset_path)
+          split_params[:object_index] += 1
+          count_index_files(asset_per_thread, threads_count)
+        end
+      end
+
+      def process_collection_files(content_type_id, dir_path, entries_per_thread_count, threads_count)
+        logger.info "Processing collection: #{content_type_id}"
+        Dir.glob("#{dir_path}/*.json") do |entry_path|
+          copy_entry(entry_path, split_params[:current_thread], content_type_id)
+          split_params[:object_index] += 1
+          count_index_files(entries_per_thread_count, threads_count)
+        end
+      end
+
+      def count_index_files(objects_per_thread_count, threads_count)
+        if split_params[:object_index] == objects_per_thread_count
+          split_params[:object_index] = 0
+          set_current_thread(threads_count)
+        end
+      end
+
+      def set_current_thread(threads_count)
+        split_params[:current_thread] += 1
+        split_params[:current_thread] = 0 if split_params[:current_thread] == threads_count
+      end
+
+      def has_contentful_structure?(collection_file)
+        File.exist?("#{config.collections_dir}/#{collection_file}.json")
+      end
+
+      def content_type_id_from_file(collection_file)
+        JSON.parse(File.read("#{config.collections_dir}/#{collection_file}.json"))['id']
+      end
+
+      def new_entry_name(content_type_id, entry_path)
+        "#{content_type_id}_#{File.basename(entry_path, '.*').match(/(\d+)/)[0]}.json"
+      end
+
+      def copy_entry(entry_path, current_thread, content_type_id)
+        FileUtils.cp entry_path, "#{config.threads_dir}/#{current_thread}/#{new_entry_name(content_type_id, entry_path)}"
+      end
+
+      def copy_asset(asset_path)
+        FileUtils.cp asset_path, "#{config.threads_dir}/assets/#{split_params[:current_thread]}/#{File.basename(asset_path)}"
+      end
+
+      def create_threads_subdirectories(threads_count, validate, dir = {})
+        validate_collections_files if validate
+        create_directory(config.threads_dir)
+        threads_count.times do |thread_id|
+          create_directory("#{config.threads_dir}/#{dir[:assets]}#{thread_id}")
+        end
+      end
+
+      def create_directory(path)
+        FileUtils.mkdir_p(path) unless File.directory?(path)
+      end
+
+      def total_entries_count
+        total_number = 0
+        Dir.glob("#{config.entries_dir}/*") do |dir_path|
+          collection_name = File.basename(dir_path)
+          total_number += Dir.glob("#{config.entries_dir}/#{collection_name}/*").count if has_contentful_structure?(collection_name)
+        end
+        total_number
+      end
+
+      def total_assets_count
+        Dir.glob("#{config.assets_dir}/**/*json").count
+      end
+
+      def validate_collections_files
+        fail ArgumentError, "Make sure the #{config.collections_dir} directory exists and the content structure resides within it. View README" unless Dir.exist?(config.collections_dir)
+        fail ArgumentError, 'Collections directory is empty! Create content types JSON files. View README' if Dir.glob("#{config.collections_dir}/*").empty?
+      end
+    end
+  end
+end

data/lib/contentful/importer/json_schema_validator.rb

@@ -0,0 +1,66 @@
+require 'json-schema'
+
+module Contentful
+  module Importer
+    class JsonSchemaValidator
+
+      attr_reader :config, :logger
+
+      def initialize(configuration)
+        @config = configuration
+        @logger = Logger.new(STDOUT)
+      end
+
+      def validate_schemas
+        Dir.glob("#{config.collections_dir}/*") do |content_type_file|
+          validate_schema(content_type_file)
+        end
+      end
+
+      def validate_schema(content_type_file)
+        schema = parse_content_type_schema(JSON.parse(File.read(content_type_file)))
+        content_type_filename = File.basename(content_type_file, '.*')
+        validate_entry(content_type_filename, schema)
+      end
+
+      def validate_entry(content_type_filename, schema)
+        Dir.glob("#{config.entries_dir}/#{content_type_filename}/*") do |entry_file|
+          entry_schema = JSON.parse(File.read(entry_file))
+          begin
+            JSON::Validator.validate!(schema, entry_schema)
+          rescue JSON::Schema::ValidationError => error
+            logger.info "#{error.message}! Path to invalid entry: #{entry_file}"
+          end
+        end
+      end
+
+      def parse_content_type_schema(ct_file)
+        new_hash = base_schema_format
+        ct_file['fields'].each do |key|
+          type = convert_type(key['type'])
+          new_hash['properties'].merge!({key['id'] => {'type' => type}})
+        end
+        new_hash
+      end
+
+      def base_schema_format
+        {'type' => 'object', 'properties' => {}}
+      end
+
+      def convert_type(type)
+        case type
+          when 'Text', 'Date', 'Symbol'
+            'string'
+          when 'Number'
+            'float'
+          when 'Asset', 'Entry'
+            'object'
+          else
+            type.downcase
+        end
+      end
+
+    end
+  end
+end
+

data/lib/contentful/importer/migrator.rb

@@ -0,0 +1,43 @@
+require_relative 'parallel_importer'
+require_relative 'configuration'
+require_relative 'converters/contentful_model_to_json'
+require_relative 'json_schema_validator'
+
+module Contentful
+  module Importer
+    class Migrator
+
+      attr_reader :importer, :converter, :config, :json_validator
+
+      def initialize(settings)
+        @config = Configuration.new(settings)
+        @importer = ParallelImporter.new(@config)
+        @converter = ContentfulModelToJson.new(@config)
+        @json_validator = JsonSchemaValidator.new(@config)
+      end
+
+      def run(action, options = {})
+        case action.to_s
+          when '--create-contentful-model-from-json'
+            converter.create_content_type_json
+          when '--import-content-types'
+            importer.create_contentful_model(options)
+          when '--import'
+            importer.import_data(options[:threads])
+          when '--convert-content-model-to-json'
+            converter.convert_to_import_form
+          when '--publish-entries'
+            importer.publish_entries_in_threads
+          when '--test-credentials'
+            importer.test_credentials
+          when '--import-assets'
+            importer.import_only_assets
+          when '--publish-assets'
+            importer.publish_assets_in_threads(options[:threads])
+          when '--validate-schema'
+            json_validator.validate_schemas
+        end
+      end
+    end
+  end
+end