drupal-exporter 0.0.1

data/drupal_exporter.gemspec

@@ -0,0 +1,34 @@
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require File.expand_path('../lib/version', __FILE__)
+
+Gem::Specification.new do |spec|
+  spec.name          = 'drupal-exporter'
+  spec.version       = Version::VERSION
+  spec.authors       = ['Contentful GmbH (Andreas Tiefenthaler)']
+  spec.email         = ['rubygems@contentful.com']
+  spec.description   = 'Drupal exporter that prepares content to be imported'
+  spec.summary       = 'Exporter for Drupal 7'
+  spec.homepage      = 'https://github.com/contentful/drupal-exporter.rb'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables    << 'drupal-exporter'
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'http', '~> 0.6'
+  spec.add_dependency 'multi_json', '~> 1'
+  spec.add_dependency 'sequel','~> 4.15'
+  spec.add_dependency 'mysql2','~> 0.3'
+  spec.add_dependency 'activesupport','~> 4.1'
+  spec.add_dependency 'pg', '~> 0.17.0'
+  spec.add_dependency 'escort','~> 0.4.0'
+  spec.add_dependency 'i18n', '~> 0.6'
+
+  spec.add_development_dependency 'rspec', '~> 3'
+  spec.add_development_dependency 'rspec-its', '~> 1.1.0'
+  spec.add_development_dependency 'bundler', '~> 1.6'
+  spec.add_development_dependency 'rake'
+end

data/drupal_settings/boolean_columns.yml

@@ -0,0 +1,2 @@
+- field_if_content_type
+- field_boolean

data/drupal_settings/drupal_content_types.json

@@ -0,0 +1,47 @@
+// machine_name_of_content_type : {
+//  contentful_api_field : column_machine_name
+// }
+
+{
+  "article": {
+    "body": "body",
+    "image": "field_image"
+  },
+  "page": {
+    "body": "body"
+  },
+  "blog": {
+    "body": "body"
+  },
+  "content_type": {
+    "body": "body",
+    "age": "field_age",
+    "if_content_type": "field_if_content_type",
+    "decimal": "field_decimal"
+  },
+  "second_content_type": {
+    "body": "body",
+    "firma": "field_firma"
+  },
+  "content_all_types": {
+    "body": "body",
+    "file": "field_file",
+    "image_sec": "field_sec_image",
+    "integer": "field_integer",
+    "boolean": "field_boolean",
+    "decimal": "field_sec_decimal",
+    "list_float": "field_list_float",
+    "list_integer": "field_list_integer",
+    "list_text": "field_list_text",
+    "long_text": "field_long_text",
+    "text_summary": "field_text_summary",
+    "term_tagging": {
+      "table": "field_term_tagging"
+    },
+    "text": "field_text"
+  },
+  "assety": {
+    "body": "body",
+    "assets": "field_asset"
+  }
+}

data/drupal_settings/drupal_settings.yml

@@ -0,0 +1,23 @@
+#PATH to all data
+data_dir: PATH_WHERE_ALL_DATA_WILL_BE_SAVED
+
+########## EXPORT DATA ##########
+
+#Connecting to a database
+adapter: mysql2
+host: localhost
+database: drupal_database_name
+user: username
+password: password
+
+# DRUPAL
+
+drupal_content_types_json: drupal_settings/drupal_content_types.json
+drupal_boolean_columns: drupal_settings/boolean_columns.yml
+drupal_base_url: http://example_hostname.com
+
+# CONVERT CONTENTFUL MODEL TO CONTENTFUL IMPORT STRUCTURE
+content_model_json: PATH_TO_CONTENTFUL_MODEL_JSON_FILE
+converted_model_dir: PATH_WHERE_CONVERTED_CONTENT_MODEL_WILL_BE_SAVED
+
+contentful_structure_dir: PATH_TO_CONTENTFUL_STRUCUTRE_JSON_FILE

data/lib/cli.rb

@@ -0,0 +1,13 @@
+require_relative 'migrator'
+require 'yaml'
+
+module Command
+  class CLI < Escort::ActionCommand::Base
+
+    def execute
+      setting_file = YAML.load_file(global_options[:file])
+      Migrator.new(setting_file).run(command_name)
+    end
+
+  end
+end

data/lib/configuration.rb

@@ -0,0 +1,46 @@
+require 'sequel'
+require 'active_support/core_ext/hash'
+
+module Contentful
+  class Configuration
+    attr_reader :space_id,
+                :config,
+                :data_dir,
+                :collections_dir,
+                :entries_dir,
+                :assets_dir,
+                :db,
+                :drupal_content_types,
+                :drupal_base_url
+
+    def initialize(settings)
+      @config = settings
+      validate_required_parameters
+      @data_dir = config['data_dir']
+      @collections_dir = "#{data_dir}/collections"
+      @entries_dir = "#{data_dir}/entries"
+      @assets_dir = "#{data_dir}/assets"
+      @space_id = config['space_id']
+      @drupal_content_types = JSON.parse(File.read(config['drupal_content_types_json']), symbolize_names: true).with_indifferent_access
+      @drupal_base_url = config['drupal_base_url']
+      @db = adapter_setup
+    end
+
+    def validate_required_parameters
+      fail ArgumentError, 'Set PATH to data_dir, the destination for all generated files. Check README' if config['data_dir'].nil?
+      fail ArgumentError, 'Set PATH to drupal_content_types_json. File with Drupal database structure. View README' if config['drupal_content_types_json'].nil?
+      define_adapter
+    end
+
+    def define_adapter
+      %w(adapter user host database).each do |param|
+        fail ArgumentError, "Set database connection parameters [adapter, host, database, user, password]. Missing the '#{param}' parameter! Password is optional. Check README!" unless config[param]
+      end
+    end
+
+    def adapter_setup
+      Sequel.connect(:adapter => config['adapter'], :user => config['user'], :host => config['host'], :database => config['database'], :password => config['password'])
+    end
+
+  end
+end

data/lib/converters/content_types_structure_creator.rb

@@ -0,0 +1,60 @@
+module Contentful
+  module Converter
+    class ContentTypesStructureCreator
+
+      attr_reader :config, :logger
+
+      def initialize(config)
+        @config = config
+        @logger = Logger.new(STDOUT)
+      end
+
+      def create_content_type_json_file(content_type_name, values)
+        collection = {
+            id: values[:id],
+            name: values[:name],
+            description: values[:description],
+            displayField: values[:displayField],
+            fields: create_fields(values[:fields])
+        }
+        write_json_to_file("#{config.collections_dir}/#{content_type_name}.json", collection)
+        logger.info "Saving #{content_type_name}.json to #{config.collections_dir}"
+      end
+
+      def create_fields(fields)
+        fields.each_with_object([]) do |(field, value), results|
+          results << {
+              name: create_field(field, value).capitalize,
+              id: create_field(field, value),
+              type: create_type_field(value),
+              link_type: create_link_type_field(value),
+              link: create_link_field(value)
+          }.compact
+        end
+      end
+
+      def create_field(field, value)
+        value.is_a?(Hash) ? value[:id] : field
+      end
+
+      def create_link_type_field(value)
+        value.is_a?(Hash) ? value[:link_type] : nil
+      end
+
+      def create_type_field(value)
+        value.is_a?(Hash) ? value[:type] : value
+      end
+
+      def create_link_field(value)
+        value.is_a?(Hash) ? value[:link] : nil
+      end
+
+      def write_json_to_file(path, data)
+        File.open(path, 'w') do |file|
+          file.write(JSON.pretty_generate(data))
+        end
+      end
+
+    end
+  end
+end

data/lib/converters/contentful_model_to_json.rb

@@ -0,0 +1,109 @@
+require_relative 'content_types_structure_creator'
+
+module Contentful
+  module Converter
+    class ContentfulModelToJson
+      attr_reader :config, :logger, :converted_model_dir, :content_types
+
+      FIELD_TYPE = %w( Link Array )
+
+      def initialize(settings)
+        @config = settings
+        @logger = Logger.new(STDOUT)
+      end
+
+      def create_content_type_json
+        contentful_structure = load_contentful_structure_file
+        logger.info 'Create JSON files with content types structure...'
+        contentful_structure.each do |content_type, values|
+          content_type_name = content_type_name(content_type)
+          create_directory(config.collections_dir)
+          ContentTypesStructureCreator.new(config).create_content_type_json_file(content_type_name, values)
+        end
+        logger.info 'Done!'
+      end
+
+      def convert_to_import_form
+        set_content_model_parameters
+        logger.info 'Converting Contentful model to Contentful import structure...'
+        File.open(converted_model_dir, 'w') { |file| file.write({}) }
+        content_type_file = JSON.parse(File.read(content_types))['items']
+        content_type_file.each do |content_type|
+          parsed_content_type = {
+              id: content_type['sys']['id'],
+              name: content_type['name'],
+              description: content_type['description'],
+              displayField: content_type['displayField'],
+              fields: create_content_type_fields(content_type)
+          }
+          import_form = JSON.parse(File.read(converted_model_dir))
+          File.open(converted_model_dir, 'w') do |file|
+            file.write(JSON.pretty_generate(import_form.merge!(content_type['name'] => parsed_content_type)))
+          end
+        end
+        logger.info "Done! Contentful import structure file saved in #{converted_model_dir}"
+      end
+
+      def create_content_type_fields(content_type)
+        content_type['fields'].each_with_object({}) do |(field, _value), results|
+          id = link_id(field)
+          results[id] = case field['type']
+                          when 'Link'
+                            {id: field['id'], type: field['linkType'], link: 'Link'}
+                          when 'Array'
+                            {id: field['id'], type: field['type'], link_type: field['items']['linkType'], link: field['items']['type']}
+                          else
+                            field['type']
+                        end
+        end
+      end
+
+      def link_id(field)
+        if FIELD_TYPE.include? field['type']
+          field['name'].capitalize
+        else
+          field['id']
+        end
+      end
+
+      def content_type_name(content_type)
+        I18n.transliterate(content_type).underscore.tr(' ', '_')
+      end
+
+      def create_directory(path)
+        FileUtils.mkdir_p(path) unless File.directory?(path)
+      end
+
+      # If contentful_structure JSON file exists, it will load the file. If not, it will automatically create an empty file.
+      # This file is required to convert contentful model to contentful import structure.
+      def load_contentful_structure_file
+        fail ArgumentError, 'Set PATH for contentful structure JSON file. View README' if config.config['contentful_structure_dir'].nil?
+        file_exists? ? load_existing_contentful_structure_file : create_empty_contentful_structure_file
+      end
+
+      def file_exists?
+        File.exists?(config.config['contentful_structure_dir'])
+      end
+
+      def create_empty_contentful_structure_file
+        File.open(config.config['contentful_structure_dir'], 'w') { |file| file.write({}) }
+        load_existing_contentful_structure_file
+      end
+
+      def load_existing_contentful_structure_file
+        JSON.parse(File.read(config.config['contentful_structure_dir']), symbolize_names: true).with_indifferent_access
+      end
+
+      def set_content_model_parameters
+        validate_content_model_files
+        @converted_model_dir = config.config['converted_model_dir']
+        @content_types = config.config['content_model_json']
+      end
+
+      def validate_content_model_files
+        fail ArgumentError, 'Set PATH for content model JSON file. View README' if config.config['content_model_json'].nil?
+        fail ArgumentError, 'Set PATH where converted content model file will be saved. View README' if config.config['converted_model_dir'].nil?
+      end
+    end
+  end
+end

data/lib/drupal/content_type.rb

@@ -0,0 +1,151 @@
+module Contentful
+  module Exporter
+    module Drupal
+      class ContentType
+
+        attr_reader :exporter, :config, :type, :schema
+
+        def initialize(exporter, config, type, schema)
+          @exporter = exporter
+          @config = config
+          @type = type
+          @schema = schema
+        end
+
+        def save_content_types_as_json
+          exporter.create_directory("#{config.entries_dir}/#{type}")
+          config.db[:node].where(type: type).each do |content_row|
+            extract_data(content_row)
+          end
+        end
+
+        private
+
+        def extract_data(content_row)
+          puts "Saving #{type} - id: #{content_row[:nid]}"
+          db_object = map_fields(content_row)
+          exporter.write_json_to_file("#{config.entries_dir}/#{type}/#{db_object[:id]}.json", db_object)
+        end
+
+        def map_fields(row, result = {})
+          result.merge!(set_default_data(row))
+          result.merge!(find_related_data(row))
+          result
+        end
+
+        def id(content_id)
+          "#{type}_#{content_id}"
+        end
+
+        def author(user_id)
+          {type: 'Author', id: "user_#{user_id}"}
+        end
+
+        def tags(entity_row_id)
+          entity_tags(entity_row_id).each_with_object([]) do |tag, tags|
+            linked_tag = {type: 'EntryTag', id: "tag_#{tag[:field_tags_tid]}"}
+            tags << linked_tag
+          end
+        end
+
+        def entity_tags(entity_id)
+          config.db[:field_data_field_tags].where(entity_id: entity_id)
+        end
+
+        def set_default_data(row, result = {})
+          result[:id] = id(row[:nid])
+          result[:title] = row[:title]
+          result[:author] = author(row[:uid])
+          result[:tags] = tags(row[:nid]) unless tags(row[:nid]).empty?
+          result[:created_at] = created_at(row[:created])
+          result
+        end
+
+        def find_related_data(row, result = {})
+          schema.each do |key, column_name|
+            result[key] = column_name.is_a?(String) ? fetch_data_from_related_table(row[:nid], column_name) : fetch_custom_tags(row[:nid], column_name)
+          end
+          result
+        end
+
+        def fetch_data_from_related_table(entity_id, table_name)
+          related_row = get_related_row(entity_id, table_name)
+          fetch_data_from_related_row(related_row, table_name)
+        end
+
+        def fetch_data_from_related_row(related_row, table_name)
+          respond_to_file?(related_row, table_name) ? get_file_id(related_row, table_name) : related_value(related_row, table_name)
+        end
+
+        def respond_to_file?(related_row, table_name)
+          file_id = "#{table_name}_fid".to_sym
+          (related_row.present? && related_row.first[file_id]) ? true : false
+        end
+
+        def get_file_id(related_row, table_name)
+          file_key = "#{table_name}_fid".to_sym
+          file_id = related_row.first[file_key]
+          file_asset_id = file_id(file_id)
+          link_asset_to_content_type(file_asset_id)
+        end
+
+        def file_id(file_id)
+          config.db[:file_managed].where(fid: file_id).first[:fid]
+        end
+
+        def get_related_row(entity_id, table_name)
+          config.db[related_table_name(table_name)].where(entity_id: entity_id)
+        end
+
+        def link_asset_to_content_type(file_asset_id)
+          {type: 'File', id: "file_#{file_asset_id}"}
+        end
+
+        def related_value(related_rows, table_name)
+          value = related_rows.empty? ? nil : related_rows.first[field_name(table_name)]
+          convert_type_value(value, table_name)
+        end
+
+        def fetch_custom_tags(entity_id, table_name)
+          custom_tag_table = "field_data_#{table_name['table']}".to_sym
+          tag_id = "#{table_name['table']}_tid".to_sym
+          config.db[custom_tag_table].where(entity_id: entity_id).each_with_object([]) do |content_tag, tags|
+            lined_tags = {type: 'EntryTag', id: "tag_#{content_tag[tag_id]}"}
+            tags << lined_tags
+          end
+        end
+
+        def related_table_name(table_name)
+          "field_data_#{table_name}".to_sym
+        end
+
+        def field_name(table_name)
+          "#{table_name}_value".to_sym
+        end
+
+        def created_at(timestamp)
+          Time.at(timestamp).to_datetime
+        end
+
+        def convert_type_value(value, column_name)
+          if value.is_a?(BigDecimal)
+            value.to_f
+          elsif boolean_column?(column_name)
+            convert_boolean_value(value)
+          else
+            value
+          end
+        end
+
+        def boolean_column?(column_name)
+          exporter.boolean_columns && exporter.boolean_columns.flatten.include?(column_name) ? true : false
+        end
+
+        def convert_boolean_value(value)
+          value.nil? ? nil : (value == 1 ? true : false)
+        end
+
+      end
+    end
+  end
+end