database-exporter 0.0.1

data/example_data/example_settings.yml

@@ -0,0 +1,25 @@
+#PATH to all data
+data_dir: PATH_TO_ALL_DATA
+
+#Connecting to a database
+adapter: postgres
+host: localhost
+database: database_name
+user: username
+password: password
+
+# Extract data from models:
+mapped:
+  tables:
+  - :table_name_1
+  - :table_name_2
+  - :table_name_3
+  - :table_name_4
+
+## MAPPING ##
+mapping_dir: example_data/mapping.json
+contentful_structure_dir: example_data/contentful_structure.json
+
+## CONVERT
+content_model_json: example_data/contentful_model.json
+converted_model_dir: example_data/contentful_structure.json

data/example_data/mapping.json

@@ -0,0 +1,119 @@
+{
+  "JobAdds": {
+    "content_type": "Jobs",
+    "type": "entry",
+    "fields": {
+      "title": "name",
+      "description": "specification"
+    },
+    "links": {
+      "aggregate_belongs": [
+        {
+          "relation_to": "Users",
+          "primary_id": "user_id",
+          "field": "first_name"
+        }
+      ],
+      "aggregate_many": [
+        {
+          "primary_id": "job_add_id",
+          "relation_to": "Comments",
+          "field": "subject",
+          "save_as": "subjects_comments"
+        }
+      ],
+      "belongs_to": [
+        {
+          "relation_to": "Images",
+          "foreign_id": "image_id"
+        },
+        {
+          "relation_to": "Users",
+          "foreign_id": "user_id"
+        }
+      ],
+      "many": [
+        {
+          "relation_to": "Comments",
+          "primary_id": "job_add_id"
+        }
+      ],
+      "many_through": [
+        {
+          "relation_to": "Skills",
+          "primary_id": "job_add_id",
+          "foreign_id": "skill_id",
+          "through": "JobAddSkills"
+        }
+      ]
+    }
+  },
+  "Comments": {
+    "content_type": "Comment",
+    "type": "entry",
+    "fields": {
+      "title": "subject",
+      "body": "content"
+    },
+    "links": {
+    }
+  },
+  "Skills": {
+    "content_type": "Job Skills",
+    "type": "entry",
+    "fields": {
+    },
+    "links": {
+    }
+  },
+  "Profiles": {
+    "content_type": "Profile",
+    "type": "entry",
+    "fields": {
+    },
+    "links": {
+      "belongs_to": [
+        {
+          "relation_to": "Users",
+          "foreign_id": "user_id"
+        }
+      ]
+    }
+  },
+  "Users": {
+    "content_type": "User",
+    "type": "entry",
+    "fields": {
+    },
+    "links": {
+      "aggregate_has_one": [
+        {
+          "relation_to": "Profiles",
+          "primary_id": "user_id",
+          "field": "nickname",
+          "save_as": "custom_nick"
+        }
+      ],
+      "has_one": [
+        {
+          "relation_to": "Profiles",
+          "primary_id": "user_id"
+        }
+      ],
+      "many": [
+        {
+          "relation_to": "JobAdds",
+          "primary_id": "user_id"
+        }
+      ]
+    }
+  },
+  "Images": {
+    "content_type": "Image",
+    "type": "asset",
+    "fields": {
+    },
+    "links": {
+    }
+  }
+}

data/lib/cli.rb

@@ -0,0 +1,13 @@
+require_relative 'migrator'
+require 'yaml'
+
+module Command
+  class CLI < Escort::ActionCommand::Base
+
+    def execute
+      setting_file = YAML.load_file(global_options[:file])
+      Migrator.new(setting_file).run(command_name)
+    end
+
+  end
+end

data/lib/configuration.rb

@@ -0,0 +1,69 @@
+require 'sequel'
+require 'active_support/core_ext/hash'
+
+module Contentful
+  class Configuration
+    attr_reader :config,
+                :data_dir,
+                :collections_dir,
+                :entries_dir,
+                :assets_dir,
+                :contentful_structure,
+                :db,
+                :helpers_dir,
+                :converted_model_dir,
+                :content_types
+
+    def initialize(settings)
+      @config = settings
+      validate_required_parameters
+      @data_dir = config['data_dir']
+      @collections_dir = "#{data_dir}/collections"
+      @entries_dir = "#{data_dir}/entries"
+      @assets_dir = "#{data_dir}/assets"
+      @helpers_dir = "#{data_dir}/helpers"
+      @contentful_structure = load_contentful_structure_file
+      @db = adapter_setup
+      @converted_model_dir = settings['converted_model_dir']
+      @content_types = config['content_model_json']
+    end
+
+    def validate_required_parameters
+      fail ArgumentError, 'Set PATH to data_dir. Folder where all data will be stored. View README' if config['data_dir'].nil?
+      fail ArgumentError, 'Set PATH to contentful structure JSON file. View README' if config['contentful_structure_dir'].nil?
+      fail ArgumentError, 'Set PATH to mapping structure JSON file. View README' if config['mapping_dir'].nil?
+      fail ArgumentError, 'Set PATH to Content model JSON file, which is downloaded structure from Contentful. View README' if config['converted_model_dir'].nil?
+      fail ArgumentError, 'Set PATH to converted contentful model and saved as JSON file. View README' if config['content_model_json'].nil?
+      define_adapter
+    end
+
+    def define_adapter
+      %w(adapter host database).each do |param|
+        fail ArgumentError, "Set database connection parameters [adapter, host, database, user, password]. Missing the '#{param}' parameter! Password and User are optional. View README!" unless config[param]
+      end
+    end
+
+    # If contentful_structure JSON file exists, it will load the file. If not, it will automatically create an empty file.
+    # This file is required to convert contentful model to contentful import structure.
+    def load_contentful_structure_file
+      file_exists? ? load_existing_contentful_structure_file : create_empty_contentful_structure_file
+    end
+
+    def file_exists?
+      File.exists?(config['contentful_structure_dir'])
+    end
+
+    def create_empty_contentful_structure_file
+      File.open(settings['contentful_structure_dir'], 'w') { |file| file.write({}) }
+      load_existing_contentful_structure_file
+    end
+
+    def load_existing_contentful_structure_file
+      JSON.parse(File.read(config['contentful_structure_dir']), symbolize_names: true).with_indifferent_access
+    end
+
+    def adapter_setup
+      Sequel.connect(:adapter => config['adapter'], :user => config['user'], :host => config['host'], :database => config['database'], :password => config['password'])
+    end
+  end
+end

data/lib/converters/content_types_structure_creator.rb

@@ -0,0 +1,58 @@
+module Contentful
+  module Converter
+    class ContentTypesStructureCreator
+
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      def create_content_type_json_file(content_type_name, values)
+        collection = {
+            id: values[:id],
+            name: values[:name],
+            description: values[:description],
+            displayField: values[:displayField],
+            fields: create_fields(values[:fields])
+        }
+        write_json_to_file("#{config.collections_dir}/#{content_type_name}.json", collection)
+      end
+
+      def create_fields(fields)
+        fields.each_with_object([]) do |(field, value), results|
+          results << {
+              name: create_field(field, value).capitalize,
+              id: create_field(field, value),
+              type: create_type_field(value),
+              link_type: create_link_type_field(value),
+              link: create_link_field(value)
+          }.compact
+        end
+      end
+
+      def create_field(field, value)
+        value.is_a?(Hash) ? value[:id] : field
+      end
+
+      def create_link_type_field(value)
+        value.is_a?(Hash) ? value[:link_type] : nil
+      end
+
+      def create_type_field(value)
+        value.is_a?(Hash) ? value[:type] : value
+      end
+
+      def create_link_field(value)
+        value.is_a?(Hash) ? value[:link] : nil
+      end
+
+      def write_json_to_file(path, data)
+        File.open(path, 'w') do |file|
+          file.write(JSON.pretty_generate(data))
+        end
+      end
+
+    end
+  end
+end

data/lib/converters/contentful_model_to_json.rb

@@ -0,0 +1,78 @@
+require_relative 'content_types_structure_creator'
+
+module Contentful
+  module Converter
+    class ContentfulModelToJson
+
+      attr_reader :config, :logger
+
+      FIELD_TYPE = %w( Link Array )
+
+      def initialize(settings)
+        @config = settings
+        @logger = Logger.new(STDOUT)
+      end
+
+      def create_content_type_json
+        logger.info 'Create JSON files with content types structure...'
+        config.contentful_structure.each do |content_type, values|
+          content_type_name = content_type_name(content_type)
+          create_directory(config.collections_dir)
+          ContentTypesStructureCreator.new(config).create_content_type_json_file(content_type_name, values)
+        end
+        logger.info 'Done!'
+      end
+
+      def convert_to_import_form
+        logger.info 'Converting Contentful model to Contentful import structure...'
+        File.open(config.converted_model_dir, 'w') { |file| file.write({}) }
+        content_type_file = JSON.parse(File.read(config.content_types))['items']
+        content_type_file.each do |content_type|
+          parsed_content_type = {
+              id: content_type['sys']['id'],
+              name: content_type['name'],
+              description: content_type['description'],
+              displayField: content_type['displayField'],
+              fields: create_content_type_fields(content_type)
+          }
+          import_form = JSON.parse(File.read(config.converted_model_dir))
+          File.open(config.converted_model_dir, 'w') do |file|
+            file.write(JSON.pretty_generate(import_form.merge!(content_type['name'] => parsed_content_type)))
+          end
+        end
+        logger.info "Done! Contentful import structure file saved in #{config.converted_model_dir}"
+      end
+
+      def create_content_type_fields(content_type)
+        content_type['fields'].each_with_object({}) do |(field, _value), results|
+          id = link_id(field)
+          results[id] = case field['type']
+                          when 'Link'
+                            {id: field['id'], type: field['linkType'], link: 'Link'}
+                          when 'Array'
+                            {id: field['id'], type: field['type'], link_type: field['items']['linkType'], link: field['items']['type']}
+                          else
+                            field['type']
+                        end
+        end
+      end
+
+      def link_id(field)
+        if FIELD_TYPE.include? field['type']
+          field['name'].capitalize
+        else
+          field['id']
+        end
+      end
+
+      def content_type_name(content_type)
+        I18n.transliterate(content_type).underscore.tr(' ', '_')
+      end
+
+      def create_directory(path)
+        FileUtils.mkdir_p(path) unless File.directory?(path)
+      end
+
+    end
+  end
+end

data/lib/database/export.rb

@@ -0,0 +1,74 @@
+require 'active_support/core_ext/string'
+require 'active_support/core_ext/hash/compact'
+require 'active_support/core_ext/hash'
+require 'fileutils'
+require 'sequel'
+require 'logger'
+require_relative 'modules/json_export'
+require_relative 'modules/relations_export'
+require_relative 'modules/utils'
+
+module Contentful
+  module Exporter
+    module Database
+      class Export
+
+        Encoding.default_external = 'utf-8'
+
+        include Contentful::Exporter::Database::JsonExport
+        include Contentful::Exporter::Database::RelationsExport
+        include Contentful::Exporter::Database::Utils
+
+        Sequel::Model.plugin :json_serializer
+        Sequel.datetime_class = DateTime
+
+        attr_reader :config, :mapping, :tables, :logger
+
+        def initialize(settings)
+          @config = settings
+          @mapping = mapping_structure
+          @tables = load_tables
+          @logger = Logger.new(STDOUT)
+        end
+
+        def tables_name
+          create_directory(config.data_dir)
+          write_json_to_file("#{config.data_dir}/table_names.json", config.db.tables)
+          logger.info "File with name of tables saved to #{"#{config.data_dir}/table_names.json"}"
+        end
+
+        def save_data_as_json
+          tables.each do |table|
+            logger.info "Extracting data from #{"#{table} table"}..."
+            model_name = table.to_s.camelize
+            fail ArgumentError, "Missing model name in your mapping.json file. To extract data from #{table}, define structure for this model in mapping.json file or remove #{table} from settings.yml - mapped tables! View README." if missing_model_structure?(model_name)
+            content_type_name = mapping[model_name][:content_type]
+            save_object_to_file(table, content_type_name, model_name, asset?(model_name) ? config.assets_dir : config.entries_dir)
+          end
+        end
+
+        def create_data_relations
+          relations_from_mapping.each do |model_name, relations|
+            generate_relations_helper_indexes(relations)
+            map_relations_to_links(model_name, relations)
+          end
+        end
+
+        def mapping_structure
+          fail ArgumentError, 'Set PATH to contentful structure JSON file. Check README' unless config.config['mapping_dir']
+          JSON.parse(File.read(config.config['mapping_dir']), symbolize_names: true).with_indifferent_access
+        end
+
+        def load_tables
+          fail ArgumentError, 'Before importing data from tables, define their names. Check README!' unless config.config['mapped'] && config.config['mapped']['tables']
+          config.config['mapped']['tables']
+        end
+
+        def missing_model_structure?(model_name)
+          mapping[model_name].nil?
+        end
+
+      end
+    end
+  end
+end

data/lib/database/modules/json_export.rb

@@ -0,0 +1,79 @@
+require 'i18n'
+
+module Contentful
+  module Exporter
+    module Database
+      module JsonExport
+
+        def asset?(model_name)
+          mapping[model_name] && mapping[model_name][:type] == 'asset'
+        end
+
+        def save_object_to_file(table, content_type_name, model_name, type)
+          content_type_name = I18n.transliterate(content_type_name).underscore.tr(' ', '_')
+          create_directory("#{type}/#{content_type_name}")
+          config.db[table].each_with_index do |row, index|
+            index = index + 1
+            result = transform_row_into_hash(model_name, content_type_name, row, index)
+            write_json_to_file("#{type}/#{content_type_name}/#{result[:id]}.json", result)
+          end
+        end
+
+        def transform_row_into_hash(model_name, content_type_name, row, index)
+          id = row[:id] || index
+          logger.info "Saving #{content_type_name} - id: #{id}"
+          db_object = map_fields(model_name, row)
+          db_object[:id] = model_id(model_name, content_type_name, id)
+          db_object[:database_id] = id
+          db_object
+        end
+
+        def model_id(model_name, content_type_name, id)
+          prefix = mapping[model_name][:prefix_id] || ''
+          "#{prefix}#{content_type_name}_#{id}"
+        end
+
+        def map_fields(model_name, row)
+          row.each_with_object({}) do |(field_name, field_value), result|
+            field_name = mapped_field_name(field_name, model_name)
+            formatted_value = formatted_field_value(field_name, field_value, model_name)
+            result[field_name] = formatted_value
+            copy_field_value(field_name, formatted_value, model_name, result) if copy_field?(field_name, model_name)
+          end
+        end
+
+        def formatted_field_value(field_name, field_value, model_name)
+          has_mapping_value?(field_name, model_name) ? format_value(field_value) : field_value
+        end
+
+        def copy_field_value(field_name, field_value, model_name, result)
+          copy_field = mapping[model_name][:copy][field_name]
+          result[copy_field] = format_value(field_value.to_s)
+        end
+
+        def format_value(field_value)
+          formatted_value = I18n.transliterate(field_value).tr(' ', '_').underscore
+          formatted_value.underscore.gsub(/\W/, '-').gsub(/\W\z/, '').gsub(/\A\W/, '').gsub('_', '-').gsub('--', '-').gsub('--', '-')
+        end
+
+        def mapped_field_name(field_name, model_name)
+          has_mapping_for?(field_name, model_name) ? mapping[model_name][:fields][field_name] : field_name
+        end
+
+        def has_mapping_for?(field_name, model_name)
+          mapping[model_name] && mapping[model_name][:fields][field_name].present?
+        end
+
+        def has_mapping_value?(field_name, model_name)
+          mapping[model_name] && mapping[model_name][:format] && mapping[model_name][:format][field_name].present?
+        end
+
+        def copy_field?(field_name, model_name)
+          mapping[model_name] && mapping[model_name][:copy] && mapping[model_name][:copy][field_name].present?
+        end
+
+      end
+    end
+  end
+end
+