database-exporter 0.0.1

data/spec/lib/migrator_spec.rb

@@ -0,0 +1,112 @@
+require 'spec_helper'
+require './lib/migrator'
+
+describe Migrator do
+
+  before do
+    @setting_file = YAML.load_file('spec/fixtures/settings/settings.yml')
+  end
+
+  it 'Fetching data from DB and mapping process' do
+    FileUtils.rm_rf('spec/fixtures/database/data')
+
+    # Extract data from database
+    Migrator.new(@setting_file).run('--extract-to-json')
+
+    expect(Dir.glob('spec/fixtures/database/data/entries/*').count).to eq 5
+    expect(Dir.glob('spec/fixtures/database/data/**/*').count).to eq 34
+
+    directory_names = %w(comment job_skills jobs profile user)
+    Dir.glob('spec/fixtures/database/data/entries/*') do |directory_name|
+      expect(directory_names.include?(File.basename(directory_name))).to be true
+    end
+
+    expect(load_fixture('database/data/entries/comment/comment_1')).to include(id: 'comment_1', subject: 'Title comment 1', content: 'Body comment 1', job_add_id: 1, database_id: 1)
+    expect(load_fixture('database/data/entries/job_skills/job_skills_1')).to include(id: 'job_skills_1', name: 'Commercial awareness', database_id: 1)
+    expect(load_fixture('database/data/entries/jobs/jobs_4')).to include(id: 'jobs_4', name: 'Awesome job', specification: 'One word - awesome!', image_id: 2, user_id: 2, database_id: 4)
+    expect(load_fixture('database/data/entries/profile/profile_1')).to include(id: 'profile_1', nickname: 'Nickname 1', user_id: 1)
+    expect(load_fixture('database/data/entries/user/user_1')).to include(id: 'user_1', first_name: 'FirstName', last_name: 'LastName', birthday: '2009-04-16T09:43:00+00:00', database_id: 1)
+
+    # Mapping
+    Migrator.new(@setting_file).run('--prepare-json')
+
+    expect(Dir.glob('spec/fixtures/database/data/**/*').count).to eq 39
+
+    user = load_fixture('database/data/entries/user/user_1')
+    expect(user['profile']).to be_a Hash
+    expect(user['profile']).to include(type: 'profile', id: 'profile_1')
+    expect(user['job_adds'].count).to eq 2
+    expect(user['job_adds']).to be_a Array
+    expect(user['job_adds'].first).to include(type: 'jobs', id: 'jobs_1')
+
+    profile = load_fixture('database/data/entries/profile/profile_1')
+    expect(profile['user']).to be_a Hash
+    expect(profile['user']).to include(type: 'Entry', id: 'user_1')
+
+    job = load_fixture('database/data/entries/jobs/jobs_1')
+    expect(job).to include first_name: 'FirstName'
+    expect(job['image']).to be_a Hash
+    expect(job['image']).to include(type: 'File', id: 'image_3')
+    expect(job['creator']).to be_a Hash
+    expect(job['creator']).to include(type: 'Entry', id: 'user_1')
+    expect(job['comments'].count).to eq 3
+    expect(job['comments']).to be_a Array
+    expect(job['comments'].first).to include(type: 'comment', id: 'comment_1')
+    expect(job['skills']).to be_a Array
+    expect(job['skills'].first).to include(type: 'job_skills', id: 'job_skills_1')
+    expect(job['subjects_comments']).to be_a Array
+    expect(job['subjects_comments']).to include('Title comment 1', 'Title comment 2', 'Title comment 3')
+  end
+
+  it 'list tables' do
+    Migrator.new(@setting_file).run('--list-tables')
+    tables = load_json('database/data/table_names')
+    expect(tables).to be_a Array
+    expect(tables.count).to eq 8
+    expect(tables).to include('schema_migrations', 'skills', 'job_adds', 'comments', 'images', 'job_add_skills', 'users', 'profiles')
+  end
+
+  it 'convert contentful model to contentful structure' do
+    Migrator.new(@setting_file).run('--convert-content-model-to-json')
+    contentful_structure = load_fixture('settings/contentful_structure_test')
+    expect(contentful_structure.count).to eq 5
+    expect(contentful_structure['Jobs']).to include(id: '4L1bg4WQ5aWQMiE82ouag', name: 'Jobs', displayField: 'title', description: nil)
+    expect(contentful_structure['Jobs']['fields'].count).to eq 5
+    expect(contentful_structure['Jobs']['fields']['Image']).to include(id: 'image', type: 'Asset', link: 'Link')
+    expect(contentful_structure['Jobs']['fields']['Creator']).to include(id: 'creator', type: 'Entry', link: 'Link')
+    expect(contentful_structure['Jobs']['fields']['Comments']).to include(id: 'comments', type: 'Array', link_type: 'Entry', link: 'Link')
+  end
+
+  it 'create content type json files from contentful structure' do
+    Migrator.new(@setting_file).run('--create-content-model-from-json')
+    expect(Dir.glob('spec/fixtures/database/data/collections/*').count).to eq 5
+    content_types_files = %w(comment.json job_skills.json jobs.json profile.json user.json)
+    Dir.glob('spec/fixtures/database/data/collections/*') do |directory_name|
+      expect(content_types_files.include?(File.basename(directory_name))).to be true
+    end
+    comment = load_fixture('database/data/collections/comment')
+    expect(comment).to include(id: '6H6pGAV1PUsuoAW26Iu48W', name: 'Comment', displayField: 'subject')
+    expect(comment['fields'].count).to eq 2
+    expect(comment['fields'].first).to include(id: 'subject', name: 'Subject', type: 'Text')
+
+    job_skills = load_fixture('database/data/collections/job_skills')
+    expect(job_skills).to include(id: '2soCP557HGKoOOK0SqmMOm', name: 'Job Skills', displayField: 'name')
+    expect(job_skills['fields'].count).to eq 1
+    expect(job_skills['fields'].first).to include(id: 'name', name: 'Name', type: 'Text')
+
+    jobs = load_fixture('database/data/collections/jobs')
+    expect(jobs).to include(id: '4L1bg4WQ5aWQMiE82ouag', name: 'Jobs', displayField: 'title')
+    expect(jobs['fields'].count).to eq 6
+    expect(jobs['fields'].last).to include(id: 'skills', name: 'Skills', type: 'Array', link_type: 'Entry', link: 'Link')
+
+    profile = load_fixture('database/data/collections/profile')
+    expect(profile).to include(id: '4WFZh4MwC4Mc0EQWAeOY8A', name: 'Profile', displayField: nil)
+    expect(profile['fields'].count).to eq 2
+    expect(profile['fields'].first).to include(id: 'nickname', name: 'Nickname', type: 'Text')
+
+    user = load_fixture('database/data/collections/user')
+    expect(user).to include(id: '1TVvxCqoRq0qUYAOQuOqys', name: 'User', displayField: 'first_name')
+    expect(user['fields'].first).to include(id: 'first_name', name: 'First_name', type: 'Text')
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'simplecov'
+SimpleCov.start 'rails'
+
+require 'bundler/setup'
+Bundler.setup
+
+require 'rspec'
+require 'rspec/its'
+require 'yaml'
+require 'json'
+
+Dir[File.dirname(__FILE__) + '/support/*.rb'].each { |f| require f }

data/spec/support/db_rows_json.rb

@@ -0,0 +1,9 @@
+require 'multi_json'
+
+def load_fixture(file, _as_json = false)
+  JSON.parse(File.read(File.dirname(__FILE__) + "/../fixtures/#{file}.json")).with_indifferent_access
+end
+
+def load_json(file, _as_json = false)
+  JSON.parse(File.read(File.dirname(__FILE__) + "/../fixtures/#{file}.json"))
+end

data/spec/support/shared_configuration.rb

@@ -0,0 +1,27 @@
+require './lib/configuration'
+
+shared_context 'shared_configuration', :a => :b do
+  before do
+    yaml_text = <<-EOF
+          data_dir: spec/fixtures/database/data
+          adapter: mysql2
+          host: localhost
+          database: database_name
+          user: username
+          password: secret_password
+
+          mapped:
+            tables:
+             - :example_model_name
+             - :example_model_name_two
+
+          mapping_dir: spec/fixtures/settings/mapping.json
+          contentful_structure_dir: spec/fixtures/settings/contentful_structure.json
+
+          content_model_json: spec/fixtures/settings/contentful_model.json
+          converted_model_dir: spec/fixtures/settings/contentful_structure.json
+    EOF
+    yaml = YAML.load(yaml_text)
+    @config = Contentful::Configuration.new(yaml)
+  end
+end

metadata

@@ -0,0 +1,358 @@
+--- !ruby/object:Gem::Specification
+name: database-exporter
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Contentful GmbH (Andreas Tiefenthaler)
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: http
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.15'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.0
+- !ruby/object:Gem::Dependency
+  name: escort
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.3.10
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: rspec-its
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Database exporter that prepares content to be imported
+email:
+- rubygems@contentful.com
+executables:
+- database-exporter
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- bin/database-exporter
+- database_exporter.gemspec
+- example_data/contentful_model.json
+- example_data/contentful_structure.json
+- example_data/example_settings.yml
+- example_data/mapping.json
+- lib/cli.rb
+- lib/configuration.rb
+- lib/converters/content_types_structure_creator.rb
+- lib/converters/contentful_model_to_json.rb
+- lib/database/export.rb
+- lib/database/modules/json_export.rb
+- lib/database/modules/relations_export.rb
+- lib/database/modules/utils.rb
+- lib/migrator.rb
+- lib/version.rb
+- spec/fixtures/database/data/assets/image/image_1.json
+- spec/fixtures/database/data/assets/image/image_2.json
+- spec/fixtures/database/data/assets/image/image_3.json
+- spec/fixtures/database/data/assets/image/image_4.json
+- spec/fixtures/database/data/collections/comment.json
+- spec/fixtures/database/data/collections/job_skills.json
+- spec/fixtures/database/data/collections/jobs.json
+- spec/fixtures/database/data/collections/profile.json
+- spec/fixtures/database/data/collections/user.json
+- spec/fixtures/database/data/entries/comment/comment_1.json
+- spec/fixtures/database/data/entries/comment/comment_2.json
+- spec/fixtures/database/data/entries/comment/comment_3.json
+- spec/fixtures/database/data/entries/comment/comment_4.json
+- spec/fixtures/database/data/entries/comment/comment_5.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_1.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_10.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_2.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_3.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_4.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_5.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_6.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_7.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_8.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_9.json
+- spec/fixtures/database/data/entries/jobs/jobs_1.json
+- spec/fixtures/database/data/entries/jobs/jobs_2.json
+- spec/fixtures/database/data/entries/jobs/jobs_4.json
+- spec/fixtures/database/data/entries/profile/profile_1.json
+- spec/fixtures/database/data/entries/profile/profile_2.json
+- spec/fixtures/database/data/entries/user/user_1.json
+- spec/fixtures/database/data/entries/user/user_2.json
+- spec/fixtures/database/data/helpers/job_add_id_comments.json
+- spec/fixtures/database/data/helpers/job_add_id_job_add_skills.json
+- spec/fixtures/database/data/helpers/user_id_job_adds.json
+- spec/fixtures/database/data/helpers/user_id_profiles.json
+- spec/fixtures/database/data/table_names.json
+- spec/fixtures/database/table_names.json
+- spec/fixtures/development.sqlite3
+- spec/fixtures/json_responses/transformed_row.json
+- spec/fixtures/json_row/row.json
+- spec/fixtures/settings/contentful_model.json
+- spec/fixtures/settings/contentful_structure.json
+- spec/fixtures/settings/contentful_structure_test.json
+- spec/fixtures/settings/mapping.json
+- spec/fixtures/settings/settings.yml
+- spec/lib/configuration_spec.rb
+- spec/lib/database/export_spec.rb
+- spec/lib/database/json_export_spec.rb
+- spec/lib/database/relations_export_spec.rb
+- spec/lib/migrator_spec.rb
+- spec/spec_helper.rb
+- spec/support/db_rows_json.rb
+- spec/support/shared_configuration.rb
+homepage: https://github.com/contentful/database-adapter.rb
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Exporter for SQL based databases
+test_files:
+- spec/fixtures/database/data/assets/image/image_1.json
+- spec/fixtures/database/data/assets/image/image_2.json
+- spec/fixtures/database/data/assets/image/image_3.json
+- spec/fixtures/database/data/assets/image/image_4.json
+- spec/fixtures/database/data/collections/comment.json
+- spec/fixtures/database/data/collections/job_skills.json
+- spec/fixtures/database/data/collections/jobs.json
+- spec/fixtures/database/data/collections/profile.json
+- spec/fixtures/database/data/collections/user.json
+- spec/fixtures/database/data/entries/comment/comment_1.json
+- spec/fixtures/database/data/entries/comment/comment_2.json
+- spec/fixtures/database/data/entries/comment/comment_3.json
+- spec/fixtures/database/data/entries/comment/comment_4.json
+- spec/fixtures/database/data/entries/comment/comment_5.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_1.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_10.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_2.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_3.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_4.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_5.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_6.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_7.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_8.json
+- spec/fixtures/database/data/entries/job_skills/job_skills_9.json
+- spec/fixtures/database/data/entries/jobs/jobs_1.json
+- spec/fixtures/database/data/entries/jobs/jobs_2.json
+- spec/fixtures/database/data/entries/jobs/jobs_4.json
+- spec/fixtures/database/data/entries/profile/profile_1.json
+- spec/fixtures/database/data/entries/profile/profile_2.json
+- spec/fixtures/database/data/entries/user/user_1.json
+- spec/fixtures/database/data/entries/user/user_2.json
+- spec/fixtures/database/data/helpers/job_add_id_comments.json
+- spec/fixtures/database/data/helpers/job_add_id_job_add_skills.json
+- spec/fixtures/database/data/helpers/user_id_job_adds.json
+- spec/fixtures/database/data/helpers/user_id_profiles.json
+- spec/fixtures/database/data/table_names.json
+- spec/fixtures/database/table_names.json
+- spec/fixtures/development.sqlite3
+- spec/fixtures/json_responses/transformed_row.json
+- spec/fixtures/json_row/row.json
+- spec/fixtures/settings/contentful_model.json
+- spec/fixtures/settings/contentful_structure.json
+- spec/fixtures/settings/contentful_structure_test.json
+- spec/fixtures/settings/mapping.json
+- spec/fixtures/settings/settings.yml
+- spec/lib/configuration_spec.rb
+- spec/lib/database/export_spec.rb
+- spec/lib/database/json_export_spec.rb
+- spec/lib/database/relations_export_spec.rb
+- spec/lib/migrator_spec.rb
+- spec/spec_helper.rb
+- spec/support/db_rows_json.rb
+- spec/support/shared_configuration.rb
+has_rdoc: