contentful-importer 0.0.2 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +6 -4
- data/bin/contentful-importer +10 -10
- data/contentful_importer.gemspec +2 -2
- data/lib/contentful/importer/cli.rb +15 -0
- data/lib/contentful/importer/configuration.rb +38 -0
- data/lib/{converters → contentful/importer/converters}/content_types_structure_creator.rb +1 -1
- data/lib/{converters → contentful/importer/converters}/contentful_model_to_json.rb +1 -1
- data/lib/contentful/importer/data_organizer.rb +119 -0
- data/lib/contentful/importer/json_schema_validator.rb +66 -0
- data/lib/contentful/importer/migrator.rb +43 -0
- data/lib/contentful/importer/mime_content_type.rb +566 -0
- data/lib/contentful/importer/parallel_importer.rb +432 -0
- data/lib/contentful/importer/version.rb +5 -0
- data/spec/lib/configuration_spec.rb +16 -14
- data/spec/lib/importer/parallel_importer_spec.rb +151 -149
- data/spec/lib/json_schema_validator_spec.rb +50 -48
- data/spec/lib/migrator_spec.rb +86 -82
- data/spec/support/shared_configuration.rb +2 -2
- metadata +13 -12
- data/lib/cli.rb +0 -13
- data/lib/configuration.rb +0 -36
- data/lib/importer/data_organizer.rb +0 -117
- data/lib/importer/mime_content_type.rb +0 -564
- data/lib/importer/parallel_importer.rb +0 -430
- data/lib/json_schema_validator.rb +0 -64
- data/lib/migrator.rb +0 -39
- data/lib/version.rb +0 -3
| @@ -1,170 +1,172 @@ | |
| 1 1 | 
             
            require 'spec_helper'
         | 
| 2 | 
            -
            require './lib/importer/parallel_importer'
         | 
| 3 | 
            -
            require './lib/configuration'
         | 
| 2 | 
            +
            require './lib/contentful/importer/parallel_importer'
         | 
| 3 | 
            +
            require './lib/contentful/importer/configuration'
         | 
| 4 4 |  | 
| 5 5 | 
             
            module Contentful
         | 
| 6 | 
            -
               | 
| 6 | 
            +
              module Importer
         | 
| 7 | 
            +
                describe ParallelImporter do
         | 
| 7 8 |  | 
| 8 | 
            -
             | 
| 9 | 
            +
                  include_context 'shared_configuration'
         | 
| 9 10 |  | 
| 10 | 
            -
             | 
| 11 | 
            -
             | 
| 12 | 
            -
                end
         | 
| 13 | 
            -
             | 
| 14 | 
            -
             | 
| 15 | 
            -
                it 'number of threads' do
         | 
| 16 | 
            -
                  number = @importer.number_of_threads
         | 
| 17 | 
            -
                  expect(number).to eq 2
         | 
| 18 | 
            -
                end
         | 
| 19 | 
            -
             | 
| 20 | 
            -
                it 'create_asset_file' do
         | 
| 21 | 
            -
                  file = @importer.create_asset_file('title', {'url' => 'www.example.com/photo.png'})
         | 
| 22 | 
            -
                  expect(file).to be_a Contentful::Management::File
         | 
| 23 | 
            -
                end
         | 
| 24 | 
            -
             | 
| 25 | 
            -
                it 'gets asset content type from params' do
         | 
| 26 | 
            -
                  file = @importer.create_asset_file('title', {'url' => 'www.example.com/image.jpg?withoutextension', 'contentType' => 'image/jpeg'})
         | 
| 27 | 
            -
                  expect(file.properties[:contentType]).to eq'image/jpeg'
         | 
| 28 | 
            -
                end
         | 
| 29 | 
            -
             | 
| 30 | 
            -
                it 'gets asset content type from params' do
         | 
| 31 | 
            -
                  file = @importer.create_asset_file('title', {'url' => 'www.example.com/image.jpg'})
         | 
| 32 | 
            -
                  expect(file.properties[:contentType]).to eq'image/jpeg'
         | 
| 33 | 
            -
                end
         | 
| 34 | 
            -
             | 
| 35 | 
            -
                it 'get_id' do
         | 
| 36 | 
            -
                  id = @importer.send(:get_id, {'id' => 'name.png'})
         | 
| 37 | 
            -
                  expect(id).to eq 'name.png'
         | 
| 38 | 
            -
                end
         | 
| 39 | 
            -
             | 
| 40 | 
            -
                context 'asset_status' do
         | 
| 41 | 
            -
                  it 'successfully imported' do
         | 
| 42 | 
            -
                    allow(CSV).to receive(:open).with('spec/fixtures/import_files/logs/success_assets.csv', 'a')
         | 
| 43 | 
            -
                    asset_file = Contentful::Management::Asset.new
         | 
| 44 | 
            -
                    asset_file.id = 'test_id'
         | 
| 45 | 
            -
                    expect_any_instance_of(Contentful::Management::Asset).to receive(:process_file)
         | 
| 46 | 
            -
                    @importer.asset_status(asset_file, {'url' => 'www.example.com/photo.png'})
         | 
| 47 | 
            -
                  end
         | 
| 48 | 
            -
                end
         | 
| 49 | 
            -
             | 
| 50 | 
            -
                it 'publish an entry' do
         | 
| 51 | 
            -
                  vcr('publish_an_entry') do
         | 
| 52 | 
            -
                    expect_any_instance_of(Contentful::ParallelImporter).to receive(:publish_status).with(Contentful::Management::Entry,
         | 
| 53 | 
            -
                                                                                                          'comment_5',
         | 
| 54 | 
            -
                                                                                                          'published_entries')
         | 
| 55 | 
            -
                    @importer.publish_entry('comment_5')
         | 
| 56 | 
            -
                  end
         | 
| 57 | 
            -
                end
         | 
| 58 | 
            -
             | 
| 59 | 
            -
                it 'import_entry' do
         | 
| 60 | 
            -
                  vcr('import_entry') do
         | 
| 61 | 
            -
                    expect_any_instance_of(Contentful::ParallelImporter).to receive(:import_status).with(Contentful::Management::Entry, './spec/fixtures/import_files/entries/job_skills/job_skills_1.json', 'success_thread_0')
         | 
| 62 | 
            -
                    expect(Contentful::Management::Entry.find('ip17s12q0ek4', 'job_skills_1')).to be_a Contentful::Management::NotFound
         | 
| 63 | 
            -
                    @importer.send(:import_entry, './spec/fixtures/import_files/entries/job_skills/job_skills_1.json', 'ip17s12q0ek4', '2soCP557HGKoOOK0SqmMOm', 'success_thread_0')
         | 
| 64 | 
            -
                    entry = Contentful::Management::Entry.find('ip17s12q0ek4', 'job_skills_1')
         | 
| 65 | 
            -
                    expect(entry).to be_a Contentful::Management::Entry
         | 
| 66 | 
            -
                    expect(entry.id).to eq 'job_skills_1'
         | 
| 67 | 
            -
                    expect(entry.name).to eq 'Commercial awareness'
         | 
| 68 | 
            -
                  end
         | 
| 69 | 
            -
                end
         | 
| 70 | 
            -
             | 
| 71 | 
            -
                context 'import_status' do
         | 
| 72 | 
            -
                  it 'successfully imported' do
         | 
| 73 | 
            -
                    allow(CSV).to receive(:open).with('spec/fixtures/import_files/logs/success_thread_0.csv', 'a')
         | 
| 74 | 
            -
                    allow(File).to receive(:basename).with('file_path')
         | 
| 75 | 
            -
                    entry = Contentful::Management::Entry.new
         | 
| 76 | 
            -
                    @importer.send(:import_status, entry, 'file_path', 'success_thread_0')
         | 
| 11 | 
            +
                  before do
         | 
| 12 | 
            +
                    @importer = ParallelImporter.new(@config)
         | 
| 77 13 | 
             
                  end
         | 
| 78 | 
            -
                end
         | 
| 79 14 |  | 
| 80 | 
            -
                it 'create space' do
         | 
| 81 | 
            -
                  vcr('create_space') do
         | 
| 82 | 
            -
                    space = @importer.send(:create_space, 'rspec_test_space')
         | 
| 83 | 
            -
                    expect(space).to be_a Contentful::Management::Space
         | 
| 84 | 
            -
                    expect(space.name).to eq 'rspec_test_space'
         | 
| 85 | 
            -
                  end
         | 
| 86 | 
            -
                end
         | 
| 87 15 |  | 
| 88 | 
            -
             | 
| 89 | 
            -
             | 
| 90 | 
            -
                     | 
| 91 | 
            -
                    expect(entry).to be_a Contentful::Management::Entry
         | 
| 92 | 
            -
                    expect(entry.id).to eq 'entry_id'
         | 
| 16 | 
            +
                  it 'number of threads' do
         | 
| 17 | 
            +
                    number = @importer.number_of_threads
         | 
| 18 | 
            +
                    expect(number).to eq 2
         | 
| 93 19 | 
             
                  end
         | 
| 94 | 
            -
                end
         | 
| 95 20 |  | 
| 96 | 
            -
             | 
| 97 | 
            -
             | 
| 98 | 
            -
                     | 
| 99 | 
            -
                    expect(asset).to be_a Contentful::Management::Asset
         | 
| 100 | 
            -
                    expect(asset.id).to eq 'asset_id'
         | 
| 21 | 
            +
                  it 'create_asset_file' do
         | 
| 22 | 
            +
                    file = @importer.create_asset_file('title', {'url' => 'www.example.com/photo.png'})
         | 
| 23 | 
            +
                    expect(file).to be_a Contentful::Management::File
         | 
| 101 24 | 
             
                  end
         | 
| 102 | 
            -
                end
         | 
| 103 25 |  | 
| 104 | 
            -
             | 
| 105 | 
            -
             | 
| 106 | 
            -
             | 
| 107 | 
            -
                end
         | 
| 108 | 
            -
             | 
| 109 | 
            -
                context 'validate_params' do
         | 
| 110 | 
            -
                  it 'Array type' do
         | 
| 111 | 
            -
                    result = @importer.send(:validate_param, [])
         | 
| 112 | 
            -
                    expect(result).to be true
         | 
| 113 | 
            -
                  end
         | 
| 114 | 
            -
                  it 'Other type' do
         | 
| 115 | 
            -
                    result = @importer.send(:validate_param, 'value')
         | 
| 116 | 
            -
                    expect(result).to be false
         | 
| 26 | 
            +
                  it 'gets asset content type from params' do
         | 
| 27 | 
            +
                    file = @importer.create_asset_file('title', {'url' => 'www.example.com/image.jpg?withoutextension', 'contentType' => 'image/jpeg'})
         | 
| 28 | 
            +
                    expect(file.properties[:contentType]).to eq'image/jpeg'
         | 
| 117 29 | 
             
                  end
         | 
| 118 | 
            -
                end
         | 
| 119 30 |  | 
| 120 | 
            -
             | 
| 121 | 
            -
             | 
| 122 | 
            -
                     | 
| 123 | 
            -
                    expect_any_instance_of(Contentful::ParallelImporter).to receive(:parse_attributes_from_hash).with({'k' => 'v'}, 'space_id', 'content_type_id')
         | 
| 124 | 
            -
                    @importer.send(:create_entry_parameters, 'content_type_id', {'Hash' => {'k' => 'v'}}, 'space_id')
         | 
| 125 | 
            -
                  end
         | 
| 126 | 
            -
                  it 'Array' do
         | 
| 127 | 
            -
                    expect_any_instance_of(Contentful::ParallelImporter).to receive(:validate_param) { true }
         | 
| 128 | 
            -
                    expect_any_instance_of(Contentful::ParallelImporter).to receive(:parse_attributes_from_array).with(['Value'], 'space_id', 'content_type_id')
         | 
| 129 | 
            -
                    @importer.send(:create_entry_parameters, 'content_type_id', {'Array' => ['Value']}, 'space_id')
         | 
| 130 | 
            -
                  end
         | 
| 131 | 
            -
                  it 'String' do
         | 
| 132 | 
            -
                    expect_any_instance_of(Contentful::ParallelImporter).to receive(:validate_param) { true }
         | 
| 133 | 
            -
                    @importer.send(:create_entry_parameters, 'content_type_id', {'key' => 'value'}, 'space_id')
         | 
| 31 | 
            +
                  it 'gets asset content type from params' do
         | 
| 32 | 
            +
                    file = @importer.create_asset_file('title', {'url' => 'www.example.com/image.jpg'})
         | 
| 33 | 
            +
                    expect(file.properties[:contentType]).to eq'image/jpeg'
         | 
| 134 34 | 
             
                  end
         | 
| 135 | 
            -
                end
         | 
| 136 35 |  | 
| 137 | 
            -
             | 
| 138 | 
            -
             | 
| 139 | 
            -
                     | 
| 140 | 
            -
                    @importer.send(:parse_attributes_from_hash, {'type' => 'Location'}, 'space_id', 'content_type_id')
         | 
| 141 | 
            -
                  end
         | 
| 142 | 
            -
                  it 'File' do
         | 
| 143 | 
            -
                    expect_any_instance_of(Contentful::ParallelImporter).to receive(:create_asset).with('space_id', {'type' => 'File'})
         | 
| 144 | 
            -
                    @importer.send(:parse_attributes_from_hash, {'type' => 'File'}, 'space_id', 'content_type_id')
         | 
| 36 | 
            +
                  it 'get_id' do
         | 
| 37 | 
            +
                    id = @importer.send(:get_id, {'id' => 'name.png'})
         | 
| 38 | 
            +
                    expect(id).to eq 'name.png'
         | 
| 145 39 | 
             
                  end
         | 
| 146 | 
            -
                  it 'Entry' do
         | 
| 147 | 
            -
                    expect_any_instance_of(Contentful::ParallelImporter).to receive(:create_entry).with({'type' => 'Entry'}, 'space_id', 'content_type_id')
         | 
| 148 | 
            -
                    @importer.send(:parse_attributes_from_hash, {'type' => 'Entry'}, 'space_id', 'content_type_id')
         | 
| 149 | 
            -
                  end
         | 
| 150 | 
            -
                  it 'Entry' do
         | 
| 151 | 
            -
                    params = @importer.send(:parse_attributes_from_hash, {'some' => 'params'}, 'space_id', 'content_type_id')
         | 
| 152 | 
            -
                    expect(params).to include('some' => 'params')
         | 
| 153 | 
            -
                  end
         | 
| 154 | 
            -
                end
         | 
| 155 40 |  | 
| 156 | 
            -
             | 
| 157 | 
            -
             | 
| 158 | 
            -
             | 
| 159 | 
            -
             | 
| 160 | 
            -
             | 
| 161 | 
            -
             | 
| 162 | 
            -
             | 
| 163 | 
            -
                     | 
| 164 | 
            -
                  end
         | 
| 165 | 
            -
             | 
| 166 | 
            -
             | 
| 167 | 
            -
                     | 
| 41 | 
            +
                  context 'asset_status' do
         | 
| 42 | 
            +
                    it 'successfully imported' do
         | 
| 43 | 
            +
                      allow(CSV).to receive(:open).with('spec/fixtures/import_files/logs/success_assets.csv', 'a')
         | 
| 44 | 
            +
                      asset_file = Contentful::Management::Asset.new
         | 
| 45 | 
            +
                      asset_file.id = 'test_id'
         | 
| 46 | 
            +
                      expect_any_instance_of(Contentful::Management::Asset).to receive(:process_file)
         | 
| 47 | 
            +
                      @importer.asset_status(asset_file, {'url' => 'www.example.com/photo.png'})
         | 
| 48 | 
            +
                    end
         | 
| 49 | 
            +
                  end
         | 
| 50 | 
            +
             | 
| 51 | 
            +
                  it 'publish an entry' do
         | 
| 52 | 
            +
                    vcr('publish_an_entry') do
         | 
| 53 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:publish_status).with(Contentful::Management::Entry,
         | 
| 54 | 
            +
                                                                                                            'comment_5',
         | 
| 55 | 
            +
                                                                                                            'published_entries')
         | 
| 56 | 
            +
                      @importer.publish_entry('comment_5')
         | 
| 57 | 
            +
                    end
         | 
| 58 | 
            +
                  end
         | 
| 59 | 
            +
             | 
| 60 | 
            +
                  it 'import_entry' do
         | 
| 61 | 
            +
                    vcr('import_entry') do
         | 
| 62 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:import_status).with(Contentful::Management::Entry, './spec/fixtures/import_files/entries/job_skills/job_skills_1.json', 'success_thread_0')
         | 
| 63 | 
            +
                      expect(Contentful::Management::Entry.find('ip17s12q0ek4', 'job_skills_1')).to be_a Contentful::Management::NotFound
         | 
| 64 | 
            +
                      @importer.send(:import_entry, './spec/fixtures/import_files/entries/job_skills/job_skills_1.json', 'ip17s12q0ek4', '2soCP557HGKoOOK0SqmMOm', 'success_thread_0')
         | 
| 65 | 
            +
                      entry = Contentful::Management::Entry.find('ip17s12q0ek4', 'job_skills_1')
         | 
| 66 | 
            +
                      expect(entry).to be_a Contentful::Management::Entry
         | 
| 67 | 
            +
                      expect(entry.id).to eq 'job_skills_1'
         | 
| 68 | 
            +
                      expect(entry.name).to eq 'Commercial awareness'
         | 
| 69 | 
            +
                    end
         | 
| 70 | 
            +
                  end
         | 
| 71 | 
            +
             | 
| 72 | 
            +
                  context 'import_status' do
         | 
| 73 | 
            +
                    it 'successfully imported' do
         | 
| 74 | 
            +
                      allow(CSV).to receive(:open).with('spec/fixtures/import_files/logs/success_thread_0.csv', 'a')
         | 
| 75 | 
            +
                      allow(File).to receive(:basename).with('file_path')
         | 
| 76 | 
            +
                      entry = Contentful::Management::Entry.new
         | 
| 77 | 
            +
                      @importer.send(:import_status, entry, 'file_path', 'success_thread_0')
         | 
| 78 | 
            +
                    end
         | 
| 79 | 
            +
                  end
         | 
| 80 | 
            +
             | 
| 81 | 
            +
                  it 'create space' do
         | 
| 82 | 
            +
                    vcr('create_space') do
         | 
| 83 | 
            +
                      space = @importer.send(:create_space, 'rspec_test_space')
         | 
| 84 | 
            +
                      expect(space).to be_a Contentful::Management::Space
         | 
| 85 | 
            +
                      expect(space.name).to eq 'rspec_test_space'
         | 
| 86 | 
            +
                    end
         | 
| 87 | 
            +
                  end
         | 
| 88 | 
            +
             | 
| 89 | 
            +
                  it 'create entry' do
         | 
| 90 | 
            +
                    vcr('create_entry') do
         | 
| 91 | 
            +
                      entry = @importer.send(:create_entry, {'id' => 'entry_id'}, 'ip17s12q0ek4', '6H6pGAV1PUsuoAW26Iu48W')
         | 
| 92 | 
            +
                      expect(entry).to be_a Contentful::Management::Entry
         | 
| 93 | 
            +
                      expect(entry.id).to eq 'entry_id'
         | 
| 94 | 
            +
                    end
         | 
| 95 | 
            +
                  end
         | 
| 96 | 
            +
             | 
| 97 | 
            +
                  it 'create asset' do
         | 
| 98 | 
            +
                    vcr('create_asset') do
         | 
| 99 | 
            +
                      asset = @importer.send(:create_asset, 'ip17s12q0ek4', {'id' => 'asset_id'})
         | 
| 100 | 
            +
                      expect(asset).to be_a Contentful::Management::Asset
         | 
| 101 | 
            +
                      expect(asset.id).to eq 'asset_id'
         | 
| 102 | 
            +
                    end
         | 
| 103 | 
            +
                  end
         | 
| 104 | 
            +
             | 
| 105 | 
            +
                  it 'create location file' do
         | 
| 106 | 
            +
                    file = @importer.send(:create_location_file, {'lat' => 'lat_v', 'lng' => 'lng_v'})
         | 
| 107 | 
            +
                    expect(file).to be_a Contentful::Management::Location
         | 
| 108 | 
            +
                  end
         | 
| 109 | 
            +
             | 
| 110 | 
            +
                  context 'validate_params' do
         | 
| 111 | 
            +
                    it 'Array type' do
         | 
| 112 | 
            +
                      result = @importer.send(:validate_param, [])
         | 
| 113 | 
            +
                      expect(result).to be true
         | 
| 114 | 
            +
                    end
         | 
| 115 | 
            +
                    it 'Other type' do
         | 
| 116 | 
            +
                      result = @importer.send(:validate_param, 'value')
         | 
| 117 | 
            +
                      expect(result).to be false
         | 
| 118 | 
            +
                    end
         | 
| 119 | 
            +
                  end
         | 
| 120 | 
            +
             | 
| 121 | 
            +
                  context 'create_entry_parameters' do
         | 
| 122 | 
            +
                    it 'Hash' do
         | 
| 123 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:validate_param) { true }
         | 
| 124 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:parse_attributes_from_hash).with({'k' => 'v'}, 'space_id', 'content_type_id')
         | 
| 125 | 
            +
                      @importer.send(:create_entry_parameters, 'content_type_id', {'Hash' => {'k' => 'v'}}, 'space_id')
         | 
| 126 | 
            +
                    end
         | 
| 127 | 
            +
                    it 'Array' do
         | 
| 128 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:validate_param) { true }
         | 
| 129 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:parse_attributes_from_array).with(['Value'], 'space_id', 'content_type_id')
         | 
| 130 | 
            +
                      @importer.send(:create_entry_parameters, 'content_type_id', {'Array' => ['Value']}, 'space_id')
         | 
| 131 | 
            +
                    end
         | 
| 132 | 
            +
                    it 'String' do
         | 
| 133 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:validate_param) { true }
         | 
| 134 | 
            +
                      @importer.send(:create_entry_parameters, 'content_type_id', {'key' => 'value'}, 'space_id')
         | 
| 135 | 
            +
                    end
         | 
| 136 | 
            +
                  end
         | 
| 137 | 
            +
             | 
| 138 | 
            +
                  context 'parse_attributes_from_hash' do
         | 
| 139 | 
            +
                    it 'Location' do
         | 
| 140 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:create_location_file).with({'type' => 'Location'})
         | 
| 141 | 
            +
                      @importer.send(:parse_attributes_from_hash, {'type' => 'Location'}, 'space_id', 'content_type_id')
         | 
| 142 | 
            +
                    end
         | 
| 143 | 
            +
                    it 'File' do
         | 
| 144 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:create_asset).with('space_id', {'type' => 'File'})
         | 
| 145 | 
            +
                      @importer.send(:parse_attributes_from_hash, {'type' => 'File'}, 'space_id', 'content_type_id')
         | 
| 146 | 
            +
                    end
         | 
| 147 | 
            +
                    it 'Entry' do
         | 
| 148 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:create_entry).with({'type' => 'Entry'}, 'space_id', 'content_type_id')
         | 
| 149 | 
            +
                      @importer.send(:parse_attributes_from_hash, {'type' => 'Entry'}, 'space_id', 'content_type_id')
         | 
| 150 | 
            +
                    end
         | 
| 151 | 
            +
                    it 'Entry' do
         | 
| 152 | 
            +
                      params = @importer.send(:parse_attributes_from_hash, {'some' => 'params'}, 'space_id', 'content_type_id')
         | 
| 153 | 
            +
                      expect(params).to include('some' => 'params')
         | 
| 154 | 
            +
                    end
         | 
| 155 | 
            +
                  end
         | 
| 156 | 
            +
             | 
| 157 | 
            +
                  context 'parse_attributes_from_array' do
         | 
| 158 | 
            +
                    it 'File' do
         | 
| 159 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:create_asset).with('space_id', {'type' => 'File'})
         | 
| 160 | 
            +
                      @importer.send(:parse_attributes_from_array, [{'type' => 'File'}], 'space_id', 'content_type_id')
         | 
| 161 | 
            +
                    end
         | 
| 162 | 
            +
                    it 'Entry' do
         | 
| 163 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:create_entry).with({'type' => 'Entry'}, 'space_id', 'content_type_id')
         | 
| 164 | 
            +
                      @importer.send(:parse_attributes_from_array, [{'type' => 'Entry'}], 'space_id', 'content_type_id')
         | 
| 165 | 
            +
                    end
         | 
| 166 | 
            +
                    it 'Entry' do
         | 
| 167 | 
            +
                      params = @importer.send(:parse_attributes_from_array, [{'some' => 'params'}], 'space_id', 'content_type_id')
         | 
| 168 | 
            +
                      expect(params).to include('some' => 'params')
         | 
| 169 | 
            +
                    end
         | 
| 168 170 | 
             
                  end
         | 
| 169 171 | 
             
                end
         | 
| 170 172 | 
             
              end
         | 
| @@ -1,63 +1,65 @@ | |
| 1 1 | 
             
            require 'spec_helper'
         | 
| 2 | 
            -
            require 'json_schema_validator'
         | 
| 2 | 
            +
            require 'contentful/importer/json_schema_validator'
         | 
| 3 3 |  | 
| 4 4 | 
             
            module Contentful
         | 
| 5 | 
            -
               | 
| 5 | 
            +
              module Importer
         | 
| 6 | 
            +
                describe JsonSchemaValidator do
         | 
| 6 7 |  | 
| 7 | 
            -
             | 
| 8 | 
            +
                  include_context 'shared_configuration'
         | 
| 8 9 |  | 
| 9 | 
            -
             | 
| 10 | 
            -
             | 
| 11 | 
            -
             | 
| 12 | 
            -
             | 
| 13 | 
            -
             | 
| 14 | 
            -
                it 'validate_schemas' do
         | 
| 15 | 
            -
                  expect_any_instance_of(Contentful::JsonSchemaValidator).to receive(:validate_schema).exactly(4).times
         | 
| 16 | 
            -
                  JsonSchemaValidator.new(@config).validate_schemas
         | 
| 17 | 
            -
                end
         | 
| 18 | 
            -
             | 
| 19 | 
            -
                it 'validate_schema' do
         | 
| 20 | 
            -
                  expect_any_instance_of(Contentful::JsonSchemaValidator).to receive(:validate_entry).with('job_skills',
         | 
| 21 | 
            -
                                                                                                           {'type' => 'object',
         | 
| 22 | 
            -
                                                                                                            'properties' => {'name' => {'type' => 'string'}
         | 
| 23 | 
            -
                                                                                                            }})
         | 
| 24 | 
            -
                  JsonSchemaValidator.new(@config).validate_schema('spec/fixtures/import_files/collections/job_skills.json')
         | 
| 25 | 
            -
                end
         | 
| 26 | 
            -
             | 
| 27 | 
            -
                it 'validate_entry' do
         | 
| 28 | 
            -
                  schema = load_json('import_files/collections/job_skills')
         | 
| 29 | 
            -
                  expect { JsonSchemaValidator.new(@config).validate_entry('job_skills', schema) }.not_to raise_error
         | 
| 30 | 
            -
                end
         | 
| 10 | 
            +
                  it 'initialize' do
         | 
| 11 | 
            +
                    validator = JsonSchemaValidator.new(@config)
         | 
| 12 | 
            +
                    expect(validator.config).to be_a Configuration
         | 
| 13 | 
            +
                  end
         | 
| 31 14 |  | 
| 32 | 
            -
             | 
| 33 | 
            -
             | 
| 34 | 
            -
             | 
| 35 | 
            -
                   | 
| 36 | 
            -
                end
         | 
| 15 | 
            +
                  it 'validate_schemas' do
         | 
| 16 | 
            +
                    expect_any_instance_of(JsonSchemaValidator).to receive(:validate_schema).exactly(4).times
         | 
| 17 | 
            +
                    JsonSchemaValidator.new(@config).validate_schemas
         | 
| 18 | 
            +
                  end
         | 
| 37 19 |  | 
| 38 | 
            -
             | 
| 39 | 
            -
             | 
| 40 | 
            -
             | 
| 41 | 
            -
             | 
| 20 | 
            +
                  it 'validate_schema' do
         | 
| 21 | 
            +
                    expect_any_instance_of(JsonSchemaValidator).to receive(:validate_entry).with('job_skills',
         | 
| 22 | 
            +
                                                                                                             {'type' => 'object',
         | 
| 23 | 
            +
                                                                                                              'properties' => {'name' => {'type' => 'string'}
         | 
| 24 | 
            +
                                                                                                              }})
         | 
| 25 | 
            +
                    JsonSchemaValidator.new(@config).validate_schema('spec/fixtures/import_files/collections/job_skills.json')
         | 
| 26 | 
            +
                  end
         | 
| 42 27 |  | 
| 43 | 
            -
             | 
| 44 | 
            -
             | 
| 45 | 
            -
                     | 
| 46 | 
            -
                    expect(result).to eq 'string'
         | 
| 28 | 
            +
                  it 'validate_entry' do
         | 
| 29 | 
            +
                    schema = load_json('import_files/collections/job_skills')
         | 
| 30 | 
            +
                    expect { JsonSchemaValidator.new(@config).validate_entry('job_skills', schema) }.not_to raise_error
         | 
| 47 31 | 
             
                  end
         | 
| 48 | 
            -
             | 
| 49 | 
            -
             | 
| 50 | 
            -
                     | 
| 32 | 
            +
             | 
| 33 | 
            +
                  it 'parse_content_type_schema' do
         | 
| 34 | 
            +
                    ct_file = load_json('import_files/collections/job_skills')
         | 
| 35 | 
            +
                    result = JsonSchemaValidator.new(@config).parse_content_type_schema(ct_file)
         | 
| 36 | 
            +
                    expect(result).to include('type' => 'object', 'properties' => {'name' => {'type' => 'string'}})
         | 
| 51 37 | 
             
                  end
         | 
| 52 | 
            -
             | 
| 53 | 
            -
             | 
| 54 | 
            -
                     | 
| 38 | 
            +
             | 
| 39 | 
            +
                  it 'base schema form' do
         | 
| 40 | 
            +
                    result = JsonSchemaValidator.new(@config).base_schema_format
         | 
| 41 | 
            +
                    expect(result).to include('type' => 'object', 'properties' => {})
         | 
| 55 42 | 
             
                  end
         | 
| 56 | 
            -
             | 
| 57 | 
            -
             | 
| 58 | 
            -
                     | 
| 43 | 
            +
             | 
| 44 | 
            +
                  context 'convert_type' do
         | 
| 45 | 
            +
                    it 'Text to String' do
         | 
| 46 | 
            +
                      result = JsonSchemaValidator.new(@config).convert_type('Text')
         | 
| 47 | 
            +
                      expect(result).to eq 'string'
         | 
| 48 | 
            +
                    end
         | 
| 49 | 
            +
                    it 'Number to Float' do
         | 
| 50 | 
            +
                      result = JsonSchemaValidator.new(@config).convert_type('Number')
         | 
| 51 | 
            +
                      expect(result).to eq 'float'
         | 
| 52 | 
            +
                    end
         | 
| 53 | 
            +
                    it 'Asset to Object(Hash)' do
         | 
| 54 | 
            +
                      result = JsonSchemaValidator.new(@config).convert_type('Asset')
         | 
| 55 | 
            +
                      expect(result).to eq 'object'
         | 
| 56 | 
            +
                    end
         | 
| 57 | 
            +
                    it 'Other to downcase format' do
         | 
| 58 | 
            +
                      result = JsonSchemaValidator.new(@config).convert_type('Integer')
         | 
| 59 | 
            +
                      expect(result).to eq 'integer'
         | 
| 60 | 
            +
                    end
         | 
| 59 61 | 
             
                  end
         | 
| 60 | 
            -
                end
         | 
| 61 62 |  | 
| 63 | 
            +
                end
         | 
| 62 64 | 
             
              end
         | 
| 63 65 | 
             
            end
         | 
    
        data/spec/lib/migrator_spec.rb
    CHANGED
    
    | @@ -1,99 +1,103 @@ | |
| 1 1 | 
             
            require 'spec_helper'
         | 
| 2 | 
            -
            require './lib/migrator'
         | 
| 2 | 
            +
            require './lib/contentful/importer/migrator'
         | 
| 3 3 |  | 
| 4 | 
            -
             | 
| 5 | 
            -
               | 
| 6 | 
            -
                 | 
| 7 | 
            -
             | 
| 8 | 
            -
             | 
| 9 | 
            -
             | 
| 10 | 
            -
             | 
| 11 | 
            -
             | 
| 12 | 
            -
             | 
| 13 | 
            -
             | 
| 14 | 
            -
             | 
| 15 | 
            -
             | 
| 16 | 
            -
             | 
| 4 | 
            +
            module Contentful
         | 
| 5 | 
            +
              module Importer
         | 
| 6 | 
            +
                describe Migrator do
         | 
| 7 | 
            +
                  before do
         | 
| 8 | 
            +
                    @setting_file = YAML.load_file('spec/fixtures/settings/settings.yml')
         | 
| 9 | 
            +
                  end
         | 
| 10 | 
            +
                  it 'convert contentful model to contentful structure' do
         | 
| 11 | 
            +
                    Migrator.new(@setting_file).run('--convert-content-model-to-json')
         | 
| 12 | 
            +
                    contentful_structure = load_fixture('settings/contentful_structure_test')
         | 
| 13 | 
            +
                    expect(contentful_structure.count).to eq 4
         | 
| 14 | 
            +
                    expect(contentful_structure['Jobs']).to include(id: '4L1bg4WQ5aWQMiE82ouag', name: 'Jobs', displayField: 'title', description: nil)
         | 
| 15 | 
            +
                    expect(contentful_structure['Jobs']['fields'].count).to eq 4
         | 
| 16 | 
            +
                    expect(contentful_structure['Jobs']['fields']['Image']).to include(id: 'image', type: 'Asset', link: 'Link')
         | 
| 17 | 
            +
                    expect(contentful_structure['Jobs']['fields']['Creator']).to include(id: 'creator', type: 'Entry', link: 'Link')
         | 
| 18 | 
            +
                  end
         | 
| 17 19 |  | 
| 18 | 
            -
             | 
| 19 | 
            -
             | 
| 20 | 
            -
             | 
| 21 | 
            -
             | 
| 22 | 
            -
             | 
| 23 | 
            -
             | 
| 24 | 
            -
             | 
| 25 | 
            -
             | 
| 26 | 
            -
             | 
| 27 | 
            -
             | 
| 28 | 
            -
             | 
| 20 | 
            +
                  it 'create content type json files from contentful structure' do
         | 
| 21 | 
            +
                    Migrator.new(@setting_file).run('--create-contentful-model-from-json')
         | 
| 22 | 
            +
                    expect(Dir.glob('spec/fixtures/import_files/collections/*').count).to eq 4
         | 
| 23 | 
            +
                    content_types_files = %w(comment.json job_skills.json jobs.json profile.json user.json)
         | 
| 24 | 
            +
                    Dir.glob('spec/fixtures/import_files/collections/*') do |directory_name|
         | 
| 25 | 
            +
                      expect(content_types_files.include?(File.basename(directory_name))).to be true
         | 
| 26 | 
            +
                    end
         | 
| 27 | 
            +
                    job_skills = load_fixture('import_files/collections/job_skills')
         | 
| 28 | 
            +
                    expect(job_skills).to include(id: '2soCP557HGKoOOK0SqmMOm', name: 'Job Skills', displayField: 'name')
         | 
| 29 | 
            +
                    expect(job_skills['fields'].count).to eq 1
         | 
| 30 | 
            +
                    expect(job_skills['fields'].first).to include(id: 'name', name: 'Name', type: 'Text')
         | 
| 29 31 |  | 
| 30 | 
            -
             | 
| 31 | 
            -
             | 
| 32 | 
            -
             | 
| 33 | 
            -
             | 
| 32 | 
            +
                    jobs = load_fixture('import_files/collections/jobs')
         | 
| 33 | 
            +
                    expect(jobs).to include(id: '4L1bg4WQ5aWQMiE82ouag', name: 'Jobs', displayField: 'title')
         | 
| 34 | 
            +
                    expect(jobs['fields'].count).to eq 5
         | 
| 35 | 
            +
                    expect(jobs['fields'].last).to include(id: 'skills', name: 'Skills', type: 'Array', link_type: 'Entry', link: 'Link')
         | 
| 34 36 |  | 
| 35 | 
            -
             | 
| 36 | 
            -
             | 
| 37 | 
            -
             | 
| 38 | 
            -
             | 
| 37 | 
            +
                    profile = load_fixture('import_files/collections/profile')
         | 
| 38 | 
            +
                    expect(profile).to include(id: '4WFZh4MwC4Mc0EQWAeOY8A', name: 'Profile', displayField: nil)
         | 
| 39 | 
            +
                    expect(profile['fields'].count).to eq 2
         | 
| 40 | 
            +
                    expect(profile['fields'].first).to include(id: 'nickname', name: 'Nickname', type: 'Text')
         | 
| 39 41 |  | 
| 40 | 
            -
             | 
| 41 | 
            -
             | 
| 42 | 
            -
             | 
| 43 | 
            -
             | 
| 42 | 
            +
                    user = load_fixture('import_files/collections/user')
         | 
| 43 | 
            +
                    expect(user).to include(id: '1TVvxCqoRq0qUYAOQuOqys', name: 'User', displayField: 'first_name')
         | 
| 44 | 
            +
                    expect(user['fields'].first).to include(id: 'first_name', name: 'First_name', type: 'Text')
         | 
| 45 | 
            +
                  end
         | 
| 44 46 |  | 
| 45 | 
            -
             | 
| 46 | 
            -
             | 
| 47 | 
            -
             | 
| 48 | 
            -
             | 
| 49 | 
            -
             | 
| 47 | 
            +
                  it 'create content type json files from contentful structure' do
         | 
| 48 | 
            +
                    vcr('import_content_types') do
         | 
| 49 | 
            +
                      Migrator.new(@setting_file).run('--import-content-types', space_id: 'space_id')
         | 
| 50 | 
            +
                    end
         | 
| 51 | 
            +
                  end
         | 
| 50 52 |  | 
| 51 | 
            -
             | 
| 52 | 
            -
             | 
| 53 | 
            -
             | 
| 54 | 
            -
             | 
| 55 | 
            -
             | 
| 56 | 
            -
             | 
| 57 | 
            -
             | 
| 58 | 
            -
             | 
| 53 | 
            +
                  it 'import an entires to Contentful with two Threads' do
         | 
| 54 | 
            +
                    vcr('import_entries') do
         | 
| 55 | 
            +
                      allow(FileUtils).to receive(:rm_r)
         | 
| 56 | 
            +
                      import = Migrator.new(@setting_file).run('--import', threads: 2)
         | 
| 57 | 
            +
                      expect(import).to be_a Array
         | 
| 58 | 
            +
                      expect(import.count).to eq 2
         | 
| 59 | 
            +
                    end
         | 
| 60 | 
            +
                  end
         | 
| 59 61 |  | 
| 60 | 
            -
             | 
| 61 | 
            -
             | 
| 62 | 
            -
             | 
| 63 | 
            -
             | 
| 64 | 
            -
             | 
| 62 | 
            +
                  it 'import an assets to Contentful' do
         | 
| 63 | 
            +
                    vcr('import_assets') do
         | 
| 64 | 
            +
                      Migrator.new(@setting_file).run('--import-assets')
         | 
| 65 | 
            +
                    end
         | 
| 66 | 
            +
                  end
         | 
| 65 67 |  | 
| 66 | 
            -
             | 
| 67 | 
            -
             | 
| 68 | 
            -
             | 
| 69 | 
            -
             | 
| 70 | 
            -
             | 
| 68 | 
            +
                  it 'publish an entires' do
         | 
| 69 | 
            +
                    vcr('publish_entries') do
         | 
| 70 | 
            +
                      Migrator.new(@setting_file).run('--publish-entries')
         | 
| 71 | 
            +
                    end
         | 
| 72 | 
            +
                  end
         | 
| 71 73 |  | 
| 72 | 
            -
             | 
| 73 | 
            -
             | 
| 74 | 
            -
             | 
| 75 | 
            -
             | 
| 76 | 
            -
             | 
| 77 | 
            -
             | 
| 78 | 
            -
             | 
| 74 | 
            +
                  it 'publish an assets' do
         | 
| 75 | 
            +
                    vcr('publish_asset') do
         | 
| 76 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:publish_status).exactly(4).times
         | 
| 77 | 
            +
                      expect_any_instance_of(ParallelImporter).to receive(:create_log_file).with('success_published_assets')
         | 
| 78 | 
            +
                      Migrator.new(@setting_file).run('--publish-assets', threads: 1)
         | 
| 79 | 
            +
                    end
         | 
| 80 | 
            +
                  end
         | 
| 79 81 |  | 
| 80 | 
            -
             | 
| 81 | 
            -
             | 
| 82 | 
            -
             | 
| 83 | 
            -
             | 
| 84 | 
            -
             | 
| 82 | 
            +
                  context 'test credentials' do
         | 
| 83 | 
            +
                    it 'when valid' do
         | 
| 84 | 
            +
                      vcr('valid_credentials') do
         | 
| 85 | 
            +
                        expect_any_instance_of(Logger).to receive(:info).with('Contentful Management API credentials: OK')
         | 
| 86 | 
            +
                        Migrator.new(@setting_file).run('--test-credentials')
         | 
| 87 | 
            +
                      end
         | 
| 88 | 
            +
                    end
         | 
| 89 | 
            +
                    it 'when invalid' do
         | 
| 90 | 
            +
                      vcr('invalid_credentials') do
         | 
| 91 | 
            +
                        expect_any_instance_of(Logger).to receive(:info).with('Contentful Management API credentials: INVALID (check README)')
         | 
| 92 | 
            +
                        Migrator.new(@setting_file).run('--test-credentials')
         | 
| 93 | 
            +
                      end
         | 
| 94 | 
            +
                    end
         | 
| 85 95 | 
             
                  end
         | 
| 86 | 
            -
             | 
| 87 | 
            -
             | 
| 88 | 
            -
             | 
| 89 | 
            -
                    expect_any_instance_of(Logger).to receive(:info).with('Contentful Management API credentials: INVALID (check README)')
         | 
| 90 | 
            -
                    Migrator.new(@setting_file).run('--test-credentials')
         | 
| 96 | 
            +
             | 
| 97 | 
            +
                  it 'validate JSON schema' do
         | 
| 98 | 
            +
                    expect { Migrator.new(@setting_file).run('--validate-schema') }.not_to raise_error
         | 
| 91 99 | 
             
                  end
         | 
| 92 | 
            -
                end
         | 
| 93 | 
            -
              end
         | 
| 94 100 |  | 
| 95 | 
            -
             | 
| 96 | 
            -
                expect { Migrator.new(@setting_file).run('--validate-schema') }.not_to raise_error
         | 
| 101 | 
            +
                end
         | 
| 97 102 | 
             
              end
         | 
| 98 | 
            -
             | 
| 99 103 | 
             
            end
         | 
| @@ -1,4 +1,4 @@ | |
| 1 | 
            -
            require './lib/configuration'
         | 
| 1 | 
            +
            require './lib/contentful/importer/configuration'
         | 
| 2 2 |  | 
| 3 3 | 
             
            shared_context 'shared_configuration' do
         | 
| 4 4 | 
             
              before do
         | 
| @@ -17,6 +17,6 @@ shared_context 'shared_configuration' do | |
| 17 17 | 
             
                      converted_model_dir: spec/fixtures/settings/contentful_structure_test.json
         | 
| 18 18 | 
             
                EOF
         | 
| 19 19 | 
             
                yaml = YAML.load(yaml_text)
         | 
| 20 | 
            -
                @config = Contentful::Configuration.new(yaml)
         | 
| 20 | 
            +
                @config = Contentful::Importer::Configuration.new(yaml)
         | 
| 21 21 | 
             
              end
         | 
| 22 22 | 
             
            end
         |