contentful-importer 0.0.2 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/README.md +6 -4
- data/bin/contentful-importer +10 -10
- data/contentful_importer.gemspec +2 -2
- data/lib/contentful/importer/cli.rb +15 -0
- data/lib/contentful/importer/configuration.rb +38 -0
- data/lib/{converters → contentful/importer/converters}/content_types_structure_creator.rb +1 -1
- data/lib/{converters → contentful/importer/converters}/contentful_model_to_json.rb +1 -1
- data/lib/contentful/importer/data_organizer.rb +119 -0
- data/lib/contentful/importer/json_schema_validator.rb +66 -0
- data/lib/contentful/importer/migrator.rb +43 -0
- data/lib/contentful/importer/mime_content_type.rb +566 -0
- data/lib/contentful/importer/parallel_importer.rb +432 -0
- data/lib/contentful/importer/version.rb +5 -0
- data/spec/lib/configuration_spec.rb +16 -14
- data/spec/lib/importer/parallel_importer_spec.rb +151 -149
- data/spec/lib/json_schema_validator_spec.rb +50 -48
- data/spec/lib/migrator_spec.rb +86 -82
- data/spec/support/shared_configuration.rb +2 -2
- metadata +13 -12
- data/lib/cli.rb +0 -13
- data/lib/configuration.rb +0 -36
- data/lib/importer/data_organizer.rb +0 -117
- data/lib/importer/mime_content_type.rb +0 -564
- data/lib/importer/parallel_importer.rb +0 -430
- data/lib/json_schema_validator.rb +0 -64
- data/lib/migrator.rb +0 -39
- data/lib/version.rb +0 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7bbaffd5d3e9186ca222dc18419f7ed4fa489115
|
4
|
+
data.tar.gz: c3d3d73c0700ebc485e27d4f6a078768c5ba6ef1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fc7e161a2a8ba9305c8c9a291dbc82d1efa41a8cb65a91cf0191e90f495f238f1b733aba09d79029fbd4591c0901938f9232aee2c60379bc19d5aa3fc3e52630
|
7
|
+
data.tar.gz: 5550652c932b12155254e6ef460b657ad76f3ee03daadc6eed800771357f12878bacf3065aa3f8abfaa450c438cdbff10ad179fb7b412d365fd2dfa99d671578
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,12 @@
|
|
1
1
|
# Change Log
|
2
2
|
|
3
|
+
## 0.1.0
|
4
|
+
### Added
|
5
|
+
* Log publish errors similar to import errors
|
6
|
+
|
7
|
+
### Fixed
|
8
|
+
* Introduce proper namespacing of all classes
|
9
|
+
|
3
10
|
## 0.0.2
|
4
11
|
### Added
|
5
12
|
* Read contentType and description from asset JSON files, falling back to contentType inference from the asset URL.
|
data/README.md
CHANGED
@@ -63,8 +63,8 @@ contentful-importer --config-file settings.yml --action
|
|
63
63
|
contentful_structure_dir: PATH_TO_CONTENTFUL_STRUCTURE_JSON_FILE
|
64
64
|
|
65
65
|
## CONVERT CONTENTFUL MODEL TO CONTENTFUL IMPORT STRUCTURE
|
66
|
-
content_model_json:
|
67
|
-
converted_model_dir:
|
66
|
+
content_model_json: PATH_TO_CONTENTFUL_MODEL_JSON_FILE ## for input
|
67
|
+
converted_model_dir: PATH_TO_CONTENTFUL_MODEL_JSON_FILE ## for output
|
68
68
|
```
|
69
69
|
|
70
70
|
2. Create the contentful_structure.json. First you need to create a content model using the [Contentful web application](www.contentful.com). Then you can download the content model using the content management api and use the content model for the import:
|
@@ -140,6 +140,7 @@ To display all actions use the `-h` option:
|
|
140
140
|
```bash
|
141
141
|
contentful-importer -h
|
142
142
|
```
|
143
|
+
|
143
144
|
#### --convert-content-model-to-json
|
144
145
|
|
145
146
|
If you already have an existing content model for a space it can be downloaded and used for the import:
|
@@ -150,7 +151,6 @@ If you already have an existing content model for a space it can be downloaded a
|
|
150
151
|
'https://api.contentful.com/spaces/SPACE_ID/content_types' > contentful_model.json
|
151
152
|
```
|
152
153
|
|
153
|
-
|
154
154
|
In the **settings.yml** specify the PATH to **contentful_model.json**.
|
155
155
|
|
156
156
|
```yaml
|
@@ -222,6 +222,7 @@ To publish all entries:
|
|
222
222
|
```bash
|
223
223
|
contentful-importer --config-file settings.yml --publish-entries
|
224
224
|
```
|
225
|
+
|
225
226
|
Number of threads that are used in the publishing of entries is dependent on `--threads` argument, which you specified when import data.
|
226
227
|
|
227
228
|
#### --publish-assets ARGS
|
@@ -231,6 +232,7 @@ You can publish all assets with single Thread:
|
|
231
232
|
```bash
|
232
233
|
contentful-importer --config-file settings.yml --publish-assets
|
233
234
|
```
|
235
|
+
|
234
236
|
or add ```--threads``` argument to use multiple Threads:
|
235
237
|
|
236
238
|
```bash
|
@@ -310,7 +312,7 @@ Before you start importing the content make sure you read [how to use it](https:
|
|
310
312
|
|
311
313
|
#### Space ID
|
312
314
|
|
313
|
-
After [importing the content types](https://github.com/contentful/generic-importer.rb#--import-content-types-args) to the Space, you need to specify the `space_id` parameter in the settings.
|
315
|
+
After [importing the content types](https://github.com/contentful/generic-importer.rb#--import-content-types-args) to the Space, you need to specify the `space_id` parameter in the settings. Please note that the content-type import only accepts the space ID as a commandline option.
|
314
316
|
|
315
317
|
|
316
318
|
Example:
|
data/bin/contentful-importer
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'escort'
|
4
|
-
require_relative '../lib/cli'
|
4
|
+
require_relative '../lib/contentful/importer/cli'
|
5
5
|
|
6
6
|
fail ArgumentError, 'Set the path to the configuration file and define an action. More information can be found in the README.' if ARGF.argv.empty?
|
7
7
|
fail ArgumentError, "Missing '--config-file' argument. Usage: 'contentful-importer --config-file PATH_TO_CONFIGURATION_FILE --action'." unless ARGV.include?('--config-file')
|
@@ -15,13 +15,13 @@ Escort::App.create do |app|
|
|
15
15
|
app.command '--create-contentful-model-from-json' do |command|
|
16
16
|
command.summary 'Create content types json files, based on contentful_structure.json file.'
|
17
17
|
command.action do |options, arguments|
|
18
|
-
|
18
|
+
Contentful::Importer::CLI.new(options, arguments).execute
|
19
19
|
end
|
20
20
|
end
|
21
21
|
app.command '--convert-content-model-to-json' do |command|
|
22
22
|
command.summary 'Transform contentful model to contentful structure ready to import. View README'
|
23
23
|
command.action do |options, arguments|
|
24
|
-
|
24
|
+
Contentful::Importer::CLI.new(options, arguments).execute
|
25
25
|
end
|
26
26
|
end
|
27
27
|
app.command '--import-content-types' do |command|
|
@@ -32,7 +32,7 @@ Escort::App.create do |app|
|
|
32
32
|
opts.opt :space_name, 'space_name', short: :none, long: '--space_name', type: :string
|
33
33
|
end
|
34
34
|
command.action do |options, arguments|
|
35
|
-
|
35
|
+
Contentful::Importer::CLI.new(options, arguments).execute
|
36
36
|
end
|
37
37
|
end
|
38
38
|
app.command '--import' do |command|
|
@@ -42,26 +42,26 @@ Escort::App.create do |app|
|
|
42
42
|
opts.validate(:threads, 'argument must be set to 1 or 2 exclusive. ') { |option| option > 0 && option <= 2 }
|
43
43
|
end
|
44
44
|
command.action do |options, arguments|
|
45
|
-
|
45
|
+
Contentful::Importer::CLI.new(options, arguments).execute
|
46
46
|
end
|
47
47
|
end
|
48
48
|
app.command '--test-credentials' do |command|
|
49
49
|
command.summary 'Check your Contentful-management credentials'
|
50
50
|
command.action do |options, arguments|
|
51
|
-
|
51
|
+
Contentful::Importer::CLI.new(options, arguments).execute
|
52
52
|
end
|
53
53
|
end
|
54
54
|
app.command '--import-assets' do |command|
|
55
55
|
command.summary 'Import only assets'
|
56
56
|
command.action do |options, arguments|
|
57
|
-
|
57
|
+
Contentful::Importer::CLI.new(options, arguments).execute
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
61
61
|
app.command '--publish-entries' do |command|
|
62
62
|
command.summary 'Publish all entries'
|
63
63
|
command.action do |options, arguments|
|
64
|
-
|
64
|
+
Contentful::Importer::CLI.new(options, arguments).execute
|
65
65
|
end
|
66
66
|
end
|
67
67
|
|
@@ -72,13 +72,13 @@ Escort::App.create do |app|
|
|
72
72
|
opts.validate(:threads, 'argument must be set to 1 or 2 exclusive. ') { |option| option > 0 && option <= 2 }
|
73
73
|
end
|
74
74
|
command.action do |options, arguments|
|
75
|
-
|
75
|
+
Contentful::Importer::CLI.new(options, arguments).execute
|
76
76
|
end
|
77
77
|
end
|
78
78
|
app.command '--validate-schema' do |command|
|
79
79
|
command.summary 'Validate entries JSON schema'
|
80
80
|
command.action do |options, arguments|
|
81
|
-
|
81
|
+
Contentful::Importer::CLI.new(options, arguments).execute
|
82
82
|
end
|
83
83
|
end
|
84
84
|
end
|
data/contentful_importer.gemspec
CHANGED
@@ -2,11 +2,11 @@
|
|
2
2
|
lib = File.expand_path('../lib', __FILE__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
4
|
|
5
|
-
require File.expand_path('../lib/version', __FILE__)
|
5
|
+
require File.expand_path('../lib/contentful/importer/version', __FILE__)
|
6
6
|
|
7
7
|
Gem::Specification.new do |spec|
|
8
8
|
spec.name = 'contentful-importer'
|
9
|
-
spec.version =
|
9
|
+
spec.version = Contentful::Importer::VERSION
|
10
10
|
spec.authors = ['Contentful GmbH (Andreas Tiefenthaler)']
|
11
11
|
spec.email = ['rubygems@contentful.com']
|
12
12
|
spec.description = 'Generic importer for contentful.com'
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require_relative 'migrator'
|
2
|
+
require 'yaml'
|
3
|
+
|
4
|
+
module Contentful
|
5
|
+
module Importer
|
6
|
+
class CLI < Escort::ActionCommand::Base
|
7
|
+
|
8
|
+
def execute
|
9
|
+
setting_file = YAML.load_file(global_options[:file])
|
10
|
+
Migrator.new(setting_file).run(command_name, command_options)
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'active_support/core_ext/hash'
|
2
|
+
module Contentful
|
3
|
+
module Importer
|
4
|
+
class Configuration
|
5
|
+
attr_reader :space_id,
|
6
|
+
:config,
|
7
|
+
:data_dir,
|
8
|
+
:collections_dir,
|
9
|
+
:entries_dir,
|
10
|
+
:assets_dir,
|
11
|
+
:log_files_dir,
|
12
|
+
:threads_dir,
|
13
|
+
:imported_entries,
|
14
|
+
:published_entries,
|
15
|
+
:published_assets,
|
16
|
+
:space_id
|
17
|
+
|
18
|
+
def initialize(settings)
|
19
|
+
@config = settings
|
20
|
+
validate_required_parameters
|
21
|
+
@data_dir = settings['data_dir']
|
22
|
+
@collections_dir = "#{data_dir}/collections"
|
23
|
+
@entries_dir = "#{data_dir}/entries"
|
24
|
+
@assets_dir = "#{data_dir}/assets"
|
25
|
+
@log_files_dir = "#{data_dir}/logs"
|
26
|
+
@threads_dir = "#{data_dir}/threads"
|
27
|
+
@imported_entries = []
|
28
|
+
@published_entries = []
|
29
|
+
@published_assets = []
|
30
|
+
@space_id = settings['space_id']
|
31
|
+
end
|
32
|
+
|
33
|
+
def validate_required_parameters
|
34
|
+
fail ArgumentError, 'Set PATH to data_dir. Folder where all data will be stored. View README' if config['data_dir'].nil?
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'thread'
|
3
|
+
require_relative 'parallel_importer'
|
4
|
+
|
5
|
+
module Contentful
|
6
|
+
module Importer
|
7
|
+
class DataOrganizer
|
8
|
+
|
9
|
+
attr_reader :config, :split_params, :logger
|
10
|
+
|
11
|
+
def initialize(settings)
|
12
|
+
@config = settings
|
13
|
+
@split_params = {object_index: 0, current_thread: 0}
|
14
|
+
@logger = Logger.new(STDOUT)
|
15
|
+
end
|
16
|
+
|
17
|
+
def execute(threads_count)
|
18
|
+
create_threads_subdirectories(threads_count, true)
|
19
|
+
split_entries(threads_count)
|
20
|
+
end
|
21
|
+
|
22
|
+
def split_assets_to_threads(threads_count)
|
23
|
+
create_threads_subdirectories(threads_count, false, {assets: 'assets/'})
|
24
|
+
split_assets(threads_count)
|
25
|
+
end
|
26
|
+
|
27
|
+
def split_entries(threads_count)
|
28
|
+
entries_per_thread_count = total_entries_count / threads_count
|
29
|
+
Dir.glob("#{config.entries_dir}/*") do |dir_path|
|
30
|
+
collection_name = File.basename(dir_path)
|
31
|
+
if has_contentful_structure?(collection_name)
|
32
|
+
content_type_id = content_type_id_from_file(collection_name)
|
33
|
+
process_collection_files(content_type_id, dir_path, entries_per_thread_count, threads_count)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def split_assets(threads_count)
|
39
|
+
asset_per_thread = total_assets_count / threads_count
|
40
|
+
Dir.glob("#{config.assets_dir}/**/*json") do |asset_path|
|
41
|
+
copy_asset(asset_path)
|
42
|
+
split_params[:object_index] += 1
|
43
|
+
count_index_files(asset_per_thread, threads_count)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def process_collection_files(content_type_id, dir_path, entries_per_thread_count, threads_count)
|
48
|
+
logger.info "Processing collection: #{content_type_id}"
|
49
|
+
Dir.glob("#{dir_path}/*.json") do |entry_path|
|
50
|
+
copy_entry(entry_path, split_params[:current_thread], content_type_id)
|
51
|
+
split_params[:object_index] += 1
|
52
|
+
count_index_files(entries_per_thread_count, threads_count)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def count_index_files(objects_per_thread_count, threads_count)
|
57
|
+
if split_params[:object_index] == objects_per_thread_count
|
58
|
+
split_params[:object_index] = 0
|
59
|
+
set_current_thread(threads_count)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def set_current_thread(threads_count)
|
64
|
+
split_params[:current_thread] += 1
|
65
|
+
split_params[:current_thread] = 0 if split_params[:current_thread] == threads_count
|
66
|
+
end
|
67
|
+
|
68
|
+
def has_contentful_structure?(collection_file)
|
69
|
+
File.exist?("#{config.collections_dir}/#{collection_file}.json")
|
70
|
+
end
|
71
|
+
|
72
|
+
def content_type_id_from_file(collection_file)
|
73
|
+
JSON.parse(File.read("#{config.collections_dir}/#{collection_file}.json"))['id']
|
74
|
+
end
|
75
|
+
|
76
|
+
def new_entry_name(content_type_id, entry_path)
|
77
|
+
"#{content_type_id}_#{File.basename(entry_path, '.*').match(/(\d+)/)[0]}.json"
|
78
|
+
end
|
79
|
+
|
80
|
+
def copy_entry(entry_path, current_thread, content_type_id)
|
81
|
+
FileUtils.cp entry_path, "#{config.threads_dir}/#{current_thread}/#{new_entry_name(content_type_id, entry_path)}"
|
82
|
+
end
|
83
|
+
|
84
|
+
def copy_asset(asset_path)
|
85
|
+
FileUtils.cp asset_path, "#{config.threads_dir}/assets/#{split_params[:current_thread]}/#{File.basename(asset_path)}"
|
86
|
+
end
|
87
|
+
|
88
|
+
def create_threads_subdirectories(threads_count, validate, dir = {})
|
89
|
+
validate_collections_files if validate
|
90
|
+
create_directory(config.threads_dir)
|
91
|
+
threads_count.times do |thread_id|
|
92
|
+
create_directory("#{config.threads_dir}/#{dir[:assets]}#{thread_id}")
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def create_directory(path)
|
97
|
+
FileUtils.mkdir_p(path) unless File.directory?(path)
|
98
|
+
end
|
99
|
+
|
100
|
+
def total_entries_count
|
101
|
+
total_number = 0
|
102
|
+
Dir.glob("#{config.entries_dir}/*") do |dir_path|
|
103
|
+
collection_name = File.basename(dir_path)
|
104
|
+
total_number += Dir.glob("#{config.entries_dir}/#{collection_name}/*").count if has_contentful_structure?(collection_name)
|
105
|
+
end
|
106
|
+
total_number
|
107
|
+
end
|
108
|
+
|
109
|
+
def total_assets_count
|
110
|
+
Dir.glob("#{config.assets_dir}/**/*json").count
|
111
|
+
end
|
112
|
+
|
113
|
+
def validate_collections_files
|
114
|
+
fail ArgumentError, "Make sure the #{config.collections_dir} directory exists and the content structure resides within it. View README" unless Dir.exist?(config.collections_dir)
|
115
|
+
fail ArgumentError, 'Collections directory is empty! Create content types JSON files. View README' if Dir.glob("#{config.collections_dir}/*").empty?
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
require 'json-schema'
|
2
|
+
|
3
|
+
module Contentful
|
4
|
+
module Importer
|
5
|
+
class JsonSchemaValidator
|
6
|
+
|
7
|
+
attr_reader :config, :logger
|
8
|
+
|
9
|
+
def initialize(configuration)
|
10
|
+
@config = configuration
|
11
|
+
@logger = Logger.new(STDOUT)
|
12
|
+
end
|
13
|
+
|
14
|
+
def validate_schemas
|
15
|
+
Dir.glob("#{config.collections_dir}/*") do |content_type_file|
|
16
|
+
validate_schema(content_type_file)
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def validate_schema(content_type_file)
|
21
|
+
schema = parse_content_type_schema(JSON.parse(File.read(content_type_file)))
|
22
|
+
content_type_filename = File.basename(content_type_file, '.*')
|
23
|
+
validate_entry(content_type_filename, schema)
|
24
|
+
end
|
25
|
+
|
26
|
+
def validate_entry(content_type_filename, schema)
|
27
|
+
Dir.glob("#{config.entries_dir}/#{content_type_filename}/*") do |entry_file|
|
28
|
+
entry_schema = JSON.parse(File.read(entry_file))
|
29
|
+
begin
|
30
|
+
JSON::Validator.validate!(schema, entry_schema)
|
31
|
+
rescue JSON::Schema::ValidationError => error
|
32
|
+
logger.info "#{error.message}! Path to invalid entry: #{entry_file}"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def parse_content_type_schema(ct_file)
|
38
|
+
new_hash = base_schema_format
|
39
|
+
ct_file['fields'].each do |key|
|
40
|
+
type = convert_type(key['type'])
|
41
|
+
new_hash['properties'].merge!({key['id'] => {'type' => type}})
|
42
|
+
end
|
43
|
+
new_hash
|
44
|
+
end
|
45
|
+
|
46
|
+
def base_schema_format
|
47
|
+
{'type' => 'object', 'properties' => {}}
|
48
|
+
end
|
49
|
+
|
50
|
+
def convert_type(type)
|
51
|
+
case type
|
52
|
+
when 'Text', 'Date', 'Symbol'
|
53
|
+
'string'
|
54
|
+
when 'Number'
|
55
|
+
'float'
|
56
|
+
when 'Asset', 'Entry'
|
57
|
+
'object'
|
58
|
+
else
|
59
|
+
type.downcase
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require_relative 'parallel_importer'
|
2
|
+
require_relative 'configuration'
|
3
|
+
require_relative 'converters/contentful_model_to_json'
|
4
|
+
require_relative 'json_schema_validator'
|
5
|
+
|
6
|
+
module Contentful
|
7
|
+
module Importer
|
8
|
+
class Migrator
|
9
|
+
|
10
|
+
attr_reader :importer, :converter, :config, :json_validator
|
11
|
+
|
12
|
+
def initialize(settings)
|
13
|
+
@config = Configuration.new(settings)
|
14
|
+
@importer = ParallelImporter.new(@config)
|
15
|
+
@converter = ContentfulModelToJson.new(@config)
|
16
|
+
@json_validator = JsonSchemaValidator.new(@config)
|
17
|
+
end
|
18
|
+
|
19
|
+
def run(action, options = {})
|
20
|
+
case action.to_s
|
21
|
+
when '--create-contentful-model-from-json'
|
22
|
+
converter.create_content_type_json
|
23
|
+
when '--import-content-types'
|
24
|
+
importer.create_contentful_model(options)
|
25
|
+
when '--import'
|
26
|
+
importer.import_data(options[:threads])
|
27
|
+
when '--convert-content-model-to-json'
|
28
|
+
converter.convert_to_import_form
|
29
|
+
when '--publish-entries'
|
30
|
+
importer.publish_entries_in_threads
|
31
|
+
when '--test-credentials'
|
32
|
+
importer.test_credentials
|
33
|
+
when '--import-assets'
|
34
|
+
importer.import_only_assets
|
35
|
+
when '--publish-assets'
|
36
|
+
importer.publish_assets_in_threads(options[:threads])
|
37
|
+
when '--validate-schema'
|
38
|
+
json_validator.validate_schemas
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|