drupal-exporter 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +39 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +8 -0
  5. data/CHANGELOG.md +4 -0
  6. data/Gemfile +7 -0
  7. data/Gemfile.lock +72 -0
  8. data/LICENSE +22 -0
  9. data/README.md +251 -0
  10. data/Rakefile +9 -0
  11. data/bin/drupal-exporter +37 -0
  12. data/drupal_exporter.gemspec +34 -0
  13. data/drupal_settings/boolean_columns.yml +2 -0
  14. data/drupal_settings/drupal_content_types.json +47 -0
  15. data/drupal_settings/drupal_settings.yml +23 -0
  16. data/lib/cli.rb +13 -0
  17. data/lib/configuration.rb +46 -0
  18. data/lib/converters/content_types_structure_creator.rb +60 -0
  19. data/lib/converters/contentful_model_to_json.rb +109 -0
  20. data/lib/drupal/content_type.rb +151 -0
  21. data/lib/drupal/export.rb +69 -0
  22. data/lib/drupal/file_managed.rb +42 -0
  23. data/lib/drupal/tag.rb +52 -0
  24. data/lib/drupal/user.rb +46 -0
  25. data/lib/drupal/vocabulary.rb +42 -0
  26. data/lib/migrator.rb +28 -0
  27. data/lib/version.rb +3 -0
  28. data/spec/fixtures/database_rows/content_type_article.json +14 -0
  29. data/spec/fixtures/database_rows/image.json +10 -0
  30. data/spec/fixtures/database_rows/node_content_type_article.json +15 -0
  31. data/spec/fixtures/database_rows/node_content_type_blog.json +15 -0
  32. data/spec/fixtures/database_rows/tag.json +8 -0
  33. data/spec/fixtures/database_rows/user.json +18 -0
  34. data/spec/fixtures/database_rows/vocabulary.json +9 -0
  35. data/spec/fixtures/drupal/assets/file/file_4.json +6 -0
  36. data/spec/fixtures/drupal/entries/article/article_5.json +24 -0
  37. data/spec/fixtures/drupal/entries/tag/tag_1.json +9 -0
  38. data/spec/fixtures/drupal/entries/user/user_1.json +6 -0
  39. data/spec/fixtures/drupal/entries/vocabulary/vocabulary_3.json +6 -0
  40. data/spec/fixtures/json_responses/article.json +24 -0
  41. data/spec/fixtures/json_responses/image.json +6 -0
  42. data/spec/fixtures/json_responses/tag.json +9 -0
  43. data/spec/fixtures/json_responses/vocabulary.json +6 -0
  44. data/spec/fixtures/settings/boolean_columns.yml +2 -0
  45. data/spec/fixtures/settings/drupal_content_types.json +47 -0
  46. data/spec/fixtures/settings/drupal_settings.yml +17 -0
  47. data/spec/lib/configuration_spec.rb +18 -0
  48. data/spec/lib/drupal/content_type_spec.rb +123 -0
  49. data/spec/lib/drupal/export_spec.rb +33 -0
  50. data/spec/lib/drupal/file_managed_spec.rb +52 -0
  51. data/spec/lib/drupal/tag_spec.rb +60 -0
  52. data/spec/lib/drupal/user_spec.rb +49 -0
  53. data/spec/lib/drupal/vocabulary_spec.rb +51 -0
  54. data/spec/spec_helper.rb +11 -0
  55. data/spec/support/db_rows_json.rb +15 -0
  56. data/spec/support/shared_configuration.rb +20 -0
  57. metadata +297 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: f48e838d6b34c48f0e78d52078eed0def0430e8b
4
+ data.tar.gz: 25476436a73bcfd87b02645d8eb1b3d775893665
5
+ SHA512:
6
+ metadata.gz: 127ead0fc0925ced6dfc0cbc1060836cb5c5fe8b95ebf77929e720303efd29616d9a9e3ab681ae1fea0ec8c8c4edb33386c00c08cf9c8f01d6e491ecf33ebb53
7
+ data.tar.gz: 4a2fa1d328dd3485f7f86dfcfceda7b42651cf796ae7d882f77e21c9a0cf9ae55af0fb32f0431318a14186c72068e52d1f03bc4a6a381400ea54f2e02e1e9602
@@ -0,0 +1,39 @@
1
+ *.gem
2
+ *.rbc
3
+ /.config
4
+ /coverage
5
+ /InstalledFiles
6
+ /pkg/
7
+ /spec/reports/
8
+ /test/tmp/
9
+ /test/version_tmp/
10
+ /tmp/
11
+
12
+ ## Specific to RubyMotion:
13
+ .dat*
14
+ .repl_history
15
+ build/
16
+
17
+ ## Documentation cache and generated files:
18
+ /.yardoc/
19
+ /_yardoc/
20
+ /doc/
21
+ /rdoc/
22
+
23
+ ## Environment normalisation:
24
+ /.bundle/
25
+ /lib/bundler/man/
26
+
27
+ # for a library or gem, you might want to ignore these files since the code is
28
+ # intended to run in multiple environments; otherwise, check them in:
29
+ # Gemfile.lock
30
+ # .ruby-version
31
+ # .ruby-gemset
32
+
33
+ # unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
34
+ .rvmrc
35
+
36
+ #RubyMine
37
+ .idea
38
+
39
+ .DS_Store
data/.rspec ADDED
@@ -0,0 +1,3 @@
1
+ --color
2
+ --format progress
3
+ --order random
@@ -0,0 +1,8 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.1.2
4
+ - 2.1.1
5
+ - 2.1
6
+ - 2.0.0
7
+ - 1.9.3
8
+ # - jruby-19mode
@@ -0,0 +1,4 @@
1
+ # Change Log
2
+ ## 0.0.1
3
+ ### Other
4
+ * Initial release
data/Gemfile ADDED
@@ -0,0 +1,7 @@
1
+ source 'https://rubygems.org'
2
+ # Specify your gem's dependencies in drupal_exporter.gemspec
3
+ gemspec
4
+
5
+ group :test do
6
+ gem 'simplecov', require: false
7
+ end
@@ -0,0 +1,72 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ drupal-exporter (0.0.1)
5
+ activesupport (~> 4.1)
6
+ escort (~> 0.4.0)
7
+ http (~> 0.6)
8
+ i18n (~> 0.6)
9
+ multi_json (~> 1)
10
+ mysql2 (~> 0.3)
11
+ pg (~> 0.17.0)
12
+ sequel (~> 4.15)
13
+
14
+ GEM
15
+ remote: https://rubygems.org/
16
+ specs:
17
+ activesupport (4.1.8)
18
+ i18n (~> 0.6, >= 0.6.9)
19
+ json (~> 1.7, >= 1.7.7)
20
+ minitest (~> 5.1)
21
+ thread_safe (~> 0.1)
22
+ tzinfo (~> 1.1)
23
+ diff-lcs (1.2.5)
24
+ docile (1.1.5)
25
+ escort (0.4.0)
26
+ nesty
27
+ http (0.6.3)
28
+ http_parser.rb (~> 0.6.0)
29
+ http_parser.rb (0.6.0)
30
+ i18n (0.6.11)
31
+ json (1.8.1)
32
+ minitest (5.4.3)
33
+ multi_json (1.10.1)
34
+ mysql2 (0.3.17)
35
+ nesty (1.0.2)
36
+ pg (0.17.1)
37
+ rake (10.4.2)
38
+ rspec (3.1.0)
39
+ rspec-core (~> 3.1.0)
40
+ rspec-expectations (~> 3.1.0)
41
+ rspec-mocks (~> 3.1.0)
42
+ rspec-core (3.1.7)
43
+ rspec-support (~> 3.1.0)
44
+ rspec-expectations (3.1.2)
45
+ diff-lcs (>= 1.2.0, < 2.0)
46
+ rspec-support (~> 3.1.0)
47
+ rspec-its (1.1.0)
48
+ rspec-core (>= 3.0.0)
49
+ rspec-expectations (>= 3.0.0)
50
+ rspec-mocks (3.1.3)
51
+ rspec-support (~> 3.1.0)
52
+ rspec-support (3.1.2)
53
+ sequel (4.17.0)
54
+ simplecov (0.9.1)
55
+ docile (~> 1.1.0)
56
+ multi_json (~> 1.0)
57
+ simplecov-html (~> 0.8.0)
58
+ simplecov-html (0.8.0)
59
+ thread_safe (0.3.4)
60
+ tzinfo (1.2.2)
61
+ thread_safe (~> 0.1)
62
+
63
+ PLATFORMS
64
+ ruby
65
+
66
+ DEPENDENCIES
67
+ bundler (~> 1.6)
68
+ drupal-exporter!
69
+ rake
70
+ rspec (~> 3)
71
+ rspec-its (~> 1.1.0)
72
+ simplecov
data/LICENSE ADDED
@@ -0,0 +1,22 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2014 Contentful
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
22
+
@@ -0,0 +1,251 @@
1
+ Drupal to Contentful Exporter
2
+ =================
3
+
4
+ # This Gem supports only Drupal version 7.
5
+
6
+ ## Description
7
+ This tool will extract the following content from a Drupal database dump file:
8
+
9
+ * Content types (Blog, Article, Page, Custom content types)
10
+ * Tags
11
+ * Vocabularies
12
+ * Users
13
+
14
+
15
+ ## Installation
16
+
17
+ ```bash
18
+ gem install drupal-exporter
19
+ ```
20
+
21
+ This will install the `drupal-exporter` executable.
22
+
23
+
24
+ ## Step by step
25
+
26
+ 1. Create a YAML file with the required parameters (eg. `settings.yml`):
27
+
28
+ ```yml
29
+ # PATH TO ALL DATA
30
+ data_dir: PATH_TO_ALL_DATA
31
+
32
+ # CONNECTING TO A DATABASE
33
+ adapter: mysql2
34
+ host: localhost
35
+ database: drupal_database_name
36
+ user: username
37
+ password: secret_password
38
+
39
+ # DRUPAL SETTINGS
40
+ drupal_content_types_json: drupal_settings/drupal_content_types.json
41
+ drupal_boolean_columns: drupal_settings/boolean_columns.yml
42
+ drupal_base_url: http://example_hostname.com
43
+
44
+ # CONVERT CONTENTFUL MODEL TO CONTENTFUL IMPORT STRUCTURE
45
+ content_model_json: PATH_TO_CONTENTFUL_MODEL_JSON_FILE/contentful_model.json
46
+ converted_model_dir: PATH_WHERE_CONVERTED_CONTENT_MODEL_WILL_BE_SAVED/contentful_structure.json
47
+
48
+ contentful_structure_dir: PATH_TO_CONTENTFUL_STRUCTURE_JSON_FILE/contentful_structure.json
49
+ ```
50
+ 2. (Not required to extract data). Create the `contentful_structure.json`. First you need to create a content model using the [Contentful web application](www.contentful.com). Then you can download the content model using the content management api and use the content model for the import:
51
+
52
+ ```bash
53
+ curl -X GET \
54
+ -H 'Authorization: Bearer ACCESS_TOKEN' \
55
+ 'https://api.contentful.com/spaces/SPACE_ID/content_types' > contentful_model.json
56
+ ```
57
+
58
+ It will create `contentful_model.json` file, which you need to transform into the `contentful_structure.json` using:
59
+
60
+ ```bash
61
+ drupal-exporter --config-file settings.yml --convert-content-model-to-json
62
+ ```
63
+
64
+ The converted content model will be saved as JSON file in the `converted_model_dir` path.
65
+
66
+ Now you can generate content types JSON files.
67
+
68
+ ```bash
69
+ drupal-exporter --config-file settings.yml --create-contentful-model-from-json
70
+ ```
71
+ It will create the content types JSON files which represent your content structure for the import.
72
+
73
+ 3. Create the `drupal_content_types.json` file. This file contains the mapped structure of your database.
74
+
75
+ Mapping structure:
76
+
77
+ ```javascript
78
+ machine_name_of_content_type : {
79
+ contentful_api_field_1 : column_machine_name_1,
80
+ contentful_api_field_2: column_machine_name_2,
81
+ contentful_api_field_3 : column_machine_name_3
82
+ }
83
+ ```
84
+
85
+ You can find a sample mapping file in the `drupal_settings/drupal_content_types.json` directory.
86
+
87
+ 4. (Optional). Boolean values. Sequel converts boolean values `0,1`, stored in the database only when the field is TINYINT(1) type.
88
+ To map the value of `0,1` to `false, true`, you have to specify the column names in the yaml file (eg. `boolean_columns.yml`) and
89
+ specify the path to this file in the `settings.yml` file, parameter `drupal_boolean_columns`.
90
+
91
+ Example:
92
+
93
+ ```yml
94
+ - field_if_content_type
95
+ - field_boolean
96
+ ```
97
+ 5. Extract the content from the database and generate the JSON files for the import:
98
+
99
+ ```bash
100
+ drupal-exporter --config-file settings.yml --extract-to-json
101
+ ```
102
+ It will _only_ extract the content and store it as JSON files, nothing will be uploaded yet.
103
+
104
+ 6. Use the [contentful-importer](https://github.com/contentful/generic-importer.rb) to import the content to [contentful.com](https://www.contentful.com)
105
+
106
+
107
+
108
+ ## Setup ##
109
+
110
+ Create settings a YML file (eg. `settings.yml`) to define all required parameters.
111
+ Assuming we are going to work with either MySQL, SQLite or a PostgreSQL database, you must define credentials to connect Drupal database.
112
+ An example configuration for connecting with a MySQL database named "drupal_database_name":
113
+
114
+ ```yml
115
+ adapter: mysql2
116
+ user: username
117
+ host: localhost
118
+ database: drupal_database_name
119
+ password: secret_password
120
+ ```
121
+
122
+ ### Available Adapters ###
123
+
124
+ * PostgreSQL => postgres
125
+ * MySQL => mysql2
126
+ * SQlite => sqlite
127
+
128
+
129
+ ## Content Types ##
130
+
131
+ To be able to properly map the Drupal content types to the Contentful content types they must be identical by name.
132
+
133
+ Example:
134
+
135
+ ```
136
+ Drupal name of content type => 'Blog'
137
+ Contentful name of content type => 'Blog'
138
+ ```
139
+
140
+ ## Tags ##
141
+
142
+ These content types are exported from the Drupal database by default and assigned to every content type. There is no need to specify them in the content type structure.
143
+
144
+ They will be saved with the following api field ids:
145
+ ```
146
+ Tags => 'tags'
147
+ ```
148
+
149
+ ### Custom tag content types ###
150
+
151
+ If you want to add tags that you define in a custom table, you need to specify them by adding an addition parameter to the hash:
152
+ ```javascript
153
+ "term_tagging":{
154
+ "table": "field_term_tagging"
155
+ }
156
+ ```
157
+
158
+ ### Machine name ###
159
+
160
+ To find `machine name` in your Drupal structure, sign in to your Admin console and open `Structure` section. Next to the name of Content type, will be located machine name in brackets.
161
+ To find `machine name` for individual field, go into `manage fields` option.
162
+
163
+ ### Booleans ###
164
+
165
+ To map columns of boolean values, you need to create YML file ( eg. boolean_columns.yml ) and define machine names of boolean columns.
166
+
167
+ Example:
168
+ ```yml
169
+ - field_if_content_type
170
+ - field_boolean
171
+ ```
172
+
173
+ The path to the `drupal_boolean_columns` file is defined in the settings.yml.
174
+
175
+ ```yml
176
+ drupal_boolean_columns: PATH_TO_YML_FILE
177
+ ```
178
+
179
+ ### Assets & Images ###
180
+
181
+ Your files and assets need to be available and accessible through the internet.
182
+ For this purpose, you must define the `drupal_base_url` in the settings.yml file so that the importer will be able to create them.
183
+
184
+ ```yml
185
+ drupal_base_url: http://example_hostname.com
186
+ ```
187
+
188
+ ### Mapping structure ###
189
+
190
+ Create JSON file with content types structure:
191
+
192
+ ```javascript
193
+ "machine_name_of_content_type" : {
194
+ "contentful_api_fiel_idd" : "column_machine_name",
195
+ "contentful_api_field_id2" : "column_machine_name2",
196
+ "contentful_api_field_id3" : "column_machine_name23"
197
+ }
198
+ ```
199
+
200
+ Example structure:
201
+
202
+ ```javascript
203
+ {
204
+ "article": {
205
+ "body": "body",
206
+ "image": "field_image"
207
+ },
208
+ "page": {
209
+ "body": "body"
210
+ },
211
+ "blog": {
212
+ "body": "body"
213
+ },
214
+ "content_type": {
215
+ "body": "body",
216
+ "age": "field_age",
217
+ "if_content_type": "field_if_content_type",
218
+ "name": "field_first_name"
219
+ }
220
+ }
221
+ ```
222
+
223
+ In settings.yml file define path to the `drupal_content_types_json` file.
224
+
225
+ ```yml
226
+ drupal_content_types_json: PATH_TO_JSON_FILE
227
+ ```
228
+
229
+ ### Example settings.yml file ###
230
+
231
+ ```yml
232
+ data_dir: /tmp/data
233
+
234
+ #Connecting to a database
235
+ adapter: mysql2
236
+ host: localhost
237
+ database: database_name
238
+ user: username
239
+ password: password
240
+
241
+ # Drupal
242
+ drupal_content_types_json: PATH_TO_FILE/drupal_content_types.json
243
+ drupal_boolean_columns: PATH_TO_FILE/boolean_columns.yml
244
+ drupal_base_url: http://example_hostname.com
245
+ ```
246
+
247
+ Command to extract data:
248
+
249
+ ```bash
250
+ drupal-exporter --config-file settings.yml --extract-to-json
251
+ ```
@@ -0,0 +1,9 @@
1
+ require 'rubygems'
2
+ require 'rake'
3
+ require 'bundler/gem_tasks'
4
+
5
+ require 'rspec/core/rake_task'
6
+ RSpec::Core::RakeTask.new('spec')
7
+
8
+
9
+ task default: :spec
@@ -0,0 +1,37 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'escort'
4
+ require_relative '../lib/cli'
5
+
6
+ fail ArgumentError, 'Set the path to the configuration file and define an action. More information can be found in the README.' if ARGF.argv.empty?
7
+ fail ArgumentError, "Missing '--config-file' argument. Usage: 'drupal-exporter --config-file PATH_TO_CONFIGURATION_FILE --action'." unless ARGV.include?('--config-file')
8
+
9
+ Escort::App.create do |app|
10
+ app.summary 'Executable file of Drupal exporter'
11
+
12
+ app.options do |opts|
13
+ opts.opt :file, '--config-file', :short => '-f', :long => '--config-file', :type => :string
14
+
15
+ app.command '--extract-to-json' do |command|
16
+ command.summary 'Extract data from Drupal database dump file and save as JSON files'
17
+ command.action do |options, arguments|
18
+ Command::CLI.new(options, arguments).execute
19
+ end
20
+ end
21
+
22
+ app.command '--create-contentful-model-from-json' do |command|
23
+ command.summary 'Create content types files, based on contentful structure json file.'
24
+ command.action do |options, arguments|
25
+ Command::CLI.new(options, arguments).execute
26
+ end
27
+ end
28
+
29
+ app.command '--convert-content-model-to-json' do |command|
30
+ command.summary 'Transform content_model file into contentful_structure import form. View README'
31
+ command.action do |options, arguments|
32
+ Command::CLI.new(options, arguments).execute
33
+ end
34
+ end
35
+
36
+ end
37
+ end