contentful-importer 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (112) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +38 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +8 -0
  5. data/CHANGELOG.md +4 -0
  6. data/Gemfile +7 -0
  7. data/Gemfile.lock +87 -0
  8. data/LICENSE +22 -0
  9. data/README.md +327 -0
  10. data/Rakefile +7 -0
  11. data/bin/contentful-importer +85 -0
  12. data/contentful_importer.gemspec +38 -0
  13. data/example_settings/contentful_model.json +288 -0
  14. data/example_settings/contentful_structure.json +78 -0
  15. data/example_settings/files_meaning.txt +15 -0
  16. data/example_settings/settings.yml +16 -0
  17. data/lib/cli.rb +13 -0
  18. data/lib/configuration.rb +36 -0
  19. data/lib/converters/content_types_structure_creator.rb +60 -0
  20. data/lib/converters/contentful_model_to_json.rb +109 -0
  21. data/lib/importer/data_organizer.rb +117 -0
  22. data/lib/importer/mime_content_type.rb +564 -0
  23. data/lib/importer/parallel_importer.rb +429 -0
  24. data/lib/json_schema_validator.rb +64 -0
  25. data/lib/migrator.rb +39 -0
  26. data/lib/version.rb +3 -0
  27. data/spec/fixtures/import_files/assets/image/image_1.json +9 -0
  28. data/spec/fixtures/import_files/assets/image/image_2.json +9 -0
  29. data/spec/fixtures/import_files/assets/image/image_3.json +9 -0
  30. data/spec/fixtures/import_files/assets/image/image_4.json +9 -0
  31. data/spec/fixtures/import_files/collections/job_skills.json +13 -0
  32. data/spec/fixtures/import_files/collections/jobs.json +37 -0
  33. data/spec/fixtures/import_files/collections/profile.json +19 -0
  34. data/spec/fixtures/import_files/collections/user.json +36 -0
  35. data/spec/fixtures/import_files/entries/job_skills/job_skills_1.json +7 -0
  36. data/spec/fixtures/import_files/entries/job_skills/job_skills_10.json +7 -0
  37. data/spec/fixtures/import_files/entries/job_skills/job_skills_2.json +7 -0
  38. data/spec/fixtures/import_files/entries/job_skills/job_skills_3.json +7 -0
  39. data/spec/fixtures/import_files/entries/job_skills/job_skills_4.json +7 -0
  40. data/spec/fixtures/import_files/entries/job_skills/job_skills_5.json +7 -0
  41. data/spec/fixtures/import_files/entries/job_skills/job_skills_6.json +7 -0
  42. data/spec/fixtures/import_files/entries/job_skills/job_skills_7.json +7 -0
  43. data/spec/fixtures/import_files/entries/job_skills/job_skills_8.json +7 -0
  44. data/spec/fixtures/import_files/entries/job_skills/job_skills_9.json +7 -0
  45. data/spec/fixtures/import_files/entries/jobs/jobs_1.json +56 -0
  46. data/spec/fixtures/import_files/entries/jobs/jobs_2.json +55 -0
  47. data/spec/fixtures/import_files/entries/jobs/jobs_4.json +49 -0
  48. data/spec/fixtures/import_files/entries/profile/profile_1.json +12 -0
  49. data/spec/fixtures/import_files/entries/profile/profile_2.json +12 -0
  50. data/spec/fixtures/import_files/entries/user/user_1.json +24 -0
  51. data/spec/fixtures/import_files/entries/user/user_2.json +20 -0
  52. data/spec/fixtures/import_files/logs/log_published_assets.csv +4 -0
  53. data/spec/fixtures/import_files/logs/log_published_entries.csv +23 -0
  54. data/spec/fixtures/import_files/logs/success_assets.csv +4 -0
  55. data/spec/fixtures/import_files/logs/success_published_assets.csv +0 -0
  56. data/spec/fixtures/import_files/logs/success_published_entries.csv +22 -0
  57. data/spec/fixtures/import_files/logs/success_thread_0.csv +12 -0
  58. data/spec/fixtures/import_files/logs/success_thread_1.csv +11 -0
  59. data/spec/fixtures/import_files/threads/0/1TVvxCqoRq0qUYAOQuOqys_2.json +20 -0
  60. data/spec/fixtures/import_files/threads/0/2soCP557HGKoOOK0SqmMOm_1.json +7 -0
  61. data/spec/fixtures/import_files/threads/0/2soCP557HGKoOOK0SqmMOm_10.json +7 -0
  62. data/spec/fixtures/import_files/threads/0/2soCP557HGKoOOK0SqmMOm_2.json +7 -0
  63. data/spec/fixtures/import_files/threads/0/2soCP557HGKoOOK0SqmMOm_3.json +7 -0
  64. data/spec/fixtures/import_files/threads/0/2soCP557HGKoOOK0SqmMOm_4.json +7 -0
  65. data/spec/fixtures/import_files/threads/0/2soCP557HGKoOOK0SqmMOm_5.json +7 -0
  66. data/spec/fixtures/import_files/threads/0/2soCP557HGKoOOK0SqmMOm_6.json +7 -0
  67. data/spec/fixtures/import_files/threads/0/2soCP557HGKoOOK0SqmMOm_7.json +7 -0
  68. data/spec/fixtures/import_files/threads/0/6H6pGAV1PUsuoAW26Iu48W_1.json +9 -0
  69. data/spec/fixtures/import_files/threads/0/6H6pGAV1PUsuoAW26Iu48W_2.json +9 -0
  70. data/spec/fixtures/import_files/threads/0/6H6pGAV1PUsuoAW26Iu48W_3.json +9 -0
  71. data/spec/fixtures/import_files/threads/0/6H6pGAV1PUsuoAW26Iu48W_4.json +9 -0
  72. data/spec/fixtures/import_files/threads/0/6H6pGAV1PUsuoAW26Iu48W_5.json +9 -0
  73. data/spec/fixtures/import_files/threads/1/1TVvxCqoRq0qUYAOQuOqys_1.json +24 -0
  74. data/spec/fixtures/import_files/threads/1/1TVvxCqoRq0qUYAOQuOqys_2.json +20 -0
  75. data/spec/fixtures/import_files/threads/1/2soCP557HGKoOOK0SqmMOm_6.json +7 -0
  76. data/spec/fixtures/import_files/threads/1/2soCP557HGKoOOK0SqmMOm_7.json +7 -0
  77. data/spec/fixtures/import_files/threads/1/2soCP557HGKoOOK0SqmMOm_8.json +7 -0
  78. data/spec/fixtures/import_files/threads/1/2soCP557HGKoOOK0SqmMOm_9.json +7 -0
  79. data/spec/fixtures/import_files/threads/1/4L1bg4WQ5aWQMiE82ouag_1.json +56 -0
  80. data/spec/fixtures/import_files/threads/1/4L1bg4WQ5aWQMiE82ouag_2.json +55 -0
  81. data/spec/fixtures/import_files/threads/1/4L1bg4WQ5aWQMiE82ouag_4.json +49 -0
  82. data/spec/fixtures/import_files/threads/1/4WFZh4MwC4Mc0EQWAeOY8A_1.json +12 -0
  83. data/spec/fixtures/import_files/threads/1/4WFZh4MwC4Mc0EQWAeOY8A_2.json +12 -0
  84. data/spec/fixtures/import_files/threads/assets/0/image_1.json +9 -0
  85. data/spec/fixtures/import_files/threads/assets/0/image_2.json +9 -0
  86. data/spec/fixtures/import_files/threads/assets/0/image_3.json +9 -0
  87. data/spec/fixtures/import_files/threads/assets/0/image_4.json +9 -0
  88. data/spec/fixtures/settings/contentful_model.json +252 -0
  89. data/spec/fixtures/settings/contentful_structure.json +73 -0
  90. data/spec/fixtures/settings/contentful_structure_test.json +66 -0
  91. data/spec/fixtures/settings/settings.yml +12 -0
  92. data/spec/fixtures/vcr_cassettes/create_asset.yml +621 -0
  93. data/spec/fixtures/vcr_cassettes/create_entry.yml +122 -0
  94. data/spec/fixtures/vcr_cassettes/create_space.yml +87 -0
  95. data/spec/fixtures/vcr_cassettes/import_assets.yml +2822 -0
  96. data/spec/fixtures/vcr_cassettes/import_content_types.yml +1915 -0
  97. data/spec/fixtures/vcr_cassettes/import_entries.yml +5066 -0
  98. data/spec/fixtures/vcr_cassettes/import_entry.yml +363 -0
  99. data/spec/fixtures/vcr_cassettes/invalid_credentials.yml +69 -0
  100. data/spec/fixtures/vcr_cassettes/publish_an_entry.yml +214 -0
  101. data/spec/fixtures/vcr_cassettes/publish_asset.yml +895 -0
  102. data/spec/fixtures/vcr_cassettes/publish_entries.yml +5121 -0
  103. data/spec/fixtures/vcr_cassettes/valid_credentials.yml +7360 -0
  104. data/spec/lib/configuration_spec.rb +22 -0
  105. data/spec/lib/importer/parallel_importer_spec.rb +161 -0
  106. data/spec/lib/json_schema_validator_spec.rb +63 -0
  107. data/spec/lib/migrator_spec.rb +99 -0
  108. data/spec/spec_helper.rb +12 -0
  109. data/spec/support/db_rows_json.rb +9 -0
  110. data/spec/support/shared_configuration.rb +22 -0
  111. data/spec/support/vcr.rb +16 -0
  112. metadata +436 -0
@@ -0,0 +1,12 @@
1
+ {
2
+ "id": "profile_1",
3
+ "nickname": "Nickname 1",
4
+ "user_id": 1,
5
+ "created_at": "2014-12-16T11:35:31+00:00",
6
+ "updated_at": "2014-12-16T11:35:31+00:00",
7
+ "database_id": 1,
8
+ "user": {
9
+ "type": "Entry",
10
+ "id": "user_1"
11
+ }
12
+ }
@@ -0,0 +1,12 @@
1
+ {
2
+ "id": "profile_2",
3
+ "nickname": "Nickname 2",
4
+ "user_id": 2,
5
+ "created_at": "2014-12-16T11:35:48+00:00",
6
+ "updated_at": "2014-12-16T11:35:48+00:00",
7
+ "database_id": 2,
8
+ "user": {
9
+ "type": "Entry",
10
+ "id": "user_2"
11
+ }
12
+ }
@@ -0,0 +1,24 @@
1
+ {
2
+ "id": "user_1",
3
+ "first_name": "FirstName",
4
+ "last_name": "LastName",
5
+ "birthday": "2009-04-16T09:43:00+00:00",
6
+ "created_at": "2014-12-16T09:43:43+00:00",
7
+ "updated_at": "2014-12-16T09:43:43+00:00",
8
+ "database_id": 1,
9
+ "custom_nick": "Nickname 1",
10
+ "profile": {
11
+ "type": "profile",
12
+ "id": "profile_1"
13
+ },
14
+ "job_adds": [
15
+ {
16
+ "type": "jobs",
17
+ "id": "jobs_1"
18
+ },
19
+ {
20
+ "type": "jobs",
21
+ "id": "jobs_2"
22
+ }
23
+ ]
24
+ }
@@ -0,0 +1,20 @@
1
+ {
2
+ "id": "user_2",
3
+ "first_name": "FirstName User 2",
4
+ "last_name": "LastName User 2",
5
+ "birthday": "2014-12-16T09:43:00+00:00",
6
+ "created_at": "2014-12-16T09:45:08+00:00",
7
+ "updated_at": "2014-12-16T09:45:08+00:00",
8
+ "database_id": 2,
9
+ "custom_nick": "Nickname 2",
10
+ "profile": {
11
+ "type": "profile",
12
+ "id": "profile_2"
13
+ },
14
+ "job_adds": [
15
+ {
16
+ "type": "jobs",
17
+ "id": "jobs_4"
18
+ }
19
+ ]
20
+ }
@@ -0,0 +1,4 @@
1
+ image_1
2
+ image_2
3
+ image_3
4
+ image_4
@@ -0,0 +1,23 @@
1
+ job_skills_1
2
+ user_1
3
+ user_2
4
+ job_skills_10
5
+ job_skills_6
6
+ job_skills_2
7
+ job_skills_7
8
+ job_skills_3
9
+ job_skills_8
10
+ job_skills_4
11
+ job_skills_9
12
+ job_skills_5
13
+ jobs_1
14
+ comment_1
15
+ jobs_2
16
+ comment_2
17
+ jobs_4
18
+ comment_3
19
+ comment_4
20
+ profile_1
21
+ profile_2
22
+ comment_5
23
+ comment_5
@@ -0,0 +1,4 @@
1
+ image_1
2
+ image_2
3
+ image_3
4
+ image_4
@@ -0,0 +1,22 @@
1
+ user_1
2
+ job_skills_1
3
+ user_2
4
+ job_skills_10
5
+ job_skills_6
6
+ job_skills_2
7
+ job_skills_7
8
+ job_skills_3
9
+ job_skills_8
10
+ job_skills_4
11
+ job_skills_9
12
+ job_skills_5
13
+ jobs_1
14
+ comment_1
15
+ jobs_2
16
+ comment_2
17
+ jobs_4
18
+ comment_3
19
+ profile_1
20
+ comment_4
21
+ profile_2
22
+ comment_5
@@ -0,0 +1,12 @@
1
+ 2soCP557HGKoOOK0SqmMOm_1.json
2
+ 2soCP557HGKoOOK0SqmMOm_10.json
3
+ 2soCP557HGKoOOK0SqmMOm_2.json
4
+ 2soCP557HGKoOOK0SqmMOm_3.json
5
+ 2soCP557HGKoOOK0SqmMOm_4.json
6
+ 2soCP557HGKoOOK0SqmMOm_5.json
7
+ 6H6pGAV1PUsuoAW26Iu48W_1.json
8
+ 6H6pGAV1PUsuoAW26Iu48W_2.json
9
+ 6H6pGAV1PUsuoAW26Iu48W_3.json
10
+ 6H6pGAV1PUsuoAW26Iu48W_4.json
11
+ 6H6pGAV1PUsuoAW26Iu48W_5.json
12
+ file_path
@@ -0,0 +1,11 @@
1
+ 1TVvxCqoRq0qUYAOQuOqys_1.json
2
+ 1TVvxCqoRq0qUYAOQuOqys_2.json
3
+ 2soCP557HGKoOOK0SqmMOm_6.json
4
+ 2soCP557HGKoOOK0SqmMOm_7.json
5
+ 2soCP557HGKoOOK0SqmMOm_8.json
6
+ 2soCP557HGKoOOK0SqmMOm_9.json
7
+ 4L1bg4WQ5aWQMiE82ouag_1.json
8
+ 4L1bg4WQ5aWQMiE82ouag_2.json
9
+ 4L1bg4WQ5aWQMiE82ouag_4.json
10
+ 4WFZh4MwC4Mc0EQWAeOY8A_1.json
11
+ 4WFZh4MwC4Mc0EQWAeOY8A_2.json
@@ -0,0 +1,20 @@
1
+ {
2
+ "id": "user_2",
3
+ "first_name": "FirstName User 2",
4
+ "last_name": "LastName User 2",
5
+ "birthday": "2014-12-16T09:43:00+00:00",
6
+ "created_at": "2014-12-16T09:45:08+00:00",
7
+ "updated_at": "2014-12-16T09:45:08+00:00",
8
+ "database_id": 2,
9
+ "custom_nick": "Nickname 2",
10
+ "profile": {
11
+ "type": "profile",
12
+ "id": "profile_2"
13
+ },
14
+ "job_adds": [
15
+ {
16
+ "type": "jobs",
17
+ "id": "jobs_4"
18
+ }
19
+ ]
20
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_1",
3
+ "name": "Commercial awareness",
4
+ "created_at": "2014-12-16T09:45:58+00:00",
5
+ "updated_at": "2014-12-16T09:45:58+00:00",
6
+ "database_id": 1
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_10",
3
+ "name": "Confidence",
4
+ "created_at": "2014-12-16T09:47:32+00:00",
5
+ "updated_at": "2014-12-16T09:47:32+00:00",
6
+ "database_id": 10
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_2",
3
+ "name": "Communication",
4
+ "created_at": "2014-12-16T09:46:07+00:00",
5
+ "updated_at": "2014-12-16T09:46:07+00:00",
6
+ "database_id": 2
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_3",
3
+ "name": "Teamwork",
4
+ "created_at": "2014-12-16T09:46:16+00:00",
5
+ "updated_at": "2014-12-16T09:46:16+00:00",
6
+ "database_id": 3
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_4",
3
+ "name": "egotiation and persuasion",
4
+ "created_at": "2014-12-16T09:46:27+00:00",
5
+ "updated_at": "2014-12-16T09:46:27+00:00",
6
+ "database_id": 4
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_5",
3
+ "name": "Problem solving",
4
+ "created_at": "2014-12-16T09:46:42+00:00",
5
+ "updated_at": "2014-12-16T09:46:42+00:00",
6
+ "database_id": 5
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_6",
3
+ "name": "Leadership",
4
+ "created_at": "2014-12-16T09:46:51+00:00",
5
+ "updated_at": "2014-12-16T09:46:51+00:00",
6
+ "database_id": 6
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_7",
3
+ "name": "Organisation",
4
+ "created_at": "2014-12-16T09:47:00+00:00",
5
+ "updated_at": "2014-12-16T09:47:00+00:00",
6
+ "database_id": 7
7
+ }
@@ -0,0 +1,9 @@
1
+ {
2
+ "id": "comment_1",
3
+ "subject": "Title comment 1",
4
+ "content": "Body comment 1",
5
+ "job_add_id": 1,
6
+ "created_at": "2014-12-16T10:26:20+00:00",
7
+ "updated_at": "2014-12-16T10:26:20+00:00",
8
+ "database_id": 1
9
+ }
@@ -0,0 +1,9 @@
1
+ {
2
+ "id": "comment_2",
3
+ "subject": "Title comment 2",
4
+ "content": "Body comment 2",
5
+ "job_add_id": 1,
6
+ "created_at": "2014-12-16T10:50:22+00:00",
7
+ "updated_at": "2014-12-16T10:50:22+00:00",
8
+ "database_id": 2
9
+ }
@@ -0,0 +1,9 @@
1
+ {
2
+ "id": "comment_3",
3
+ "subject": "Title comment 3",
4
+ "content": "Body comment 3",
5
+ "job_add_id": 1,
6
+ "created_at": "2014-12-16T10:51:19+00:00",
7
+ "updated_at": "2014-12-16T10:51:19+00:00",
8
+ "database_id": 3
9
+ }
@@ -0,0 +1,9 @@
1
+ {
2
+ "id": "comment_4",
3
+ "subject": "Comment 1",
4
+ "content": "Comment body 1",
5
+ "job_add_id": 2,
6
+ "created_at": "2014-12-16T10:53:47+00:00",
7
+ "updated_at": "2014-12-16T10:53:47+00:00",
8
+ "database_id": 4
9
+ }
@@ -0,0 +1,9 @@
1
+ {
2
+ "id": "comment_5",
3
+ "subject": "Comment 2",
4
+ "content": "Body Comment 2",
5
+ "job_add_id": 2,
6
+ "created_at": "2014-12-16T11:34:46+00:00",
7
+ "updated_at": "2014-12-16T11:34:46+00:00",
8
+ "database_id": 5
9
+ }
@@ -0,0 +1,24 @@
1
+ {
2
+ "id": "user_1",
3
+ "first_name": "FirstName",
4
+ "last_name": "LastName",
5
+ "birthday": "2009-04-16T09:43:00+00:00",
6
+ "created_at": "2014-12-16T09:43:43+00:00",
7
+ "updated_at": "2014-12-16T09:43:43+00:00",
8
+ "database_id": 1,
9
+ "custom_nick": "Nickname 1",
10
+ "profile": {
11
+ "type": "profile",
12
+ "id": "profile_1"
13
+ },
14
+ "job_adds": [
15
+ {
16
+ "type": "jobs",
17
+ "id": "jobs_1"
18
+ },
19
+ {
20
+ "type": "jobs",
21
+ "id": "jobs_2"
22
+ }
23
+ ]
24
+ }
@@ -0,0 +1,20 @@
1
+ {
2
+ "id": "user_2",
3
+ "first_name": "FirstName User 2",
4
+ "last_name": "LastName User 2",
5
+ "birthday": "2014-12-16T09:43:00+00:00",
6
+ "created_at": "2014-12-16T09:45:08+00:00",
7
+ "updated_at": "2014-12-16T09:45:08+00:00",
8
+ "database_id": 2,
9
+ "custom_nick": "Nickname 2",
10
+ "profile": {
11
+ "type": "profile",
12
+ "id": "profile_2"
13
+ },
14
+ "job_adds": [
15
+ {
16
+ "type": "jobs",
17
+ "id": "jobs_4"
18
+ }
19
+ ]
20
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_6",
3
+ "name": "Leadership",
4
+ "created_at": "2014-12-16T09:46:51+00:00",
5
+ "updated_at": "2014-12-16T09:46:51+00:00",
6
+ "database_id": 6
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_7",
3
+ "name": "Organisation",
4
+ "created_at": "2014-12-16T09:47:00+00:00",
5
+ "updated_at": "2014-12-16T09:47:00+00:00",
6
+ "database_id": 7
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_8",
3
+ "name": "Perseverance and motivation",
4
+ "created_at": "2014-12-16T09:47:11+00:00",
5
+ "updated_at": "2014-12-16T09:47:11+00:00",
6
+ "database_id": 8
7
+ }
@@ -0,0 +1,7 @@
1
+ {
2
+ "id": "job_skills_9",
3
+ "name": "Ability to work under pressure",
4
+ "created_at": "2014-12-16T09:47:23+00:00",
5
+ "updated_at": "2014-12-16T09:47:23+00:00",
6
+ "database_id": 9
7
+ }
@@ -0,0 +1,56 @@
1
+ {
2
+ "id": "jobs_1",
3
+ "name": "First new Job",
4
+ "specification": "Job description",
5
+ "created_at": "2014-12-16T10:26:03+00:00",
6
+ "updated_at": "2014-12-16T10:26:03+00:00",
7
+ "image_id": 3,
8
+ "user_id": 1,
9
+ "database_id": 1,
10
+ "first_name": "FirstName",
11
+ "subjects_comments": [
12
+ "Title comment 1",
13
+ "Title comment 2",
14
+ "Title comment 3"
15
+ ],
16
+ "image": {
17
+ "type": "File",
18
+ "id": "image_3"
19
+ },
20
+ "creator": {
21
+ "type": "Entry",
22
+ "id": "user_1"
23
+ },
24
+ "comments": [
25
+ {
26
+ "type": "comment",
27
+ "id": "comment_1"
28
+ },
29
+ {
30
+ "type": "comment",
31
+ "id": "comment_2"
32
+ },
33
+ {
34
+ "type": "comment",
35
+ "id": "comment_3"
36
+ }
37
+ ],
38
+ "skills": [
39
+ {
40
+ "type": "job_skills",
41
+ "id": "job_skills_1"
42
+ },
43
+ {
44
+ "type": "job_skills",
45
+ "id": "job_skills_3"
46
+ },
47
+ {
48
+ "type": "job_skills",
49
+ "id": "job_skills_8"
50
+ },
51
+ {
52
+ "type": "job_skills",
53
+ "id": "job_skills_10"
54
+ }
55
+ ]
56
+ }