drupal-exporter 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f48e838d6b34c48f0e78d52078eed0def0430e8b
+  data.tar.gz: 25476436a73bcfd87b02645d8eb1b3d775893665
+SHA512:
+  metadata.gz: 127ead0fc0925ced6dfc0cbc1060836cb5c5fe8b95ebf77929e720303efd29616d9a9e3ab681ae1fea0ec8c8c4edb33386c00c08cf9c8f01d6e491ecf33ebb53
+  data.tar.gz: 4a2fa1d328dd3485f7f86dfcfceda7b42651cf796ae7d882f77e21c9a0cf9ae55af0fb32f0431318a14186c72068e52d1f03bc4a6a381400ea54f2e02e1e9602

data/.gitignore

@@ -0,0 +1,39 @@
+*.gem
+*.rbc
+/.config
+/coverage
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+#RubyMine
+.idea
+
+.DS_Store

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format progress
+--order random

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 2.1.2
+  - 2.1.1
+  - 2.1
+  - 2.0.0
+  - 1.9.3
+#  - jruby-19mode

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+# Change Log
+## 0.0.1
+### Other
+* Initial release

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+# Specify your gem's dependencies in drupal_exporter.gemspec
+gemspec
+
+group :test do
+  gem 'simplecov', require: false
+end

data/Gemfile.lock

@@ -0,0 +1,72 @@
+PATH
+  remote: .
+  specs:
+    drupal-exporter (0.0.1)
+      activesupport (~> 4.1)
+      escort (~> 0.4.0)
+      http (~> 0.6)
+      i18n (~> 0.6)
+      multi_json (~> 1)
+      mysql2 (~> 0.3)
+      pg (~> 0.17.0)
+      sequel (~> 4.15)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.1.8)
+      i18n (~> 0.6, >= 0.6.9)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.1)
+      tzinfo (~> 1.1)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    escort (0.4.0)
+      nesty
+    http (0.6.3)
+      http_parser.rb (~> 0.6.0)
+    http_parser.rb (0.6.0)
+    i18n (0.6.11)
+    json (1.8.1)
+    minitest (5.4.3)
+    multi_json (1.10.1)
+    mysql2 (0.3.17)
+    nesty (1.0.2)
+    pg (0.17.1)
+    rake (10.4.2)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-its (1.1.0)
+      rspec-core (>= 3.0.0)
+      rspec-expectations (>= 3.0.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+    sequel (4.17.0)
+    simplecov (0.9.1)
+      docile (~> 1.1.0)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  drupal-exporter!
+  rake
+  rspec (~> 3)
+  rspec-its (~> 1.1.0)
+  simplecov

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Contentful
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,251 @@
+Drupal to Contentful Exporter
+=================
+
+# This Gem supports only Drupal version 7.
+
+## Description
+This tool will extract the following content from a Drupal database dump file:
+
+* Content types (Blog, Article, Page, Custom content types)
+* Tags
+* Vocabularies
+* Users
+
+
+## Installation
+
+```bash
+gem install drupal-exporter
+```
+
+This will install the `drupal-exporter` executable.
+
+
+## Step by step
+
+1. Create a YAML file with the required parameters (eg. `settings.yml`):
+
+    ```yml
+    # PATH TO ALL DATA
+    data_dir: PATH_TO_ALL_DATA
+
+    # CONNECTING TO A DATABASE
+    adapter: mysql2
+    host: localhost
+    database: drupal_database_name
+    user: username
+    password: secret_password
+
+    # DRUPAL SETTINGS
+    drupal_content_types_json: drupal_settings/drupal_content_types.json
+    drupal_boolean_columns: drupal_settings/boolean_columns.yml
+    drupal_base_url: http://example_hostname.com
+
+    # CONVERT CONTENTFUL MODEL TO CONTENTFUL IMPORT STRUCTURE
+    content_model_json: PATH_TO_CONTENTFUL_MODEL_JSON_FILE/contentful_model.json
+    converted_model_dir: PATH_WHERE_CONVERTED_CONTENT_MODEL_WILL_BE_SAVED/contentful_structure.json
+
+    contentful_structure_dir: PATH_TO_CONTENTFUL_STRUCTURE_JSON_FILE/contentful_structure.json
+    ```
+2. (Not required to extract data). Create the `contentful_structure.json`. First you need to create a content model using the [Contentful web application](www.contentful.com). Then you can download the content model using the content management api and use the content model for the import:
+
+       ```bash
+        curl -X GET \
+             -H 'Authorization: Bearer ACCESS_TOKEN' \
+             'https://api.contentful.com/spaces/SPACE_ID/content_types' > contentful_model.json
+       ```
+
+       It will create `contentful_model.json` file, which you need to transform into the `contentful_structure.json` using:
+
+       ```bash
+        drupal-exporter --config-file settings.yml --convert-content-model-to-json
+       ```
+
+       The converted content model will be saved as JSON file in the `converted_model_dir` path.
+
+       Now you can generate content types JSON files.
+
+       ```bash
+       drupal-exporter --config-file settings.yml  --create-contentful-model-from-json
+       ```
+       It will create the content types JSON files which represent your content structure for the import.
+
+3. Create the `drupal_content_types.json` file. This file contains the mapped structure of your database.
+
+    Mapping structure:
+
+    ```javascript
+        machine_name_of_content_type : {
+            contentful_api_field_1 : column_machine_name_1,
+            contentful_api_field_2: column_machine_name_2,
+            contentful_api_field_3 : column_machine_name_3
+        }
+    ```
+
+    You can find a sample mapping file in the `drupal_settings/drupal_content_types.json` directory.
+
+4. (Optional). Boolean values. Sequel converts boolean values `0,1`, stored in the database only when the field is TINYINT(1) type.
+    To map the value of `0,1` to `false, true`, you have to specify the column names in the yaml file (eg. `boolean_columns.yml`) and
+    specify the path to this file in the `settings.yml` file, parameter `drupal_boolean_columns`.
+
+    Example:
+
+    ```yml
+    - field_if_content_type
+    - field_boolean
+    ```
+5. Extract the content from the database and generate the JSON files for the import:
+
+    ```bash
+    drupal-exporter --config-file settings.yml --extract-to-json
+    ```
+    It will _only_ extract the content and store it as JSON files, nothing will be uploaded yet.
+
+6. Use the [contentful-importer](https://github.com/contentful/generic-importer.rb) to import the content to [contentful.com](https://www.contentful.com)
+
+
+
+## Setup ##
+
+Create settings a YML file (eg. `settings.yml`) to define all required parameters.
+Assuming we are going to work with either MySQL, SQLite or a PostgreSQL database, you must define credentials to connect Drupal database.
+An example configuration for connecting with a MySQL database named "drupal_database_name":
+
+```yml
+adapter: mysql2
+user: username
+host: localhost
+database: drupal_database_name
+password: secret_password
+```
+
+### Available Adapters ###
+
+* PostgreSQL => postgres
+* MySQL => mysql2
+* SQlite => sqlite
+
+
+## Content Types ##
+
+To be able to properly map the Drupal content types to the Contentful content types they must be identical by name.
+
+Example:
+
+```
+Drupal name of content type => 'Blog'
+Contentful name of content type => 'Blog'
+```
+
+## Tags ##
+
+These content types are exported from the Drupal database by default and assigned to every content type. There is no need to specify them in the content type structure.
+
+They will be saved with the following api field ids:
+```
+Tags => 'tags'
+```
+
+### Custom tag content types ###
+
+If you want to add tags that you define in a custom table, you need to specify them by adding an addition parameter to the hash:
+```javascript
+"term_tagging":{
+    "table": "field_term_tagging"
+}
+```
+
+### Machine name ###
+
+To find `machine name` in your Drupal structure, sign in to your Admin console and open `Structure` section. Next to the name of Content type, will be located machine name in brackets.
+To find `machine name` for individual field, go into `manage fields` option.
+
+### Booleans ###
+
+To map columns of boolean values, you need to create YML file ( eg. boolean_columns.yml ) and define machine names of boolean columns.
+
+Example:
+```yml
+- field_if_content_type
+- field_boolean
+```
+
+The path to the  `drupal_boolean_columns` file is defined in the settings.yml.
+
+```yml
+drupal_boolean_columns: PATH_TO_YML_FILE
+```
+
+### Assets & Images ###
+
+Your files and assets need to be available and accessible through the internet.
+For this purpose, you must define the `drupal_base_url` in the settings.yml file so that the importer will be able to create them.
+
+```yml
+drupal_base_url: http://example_hostname.com
+```
+
+### Mapping structure ###
+
+Create JSON file with content types structure:
+
+```javascript
+"machine_name_of_content_type" : {
+"contentful_api_fiel_idd" : "column_machine_name",
+"contentful_api_field_id2" : "column_machine_name2",
+"contentful_api_field_id3" : "column_machine_name23"
+}
+```
+
+Example structure:
+
+```javascript
+{
+    "article": {
+        "body": "body",
+        "image": "field_image"
+    },
+    "page": {
+        "body": "body"
+    },
+    "blog": {
+        "body": "body"
+    },
+    "content_type": {
+        "body": "body",
+        "age": "field_age",
+        "if_content_type": "field_if_content_type",
+        "name": "field_first_name"
+    }
+}
+```
+
+In settings.yml file define path to the `drupal_content_types_json` file.
+
+```yml
+drupal_content_types_json: PATH_TO_JSON_FILE
+```
+
+### Example settings.yml file ###
+
+```yml
+data_dir: /tmp/data
+
+#Connecting to a database
+adapter: mysql2
+host: localhost
+database: database_name
+user: username
+password: password
+
+# Drupal
+drupal_content_types_json: PATH_TO_FILE/drupal_content_types.json
+drupal_boolean_columns: PATH_TO_FILE/boolean_columns.yml
+drupal_base_url: http://example_hostname.com
+```
+
+Command to extract data:
+
+```bash
+drupal-exporter --config-file settings.yml --extract-to-json
+```

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'rake'
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new('spec')
+
+
+task default: :spec

data/bin/drupal-exporter

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'escort'
+require_relative '../lib/cli'
+
+fail ArgumentError, 'Set the path to the configuration file and define an action. More information can be found in the README.' if ARGF.argv.empty?
+fail ArgumentError, "Missing '--config-file' argument. Usage: 'drupal-exporter --config-file PATH_TO_CONFIGURATION_FILE --action'." unless ARGV.include?('--config-file')
+
+Escort::App.create do |app|
+  app.summary 'Executable file of Drupal exporter'
+
+  app.options do |opts|
+    opts.opt :file, '--config-file', :short => '-f', :long => '--config-file', :type => :string
+
+    app.command '--extract-to-json' do |command|
+      command.summary 'Extract data from Drupal database dump file and save as JSON files'
+      command.action do |options, arguments|
+        Command::CLI.new(options, arguments).execute
+      end
+    end
+
+    app.command '--create-contentful-model-from-json' do |command|
+      command.summary 'Create content types files, based on contentful structure json file.'
+      command.action do |options, arguments|
+        Command::CLI.new(options, arguments).execute
+      end
+    end
+
+    app.command '--convert-content-model-to-json' do |command|
+      command.summary 'Transform content_model file into contentful_structure import form. View README'
+      command.action do |options, arguments|
+        Command::CLI.new(options, arguments).execute
+      end
+    end
+
+  end
+end