wordpress-exporter 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 680c753308782d12dfc5e7712966c05cd952836f
+  data.tar.gz: 3b82f2f3f1dc436ec46d2d62793337852a98cd12
+SHA512:
+  metadata.gz: ec2f0aebefdd4117051a04363d48b013b325dd35e1e4c421d7469e8b49809de06f168ff47cc01efbb41a3fd32e8e7ad55d81eef2ea7c0fd3bf5f514a1f6eadf8
+  data.tar.gz: 4287679aeecca10f83554762e11850583801a42ea8de7a7f10b2db060d968f9af955bfaeaace9be877925b3e4c51a62b9ec4c6e318629d244cb828745bf978cf

data/.gitignore

@@ -0,0 +1,39 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalisation:
+/.bundle/
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+#RubyMine
+.idea
+
+.DS_Store

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format progress
+--order random

data/.travis.yml

@@ -0,0 +1,8 @@
+language: ruby
+rvm:
+  - 2.1.2
+  - 2.1.1
+  - 2.1
+  - 2.0.0
+  - 1.9.3
+  - jruby-19mode

data/CHANGELOG.md

@@ -0,0 +1,4 @@
+# Change Log
+## 0.0.1
+### Other
+* Initial release

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+# Specify your gem's dependencies in wordpress_exporter.gemspec
+gemspec
+
+group :test do
+  gem 'simplecov', require: false
+end

data/Gemfile.lock

@@ -0,0 +1,75 @@
+PATH
+  remote: .
+  specs:
+    wordpress-exporter (0.0.1)
+      activesupport (~> 4.1)
+      escort (~> 0.4.0)
+      http (~> 0.6)
+      i18n (~> 0.6)
+      multi_json (~> 1)
+      nokogiri (~> 1.6.3.1)
+      reverse_markdown (~> 0.6.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.0)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    escort (0.4.0)
+      nesty
+    form_data (0.1.0)
+    http (0.7.1)
+      form_data (~> 0.1.0)
+      http_parser.rb (~> 0.6.0)
+    http_parser.rb (0.6.0)
+    i18n (0.7.0)
+    json (1.8.2)
+    mini_portile (0.6.0)
+    minitest (5.5.0)
+    multi_json (1.10.1)
+    nesty (1.0.2)
+    nokogiri (1.6.3.1)
+      mini_portile (= 0.6.0)
+    rake (10.4.2)
+    reverse_markdown (0.6.0)
+      nokogiri
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-its (1.1.0)
+      rspec-core (>= 3.0.0)
+      rspec-expectations (>= 3.0.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+    simplecov (0.9.1)
+      docile (~> 1.1.0)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.8.0)
+    simplecov-html (0.8.0)
+    thread_safe (0.3.4)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.6)
+  rake
+  rspec (~> 3)
+  rspec-its (~> 1.1.0)
+  simplecov
+  wordpress-exporter!

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Contentful
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,71 @@
+Wordpress to Contentful Exporter
+=================
+
+## Description
+This adapter will allow you to extract content from a WordPress Blog and prepare it to be imported to [Contentful](https://wwww.contentful.com).
+
+The following content will be extracted:
+
+* Blog with posts
+* Categories, tags and terms from custom taxonomies
+* Attachments
+
+
+## Installation
+
+```bash
+gem install wordpress-exporter
+```
+
+This will install the `wordpress-exporter` executable on your system.
+
+
+## Setup
+To extract the blog content you need to [export](http://en.support.wordpress.com/export/) it from the WordPress blog and save it as a XML file.
+
+Further you need to define where the tool can find the XML file and the destination of the transformed content.
+Create a `settings.yml` file and fill in the `data_dir` and `wordpress_xml_path`:
+
+``` yaml
+data_dir: PATH_TO_ALL_DATA
+wordpress_xml_path: PATH_TO_XML/file.xml
+```
+
+To extract the content run:
+
+```bash
+wordpress-exporter --config-file settings.yml --extract-to-json
+```
+
+The result will be a directory structure with the WordPress content transformed into JSON files that are ready for import.
+
+Use the [generic-importer](https://github.com/contentful/generic-importer.rb) to import your blog then to Contentful.
+
+## Step by step
+
+1. [Export](http://en.support.wordpress.com/export/) the content of the blog from WordPress and save it as XML file.
+2. Create YAML file with settings (eg. settings.yml) and fill in the required parameters.
+   The generated `content types`, `entries` and `assets` will be saved to the `data_dir`.
+
+3. Extract the content from the XML file and generate the content model and JSON files for the import:
+
+    ```bash
+    wordpress-exporter --config-file settings.yml --extract-to-json
+    ```
+    If you want to create a different content model for your blog you can use `--omit-content-model`
+
+    ```bash
+    wordpress-exporter --config-file settings.yml --extract-to-json --omit-content-model
+    ```
+
+    It will only extract the content and store it as JSON, you need to take care about the content mapping yourself.
+    See the [Contentful-importer](https://github.com/contentful/generic-importer.rb) for details on how this needs to be done.
+
+4. (Optional). HTML markup can be converted to markdown:
+
+    ```bash
+    wordpress-exporter --config-file settings.yml --convert-markup
+    ```
+    This will only touch the content body of a blog post, other attributes will not be changed.
+
+5. Use the [contentful-importer](https://github.com/contentful/generic-importer.rb) to import the content to [contentful.com](https://www.contentful.com)

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'rake'
+require 'bundler/gem_tasks'
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new('spec')
+
+task default: :spec

data/bin/wordpress-exporter

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+require 'escort'
+require_relative '../lib/cli'
+
+I18n.enforce_available_locales = false
+
+fail ArgumentError, 'Set path for configuration file and define action. More details you can find in README.' if ARGF.argv.empty?
+fail ArgumentError, "Missing '--config-file' argument. The correct is form: 'wordpress-exporter --config-file PATH_TO_CONFIGURATION_FILE --action'. View README." unless ARGV.include?('--config-file')
+
+Escort::App.create do |app|
+  app.summary 'Executable file of Wordpress exporter'
+
+  app.options do |opts|
+    opts.opt :file, '--config-file', short: '-f', long: '--config-file', type: :string
+
+    app.command '--extract-to-json' do |command|
+      command.summary 'Extract data from Wordpress XML dump file and save as JSON files'
+      command.options do |opts|
+        opts.opt :omit_content_model, 'omit', short: '-o', long: '--omit-content-model', type: :boolean, default: false
+      end
+      command.action do |options, arguments|
+        Command::CLI.new(options, arguments).execute
+      end
+    end
+    app.command '--create-contentful-model-from-json' do |command|
+      command.summary 'Create content types files, based on contentful structure json file. View README'
+      command.action do |options, arguments|
+        Command::CLI.new(options, arguments).execute
+      end
+    end
+    app.command '--convert-content-model-to-json' do |command|
+      command.summary 'Transform content_model file into contentful_structure import form. View README'
+      command.action do |options, arguments|
+        Command::CLI.new(options, arguments).execute
+      end
+    end
+
+    app.command '--convert-markup' do |command|
+      command.summary 'Convert markup to markdown in post content. View README'
+      command.action do |options, arguments|
+        Command::CLI.new(options, arguments).execute
+      end
+    end
+  end
+end

data/lib/cli.rb

@@ -0,0 +1,11 @@
+require_relative 'migrator'
+require 'yaml'
+
+module Command
+  class CLI < Escort::ActionCommand::Base
+    def execute
+      setting_file = YAML.load_file(global_options[:file])
+      Migrator.new(setting_file).run(command_name, command_options)
+    end
+  end
+end

data/lib/configuration.rb

@@ -0,0 +1,28 @@
+require 'active_support/core_ext/hash'
+module Contentful
+  class Configuration
+    attr_reader :space_id,
+                :data_dir,
+                :collections_dir,
+                :entries_dir,
+                :assets_dir,
+                :wordpress_xml,
+                :settings
+
+    def initialize(settings)
+      @settings = settings
+      @data_dir = settings['data_dir']
+      validate_required_parameters
+      @wordpress_xml = settings['wordpress_xml_path']
+      @collections_dir = "#{data_dir}/collections"
+      @entries_dir = "#{data_dir}/entries"
+      @assets_dir = "#{data_dir}/assets"
+      @space_id = settings['space_id']
+    end
+
+    def validate_required_parameters
+      fail ArgumentError, 'Set PATH to data_dir. Folder where all data will be stored. View README' if settings['data_dir'].nil?
+      fail ArgumentError, 'Set PATH to Wordpress XML file. View README' if settings['wordpress_xml_path'].nil?
+    end
+  end
+end

data/lib/converters/content_types_structure_creator.rb

@@ -0,0 +1,58 @@
+module Contentful
+  module Converter
+    class ContentTypesStructureCreator
+      attr_reader :config, :logger
+
+      def initialize(config)
+        @config = config
+        @logger = Logger.new(STDOUT)
+      end
+
+      def create_content_type_json_file(content_type_name, values)
+        collection = {
+          id: values[:id],
+          name: values[:name],
+          description: values[:description],
+          displayField: values[:displayField],
+          fields: create_fields(values[:fields])
+        }
+        write_json_to_file("#{config.collections_dir}/#{content_type_name}.json", collection)
+        logger.info "Saving #{content_type_name}.json to #{config.collections_dir}"
+      end
+
+      def create_fields(fields)
+        fields.each_with_object([]) do |(field, value), results|
+          results << {
+            name: create_field(field, value).capitalize,
+            id: create_field(field, value),
+            type: create_type_field(value),
+            link_type: create_link_type_field(value),
+            link: create_link_field(value)
+          }.compact
+        end
+      end
+
+      def create_field(field, value)
+        value.is_a?(Hash) ? value[:id] : field
+      end
+
+      def create_link_type_field(value)
+        value.is_a?(Hash) ? value[:link_type] : nil
+      end
+
+      def create_type_field(value)
+        value.is_a?(Hash) ? value[:type] : value
+      end
+
+      def create_link_field(value)
+        value.is_a?(Hash) ? value[:link] : nil
+      end
+
+      def write_json_to_file(path, data)
+        File.open(path, 'w') do |file|
+          file.write(JSON.pretty_generate(data))
+        end
+      end
+    end
+  end
+end

data/lib/converters/contentful_model_to_json.rb

@@ -0,0 +1,119 @@
+require_relative 'content_types_structure_creator'
+
+module Contentful
+  module Converter
+    class ContentfulModelToJson
+      attr_reader :config, :logger, :converted_model_dir, :content_types, :omit_content_model
+
+      FIELD_TYPE = %w( Link Array )
+
+      def initialize(settings)
+        @config = settings
+        @logger = Logger.new(STDOUT)
+      end
+
+      def create_content_type_json(omit_content_model_flag = true)
+        @omit_content_model = omit_content_model_flag
+        contentful_structure = load_contentful_structure_file
+        logger.info 'Create JSON files with content types structure...'
+        contentful_structure.each do |content_type, values|
+          content_type_name = content_type_name(content_type)
+          create_directory(config.collections_dir)
+          ContentTypesStructureCreator.new(config).create_content_type_json_file(content_type_name, values)
+        end
+        logger.info 'Done!'
+      end
+
+      def convert_to_import_form
+        set_content_model_parameters
+        logger.info 'Converting Contentful model to Contentful import structure...'
+        File.open(converted_model_dir, 'w') { |file| file.write({}) }
+        contentful_file = JSON.parse(File.read(content_types))['items']
+        contentful_file.each do |content_type|
+          parsed_content_type = {
+            id: content_type['sys']['id'],
+            name: content_type['name'],
+            description: content_type['description'],
+            displayField: content_type['displayField'],
+            fields: {}.merge!(create_contentful_fields(content_type))
+          }
+          import_form = JSON.parse(File.read(converted_model_dir))
+          File.open(converted_model_dir, 'w') do |file|
+            file.write(JSON.pretty_generate(import_form.merge!(content_type['name'] => parsed_content_type)))
+          end
+        end
+        logger.info "Done! Contentful import structure file saved in #{converted_model_dir}"
+      end
+
+      def create_contentful_fields(content_type)
+        content_type['fields'].each_with_object({}) do |(field, _value), results|
+          id = link_id(field)
+          results[id] = case field['type']
+                          when 'Link'
+                            { id: field['id'], type: field['linkType'], link: 'Link' }
+                          when 'Array'
+                            { id: field['id'], type: field['type'], link_type: field['items']['linkType'], link: field['items']['type'] }
+                          else
+                            field['type']
+                        end
+        end
+      end
+
+      def link_id(field)
+        if FIELD_TYPE.include? field['type']
+          field['name'].capitalize
+        else
+          field['id']
+        end
+      end
+
+      def content_type_name(content_type)
+        I18n.transliterate(content_type).underscore.tr(' ', '_')
+      end
+
+      def create_directory(path)
+        FileUtils.mkdir_p(path) unless File.directory?(path)
+      end
+
+      # If contentful_structure JSON file exists, it will load the file. If not, it will automatically create an empty file.
+      # This file is required to convert contentful model to contentful import structure.
+      def load_contentful_structure_file
+        fail ArgumentError, 'Set PATH for contentful structure JSON file. View README' if omit_content_model?
+        file_exists? ? load_existing_contentful_structure_file : create_empty_contentful_structure_file
+      end
+
+      def omit_content_model?
+        omit_content_model == true ? config.settings['contentful_structure_dir'].nil? : false
+      end
+
+      def file_exists?
+        omit_content_model == false ? true : File.exist?(config.settings['contentful_structure_dir'])
+      end
+
+      def create_empty_contentful_structure_file
+        File.open(config.settings['contentful_structure_dir'], 'w') { |file| file.write({}) }
+        load_existing_contentful_structure_file
+      end
+
+      def load_existing_contentful_structure_file
+        if omit_content_model == true
+          JSON.parse(File.read(config.settings['contentful_structure_dir']), symbolize_names: true).with_indifferent_access
+        else
+          spec = Gem::Specification.find_by_name('wordpress-exporter')
+          JSON.parse(File.read("#{spec.gem_dir}/wordpress_settings/default_contentful_structure.json"), symbolize_names: true).with_indifferent_access
+        end
+      end
+
+      def set_content_model_parameters
+        validate_content_model_files
+        @converted_model_dir = config.settings['converted_model_dir']
+        @content_types = config.settings['content_model_json']
+      end
+
+      def validate_content_model_files
+        fail ArgumentError, 'Set PATH for content model JSON file. View README' if config.settings['content_model_json'].nil?
+        fail ArgumentError, 'Set PATH where converted content model file will be saved. View README' if config.settings['converted_model_dir'].nil?
+      end
+    end
+  end
+end