locomotivecms-freight 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1fc95af0db9dc7fbcca0bfc5b46cefb7d81e54f9
+  data.tar.gz: 7d775e2262246f556a7804a0216ee43eb13374c4
+SHA512:
+  metadata.gz: 2bb89b63aa9c160f5d7adbab3e40236ee47812b41e45372f41909c3c04920dcd6f0ce3fa496f70b5ad5d2c4fd3685964453a92ae7bbd72fe884a9e2be25d9a94
+  data.tar.gz: b09c31d678839c33f5153f9da6bd46c1da4a3161087828c7f59ae947f9bf8703b3d8541a1d6df8ef9e9ed12594e50881c2a31eda9cc39e28c4b5abfba665191d

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.2.1
+before_install: gem install bundler -v 1.10.6

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in locomotivecms-freight.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Joel Helbling
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,82 @@
+# Locomotivecms::Freight
+
+Imports posts, comments and images from any WordPress site.  Rewrites image tags and internal links.
+
+## Installation
+
+This gem adds rake tasks to a LocomotiveCMS Wagon-generated project directory.
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'locomotivecms-freight'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install locomotivecms-freight
+
+Finally, place this somewhere in your project's Rakefile:
+
+```ruby
+require 'locomotivecms/freight/tasks'
+```
+
+## Usage
+
+First, on your WordPress site, log in as an admin user, and navigate to Tools -> Export.  Select
+"All content" and click "Download Export File".  Save the file and remember the location.
+
+Next, install the LocomotiveCMS content_types needed for WordPress posts and comments:
+
+    bundle exec rake wp:install_content_types
+
+Each post's body will be converted to markdown.  After this step, you will need to push the new
+content types to each engine to which you will be deploying:
+
+    bundle exec wagon sync production -v -r content_types
+
+Now that we have the necessary content_types to receive them, we can import all posts, comments and
+images from a WordPress site, run the following command:
+
+    bundle exec rake wp:import TARGET=production XML=/path/to/my-wordpress-export.xml
+
+Note that `TARGET` should reference one of the environments defined in your `config/deploy.yml` file.
+Also be aware that image importing will not work unless the WordPress site from which you are exporting
+is up and operational.  This is because the XML export file from a WordPress site contains pages, posts
+and comments, but not other file resources such as images, audio/video files etc.
+
+Also be aware that as of this writing, Freight's only concern is with posts and comments; it ignores
+pages.
+
+If there were images downloaded then you will need to push them up to your target LocomotiveCMS engine:
+
+    bundle exec wagon sync production -v
+
+To remove all _imported_ posts and associated comments, run this:
+
+    bundle exec rake wp:clean TARGET=production
+
+And finally, to remove all imported posts and associated comments and then re-import, do this:
+
+    bundle exec rake wp:reload TARGET=production XML=/path/to/my-wordpress-export.xml
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/joelhelbling/locomotivecms-freight.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "locomotivecms/freight/tasks"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "rake"
+require "locomotivecms/freight"
+require "locomotivecms/freight/tasks"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/freight

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(File.join File.dirname(__FILE__), '..', 'lib', 'locomotivecms', 'freight')
+
+wp_export = ARGV[0]
+
+puts <<-SORRY
+So sorry, but this script is not yet functional.  For now, if you'll just add
+
+require 'locomotivecms/freight/tasks'
+
+to your wagon project's Rakefile, you should get a slew of useful tasks for importing
+from a WordPress site.
+SORRY

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/locomotivecms/freight/content_types/comments.yml.erb

@@ -0,0 +1,67 @@
+name: Comments
+slug: comments
+description: Comments on blog posts
+label_field_name: email
+order_by: created_at
+order_direction: asc
+
+# Activate public 'create' API (e.g for a contact form)
+# public_submission_enabled: false
+
+# Array of emails to be notified of new entries made with the public API
+# public_submission_accounts: ['john@example.com']
+
+# Control the display of the content type in the back-office.
+# display_settings:
+#   seo: false              # display the SEO tab for the content entries
+#   advanced: false         # display the Advanced tab for the content entries
+#   position: 1             # position in the sidebar menu
+#   hidden: false           # hidden for authors?
+
+entry_template: |
+  <a href={{ link }}>{{ entry.name }} &lt;{{  entry._label }}&gt;</a>
+  [ {{ entry.commented_at }} ]<br>
+  {{ entry.content }}
+
+fields:
+- name:
+    label: Name
+    type: string
+    required: false
+    hint: the huumon who made the wisecrack
+    localized: false
+
+- email:
+    label: Email
+    type: string
+    required: false
+    hint: commenter's email address
+    localized: false
+
+- url:
+    label: Url
+    type: string
+    required: false
+    hint: the commenter's (optional) homepage url
+    localized: false
+
+- commented_at:
+    label: Commented at
+    type: date_time
+    required: true
+    hint: when what was said, was said, exactly
+    localized: false
+
+- content:
+    label: Comment
+    type: text
+    required: true
+    hint: Explanatory text displayed in the back office
+    localized: false
+    text_formatting: html
+
+- post:
+    label: Post
+    hint: The blog post this comment was commenting on
+    type: belongs_to
+    class_name: posts

data/lib/locomotivecms/freight/content_types/posts.yml.erb

@@ -0,0 +1,115 @@
+# Human readable name of this type
+name: Posts
+
+# Lowercase, underscored handle used to access this type
+slug: posts
+
+# Explanatory text displayed in the back-office
+description: Blog post
+
+# Slug of field used to identify entries by default, such as the title
+label_field_name: title
+
+# Valid values: manually, created_at, updated_at, or the slug of any field
+order_by: posted_at
+
+# Valid values: asc (ascending) and desc (descending). Set to asc by default.
+order_direction: desc
+
+# Specify a field slug to group entries by that field in the back-office.
+group_by: _visible
+
+# Activate public 'create' API (e.g for a contact form)
+# public_submission_enabled: false
+
+# Array of emails to be notified of new entries made with the public API
+# public_submission_accounts: ['john@example.com']
+
+# Control the display of the content type in the back-office.
+display_settings:
+  seo: true              # display the SEO tab for the content entries
+  advanced: true         # display the Advanced tab for the content entries
+#   position: 1             # position in the sidebar menu
+#   hidden: false           # hidden for authors?
+
+# By default, the back-office displays the _label property (see label_field_name) of the content entry. This can be modified by writing your own Liquid template below:
+# entry_template: '<a href="{{ link }}">{{ entry._label }}</a>' # The default template
+entry_template: |
+  <a href={{ link }}>
+    <h4 class=text-info>{{ entry.title }}</h4>
+  </a>
+  <small><em class=text-muted>{{ entry.posted_at | date: "%I:%M %p, %a, %b %d, %Y" }}</em></small><br />
+  <p>{{ entry.teaser }}</p>
+
+# A list describing each field
+fields:
+- title:
+    label: Title
+    type: string
+    required: true
+    localized: false
+
+- keywords:
+    label: Keywords
+    type: tags
+    required: false
+    hint: one or more tags or keywords for this post
+    localized: false
+
+- posted_at:
+    label: Posted at
+    type: date_time
+    required: false
+    hint: the date when the article was or will be posted
+    localized: false
+
+- body:
+    label: Body
+    type: text
+    required: false
+    hint: <%= @format == 'markdown' ? 'uses markdown' : 'Use the WYSIWYG, Luke...' %>
+    localized: false
+    text_formatting: <%= @format %>
+
+- teaser:
+    label: Teaser / synopsys
+    type: text
+    required: false
+    hint: <%= @format == 'markdown' ? 'uses markdown' : 'Use the WYSIWYG, Luke...' %>
+    localized: false
+    text_formatting: <%= @format %>
+
+- featured_image:
+    label: Featured image
+    type: file
+    required: false
+    hint: An image to accompany the heading of the post
+    localized: false
+
+- open_for_comment: # WordPress: comment_status
+    label: Open for comments?
+    type: boolean
+    required: true
+    hint: Allow comments on this post (or don't)
+    localized: false
+
+# Some other fields to consider (culled from the CJ WP export):
+# - comment_status (open/closed) --make a bool open_for_comment
+# - creator/author
+# - published (boolean)
+# - category (belongs_to) try using tags
+
+- source:
+    label: Content source
+    type: string
+    required: false
+    hint: For use by import scripts
+    localized: false
+
+- comments:
+    label: Comments
+    hint: A description of the relationship for the editors
+    type: has_many
+    class_name: comments
+    inverse_of: post
+    ui_enabled: true

data/lib/locomotivecms/freight/html_to_markdown.rb

@@ -0,0 +1,138 @@
+require 'active_support/core_ext/object/blank'
+require 'ostruct'
+
+module Locomotivecms
+  module Freight
+
+    module HtmlToMarkdown
+      class << self
+        def convert_to_markdown html
+          html = convert_italics html
+          html = convert_bold html
+          html = convert_tables html
+          html = convert_preformatted html
+          html = convert_paragraphs html
+          html = convert_headings html
+          html = convert_images html
+          html = convert_links html
+          html
+        end
+
+        def convert_italics html
+          html \
+            .gsub(/<(i .*?>|i>|em.*?>)[\n\s]*/i, '_')
+            .gsub(/[\n\s]*\<(\/i|\/em)[^\>]*\>/i, '_')
+        end
+
+        def convert_bold html
+          html \
+            .gsub(/\<(b .*?>|b>|strong.*?>)[\n\s]*/i, '**')
+            .gsub(/[\n\s]*\<(\/b|\/strong)[^\>]*\>/i, '**')
+        end
+
+        def convert_preformatted html
+          html.gsub(/<pre.*?>.*?<\/pre.*?>/m) do |pre|
+            text = pre.match(/<pre.*?>(.*?)<\/pre.*?>/m)[1]
+            text \
+              .strip
+              .split(/\n\n/m)
+              .map do |para|
+                para.split(/\n/).join("<br />\n")
+              end \
+              .join("\n\n")
+              .chomp
+          end.gsub(/\n\n\n/, "\n\n")
+        end
+
+        def convert_tables html
+          html.gsub(/\s*<table.*?>.*?<\/table>/mi) do |table|
+            markdown = "\n\n|"
+            table.scan(/<tr.*?>.*?<\/tr>/mi) do |row|
+              row.scan(/<td.*?>.*?<\/td>/mi) do |cell|
+                markdown << cell.gsub(/^<td.*?>/mi, '').gsub(/<\/td>$/mi, '')
+                markdown << "|"
+              end
+            end
+            markdown << "\n"
+          end
+        end
+
+        def convert_paragraphs html
+          if html.match(/<\/?p\s?.*?>/)
+            html = html.dup \
+              .split(/[\s\n]*\<\/{0,1}p.*?\>[\s\n]*/i)
+              .reject{|chunk| chunk.match /^[\s\n]*$/ }
+              .join("\n\n")
+              .chomp + "\n"
+          end
+          html
+        end
+
+        def convert_headings html
+          html.gsub(/\<h\d.*?\>.*?\<\/h\d\>/mi) { |heading|
+            match = heading.match /\<h(\d).*?\>(.*?)\<\/h\d\>/mi
+            heading_prefix = "#" * match[1].to_i
+            heading_text = match[2].gsub(/\n/, ' ').gsub(/^\s*/, '').gsub(/\s*$/, '')
+            "#{heading_prefix} #{heading_text}"
+          }
+        end
+
+        def convert_links html
+          html = html.dup
+          urls = []
+          html.gsub!(/\<a.*?\>.*?\<\/a\>/mi) { |link|
+            index = urls.length
+            match = link.match /\<a.*href=["']?([^\s"']+)["']?.*\>(.*?)\<\/a\>/mi
+            if match.nil?
+              raise "No match for link: #{link}"
+            end
+            link_href = match[1]
+            link_text = match[2]
+            urls[index] = link_href
+            "[#{link_text}][#{index + 1}]"
+          }
+          html << "\n"
+          html << urls.each_with_index.map do |url, i|
+            index = i + 1
+            "[#{index}]: #{url}"
+          end.join("\n")
+          html << "\n"
+        end
+
+        def convert_images html
+          html = html.dup
+          images = {}
+          html.gsub!(/<img.*?\/?>/mi) { |img|
+            index = images.length
+            src_match = img.match /<img.*?src=["']?([^\s"']+)["']?.*?\/?>/mi
+            return img unless src_match.present?
+            src = src_match[1]
+            alt_match = img.match /<img.*?alt=["']([^"']+)["'].*?\/?>/mi
+            alt = alt_match[1] if alt_match.present?
+            title_match = img.match /<img.*?title=["']([^"']+)["'].*?\/?>/mi
+            title = title_match[1] if title_match.present?
+            if alt.blank?
+              if title.present?
+                alt = title
+              else
+                alt = src.gsub(/^.*\//,'').gsub(/\..*$/,'')
+              end
+            elsif title.blank?
+              title = alt
+            end
+            title = title.present? ? " \"#{title}\"" : nil
+            index = "img-#{(images.size + 1).to_s.rjust(2, '0')}"
+            images[index] = OpenStruct.new({ url: src, title: title })
+            "![#{alt}][#{index}]"
+          }
+          html << "\n"
+          html << images.each_pair.map do |index, img|
+            "[#{index}]: #{img.url}#{img.title}"
+          end.join("\n")
+          html << "\n"
+        end
+      end
+    end
+
+  end
+end

data/lib/locomotivecms/freight/post.rb

@@ -0,0 +1,37 @@
+module Locomotivecms
+  module Freight
+
+    class Post < Squares::Base
+      IMPORTABLE_PROPERTIES = [
+        :title, :keywords, :posted_at, :body, :teaser, :source, :_visible
+      ]
+      properties :title, :keywords, :posted_at, :body, :teaser, :source,
+        :wp_id
+      property :_visible, default: true
+      property :_id
+      property :_slug
+      property :xml
+
+      def links
+        body.scan(/^\[\d+\]\:.*$/).map do |link|
+          link.chomp.gsub(/^\[.*?\]:\s?/, '')
+        end
+      end
+
+      def image_urls
+        body.scan(/^\[img-\d+\]:.*$/).map do |img_url|
+          img_url.chomp.gsub(/^\[img-\d+\]:\s?/, '').gsub(/\s".*$/, '')
+        end
+      end
+
+      def cms_params
+        IMPORTABLE_PROPERTIES.reduce({}) do |params, property|
+          params.tap do |p|
+            p[property] = self.send property
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/locomotivecms/freight/tasks.rake

@@ -0,0 +1,93 @@
+require 'nokogiri'
+require 'locomotive/coal'
+require 'yaml'
+require 'erb'
+require 'pry'
+require 'locomotivecms/freight/wordpress_importer'
+
+namespace :wp do
+  desc "install post and comment content_types"
+  task :install_content_types do
+    unless File.exists?('app/content_types')
+      raise "Are you sure this is a LocomotiveCMS Wagon project directory?  I don't see 'app/content_types'\n\n"
+    end
+    @format = ENV['FORMAT'] || 'html'
+    template_dir = File.expand_path('../content_types', __FILE__)
+    %w(posts comments).each do |template_name|
+      template_file = "#{template_name}.yml"
+      template = File.read(File.join template_dir, "#{template_file}.erb")
+      renderer = ERB.new posts_template
+      File.open("app/content_types/#{template_file}", 'w') do |fh|
+        fh.write renderer.result(binding)
+      end
+    end
+  end
+
+  desc "import WordPress export xml"
+  task import: [:importer, :input_file] do
+    @importer.import @input_file, 'markdown'
+    @importer.rewrite_internal_urls
+    @importer.rewrite_images
+  end
+
+  desc "remove all WordPress posts from the targeted environment"
+  task clean: :importer do
+    @importer.clean!
+  end
+
+  desc "clean and then re-import"
+  task reload: [:clean, :import]
+
+  task testing: :importer do
+    binding.pry
+  end
+
+  task all_the_links: [:importer, :input_file] do
+    puts @importer.all_links(@input_file).join("\n")
+  end
+
+  task all_the_images: [:importer, :input_file] do
+    puts @importer.all_images(@input_file).join("\n")
+  end
+
+  task all_the_tables: [:importer, :input_file] do
+    puts @importer.all_tables(@input_file).join("\n")
+  end
+
+  task remove_non_visible: :importer do
+    @importer.remove_non_visible_posts!
+  end
+
+  task input_file: :importer do
+    @input_file = ENV['XML'] || ENV['XML_FILE'] || ENV['FILE']
+
+    unless @input_file
+      raise "You must provide an xml file by adding 'XML=/path/to/my.xml'"
+    end
+  end
+
+  task importer: :client do
+    @importer = WordpressImporter.new @client
+  end
+
+  task client: :config do
+    @client = Locomotive::Coal::Client.new(@host, { email: @email, api_key: @api_key }).scope_by(@handle)
+  end
+
+  task :config do
+    target_env = ENV['TARGET']
+    config = YAML.load(File.read('config/deploy.yml'))
+
+    unless target_env && config.keys.include?(target_env)
+      raise "you must set the target environment, one of #{config.keys.join(', ')}"
+    end
+
+    @host    = config[target_env]['host']
+    @handle  = config[target_env]['handle']
+    @email   = config[target_env]['email']
+    @api_key = config[target_env]['api_key']
+  end
+
+end # namespace :wp
+
+

data/lib/locomotivecms/freight/tasks.rb

@@ -0,0 +1,14 @@
+module Locomotivecms
+  module Freight
+    class Tasks
+      include Rake::DSL if defined? Rake::DSL
+
+      def install_tasks
+        load File.expand_path('../tasks.rake', __FILE__)
+      end
+    end
+  end
+end
+
+Locomotivecms::Freight::Tasks.new.install_tasks
+

data/lib/locomotivecms/freight/version.rb

@@ -0,0 +1,5 @@
+module Locomotivecms
+  module Freight
+    VERSION = "0.0.1"
+  end
+end

data/lib/locomotivecms/freight/wordpress_importer.rb

@@ -0,0 +1,261 @@
+require 'active_support/core_ext/object/blank'
+require 'ostruct'
+require 'squares'
+require 'locomotivecms/freight/post'
+require 'locomotivecms/freight/html_to_markdown'
+
+module Locomotivecms
+  module Freight
+
+    class WordpressImporter
+      attr_reader :client
+
+      SOURCE='WordPress Importer'
+
+      def initialize(client)
+        @client = client
+      end
+
+      def all_links input_file
+        parse_all_posts(input_file).map do |item|
+          body = item.xpath('content:encoded').text
+          body.scan(/<a .*?<\/a>/mi).map{|link| link.strip }
+        end.flatten
+      end
+
+      def all_images input_file
+        parse_all_posts(input_file).map do |item|
+          body = item.xpath('content:encoded').text
+          body.scan(/<img.*?>/mi).map{|img| img.strip }
+        end.flatten
+      end
+
+      def all_tables input_file
+        parse_all_posts(input_file).map do |item|
+          body = item.xpath('content:encoded').text
+          body.scan(/<table.*?<\/table>/i).map{|table| table.strip }
+        end.flatten
+      end
+
+      def all_imported_posts filter={ source: SOURCE }
+        [].tap do |all_posts|
+          page = 1
+          while page do
+            posts = client.contents.posts.all(filter, page: page)
+            all_posts << posts
+            page = posts._next_page
+          end
+        end.flatten
+      end
+
+      def all_comments filter={}
+        [].tap do |all_comments|
+          page = 1
+          while page do
+            comments = client.contents.comments.all(filter, page: page)
+            all_comments << comments
+            page = comments._next_page
+          end
+        end.flatten
+      end
+
+      def import import_file, format=html
+        metrics = OpenStruct.new posts: 0, comments: 0
+        parse_all_posts(import_file).each do |item|
+
+          post = post_for item
+          puts "creating post [[ #{post.title} ]]"
+          p = client.contents.posts.create post.cms_params
+          post._id = p._id
+          post.save
+          metrics.posts += 1
+
+          comments = comments_for item, post
+          unless comments.empty?
+            puts "  -- creating #{comments.count} comments..."
+            comments.each do |comment|
+              client.contents.comments.create comment
+              metrics.comments += 1
+            end
+          end
+        end
+        puts "Imported #{metrics.posts} posts and #{metrics.comments} comments."
+      end
+
+      def rewrite_internal_urls
+        return unless Post.any?
+        metrics = OpenStruct.new links: 0
+
+        Post.each do |post|
+          needs_update = false
+          post.links.each do |link|
+            if linked_post = Post[link]
+              metrics.links += 1
+              puts "[ #{post.title} ]: rewriting \"#{link}\" -> \"/posts/#{linked_post._slug}\""
+              post.body.gsub! /#{link}/, "/posts/#{linked_post._slug}"
+              post.save
+              needs_update = true
+            end
+          end
+          client.contents.posts.update(post._id, { body: post.body }) if needs_update
+        end
+
+        puts "Rewrote #{metrics.links} links in #{Post.count} posts."
+      end
+
+      def rewrite_images
+        return unless Post.any?
+        metrics = OpenStruct.new images: 0
+
+        # ensure /public/images/posts exists
+
+        Post.each do |post|
+          needs_update = false
+          default_host = post.id.gsub(/^(https?:\/\/[^\/]*).*$/, '\1')
+
+          post.image_urls.each do |image_url|
+            original_image_url = image_url.dup
+            image_url = "#{default_host}#{image_url}" if image_url.match(/^\//)
+            unless (File.directory?('public/images/posts'))
+              Dir.mkdir('public/images/posts')
+            end
+
+            image_file_name = image_url.gsub(/^.*\//, '')
+            puts <<-TEXT.strip_heredoc
+
+              image_url: #{image_url}
+              image_file_name: #{image_file_name}
+              ================================================================================
+            TEXT
+            curl = `curl #{image_url} > public/images/posts/#{image_file_name}`
+            puts curl
+            if curl.match /<url> malformed/m
+              raise <<-ERROR.strip_heredoc
+                Problem downloading image "#{image_url}"
+                image_file_name: #{image_file_name}
+                post.body:
+
+                #{post.body}
+
+              ERROR
+            end
+            metrics.images += 1
+
+            post.body.gsub! /#{original_image_url}/, "/sites/#{site_id}/theme/images/posts/#{image_file_name}"
+            post.save
+            needs_update = true
+          end
+          client.contents.posts.update(post._id, { body: post.body }) if needs_update
+        end
+
+        puts "Rewrote #{metrics.images} images in #{Post.count} posts."
+        puts "Don't forget to `bundle exec wagon push ENV -r theme_assets`"
+      end
+
+      def clean!
+        remove_posts
+      end
+
+      def remove_non_visible_posts!
+        remove_posts _visible: false
+      end
+
+      def destroy thing
+        content_type = thing.content_type_slug.to_sym
+        contents_action content_type, :destroy, thing._id
+      end
+
+      def create content_type, params
+        contents_action content_type.to_sym, :create, params
+      end
+
+      private
+
+      def contents_action content_type, action, params
+        client.contents.send(content_type).send(action, params)
+      end
+
+      def parse_xml input_file
+        File.open(input_file) { |fh| Nokogiri::XML(fh) }
+      end
+
+      def parse_all_posts import_file
+        parse_xml(import_file).xpath('//item').select do |item|
+          item.xpath('wp:post_type').text == 'post' && filtered?(item)
+        end
+      end
+
+      def filtered? item
+        !ENV['NAME'] || item.xpath('wp:post_name').text == ENV['NAME']
+      end
+
+      def remove_posts filter={ source: SOURCE }
+        metrics = OpenStruct.new posts: 0, comments: 0
+        posts = all_imported_posts filter
+        puts "Preparing to delete #{posts.count} posts..."
+        posts.each do |post|
+          puts "removing [[ #{post.title} ]]"
+          destroy post
+          metrics.posts += 1
+          post_comments = all_comments post: post._id
+          if post_comments.count > 0
+            puts "  -- also removing #{post_comments.count} comments..."
+            post_comments.each do |comment|
+              destroy comment
+              metrics.comments += 1
+            end
+          end
+        end
+        puts "Removed #{metrics.posts} posts and #{metrics.comments} comments."
+      end
+
+      def post_for item
+        post = Post.new item.xpath('link').text,
+          title:     resolve_post_title(item),
+          keywords:  item.xpath('category').text,
+          posted_at: item.xpath('pubDate').text,
+          body:      HtmlToMarkdown.convert_to_markdown(item.xpath('content:encoded').text),
+          teaser:    item.xpath('excerpt:encoded').text,
+          wp_id:     item.xpath('wp:post_id').text,
+          source:    SOURCE
+        post.tap do |post|
+          if post.teaser.nil? || post.teaser == ""
+            post[:teaser] = post[:body].gsub(/\<.*?\>/, '')[0..300] + '...'
+          end
+          post.save
+        end
+      end
+
+      def comments_for item, post
+        item.xpath('wp:comment').map do |comment|
+          {
+            name:         comment.xpath('wp:comment_author').text,
+            email:        comment.xpath('wp:comment_author_email').text,
+            url:          comment.xpath('wp:comment_author_url').text,
+            commented_at: comment.xpath('wp:comment_date').text,
+            content:      comment.xpath('wp:comment_content').text,
+            post:         post._id
+          }.tap do |comment|
+            if comment[:email].nil? || comment[:email] == ''
+              comment[:email] = "n/a (#{comment[:name]})"
+            end
+          end
+        end
+      end
+
+      def resolve_post_title item
+        title = item.xpath('title').text
+        title = item.xpath('wp:post_name').text if title.blank?
+        title = item.xpath('wp:post_id').text if title.blank?
+        title
+      end
+
+      def site_id
+        @handle ||= @client.options['handle']
+        @site_id ||= @client.sites.all.select{ |site| site.handle == @handle }.first._id
+      end
+
+    end
+
+  end
+end

data/lib/locomotivecms/freight.rb

@@ -0,0 +1,7 @@
+require "locomotivecms/freight/version"
+
+module Locomotivecms
+  module Freight
+    # Your code goes here...
+  end
+end

data/locomotivecms-freight.gemspec

@@ -0,0 +1,38 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'locomotivecms/freight/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "locomotivecms-freight"
+  spec.version       = Locomotivecms::Freight::VERSION
+  spec.authors       = ["Joel Helbling"]
+  spec.email         = ["joel@joelhelbling.com"]
+
+  spec.summary       = %q{Loads WordPress content into LocomotiveCMS.}
+  spec.description   = %q{Specifically posts, comments and images.  Rewrites images and internal links.}
+  spec.homepage      = "https://github.com/joelhelbling/locomotivecms-freight"
+  spec.license       = "MIT"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri", "~> 1.6.7"
+  spec.add_dependency "locomotivecms_coal", "~> 1.0"
+  spec.add_dependency "activesupport", "~> 4.2"
+  spec.add_dependency "squares", "~> 0.3"
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
+end

metadata

@@ -0,0 +1,178 @@
+--- !ruby/object:Gem::Specification
+name: locomotivecms-freight
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Joel Helbling
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-03-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.7
+- !ruby/object:Gem::Dependency
+  name: locomotivecms_coal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  name: squares
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Specifically posts, comments and images.  Rewrites images and internal
+  links.
+email:
+- joel@joelhelbling.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/freight
+- bin/setup
+- lib/locomotivecms/freight.rb
+- lib/locomotivecms/freight/content_types/comments.yml.erb
+- lib/locomotivecms/freight/content_types/posts.yml.erb
+- lib/locomotivecms/freight/html_to_markdown.rb
+- lib/locomotivecms/freight/post.rb
+- lib/locomotivecms/freight/tasks.rake
+- lib/locomotivecms/freight/tasks.rb
+- lib/locomotivecms/freight/version.rb
+- lib/locomotivecms/freight/wordpress_importer.rb
+- locomotivecms-freight.gemspec
+homepage: https://github.com/joelhelbling/locomotivecms-freight
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: Loads WordPress content into LocomotiveCMS.
+test_files: []