flee_to_md 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7613246f2d49ea4c2e501f142b1259cf8828ca70
+  data.tar.gz: be43b9076d3103d9dcb0a8801d2cbe2387501c37
+SHA512:
+  metadata.gz: 775c34d0f86676a94589aaf9ffa56fdb30539baa69e6e52862f21ca84cd950f85136ce911e048db1876e34672dba9ffd57afa72e626a096e4cf41499014c9a0f
+  data.tar.gz: dbd8028790806c364e47adb69d5583e6fbbe627efdf9628a21b02ba15a9195dec413ae66ef73e3f7300f2026e277acd2f3a1d1fbcdd113915f9ae8b11114798c

data/.gitignore

@@ -0,0 +1,4 @@
+*.xml
+*.zip
+.DS_STORE
+*.gem

data/README.md

@@ -0,0 +1,50 @@
+# flee to md
+
+helps convert a big xml file (like from squarespace) to markdown files
+
+## installation
+
+`gem install flee_to_md`
+
+(you'll need [rubygems](https://rubygems.org/pages/download))
+
+## usage
+
+run `flee_to_md {your_xml_file}.xml`
+
+## changelog
+
+* **2013-04-10**, v0.1.0,
+    * repackaged as a rubygem for easier installation / usage
+* **2013-04-01**, v0.0.6
+    * better support for windows (I think)
+* **2013-03-23**, v0.0.5
+    * downloads attachments
+* **2013-03-21**, v0.0.4
+    * removed some of the escaping `\` characters that kramdown was adding
+    * removed the trailing hyphens in some filenames (where the title ends in a -> like character)
+* **2013-03-20**, v0.0.3
+    * removed web interface ([too slow for (heroku's) love](http://www.youtube.com/watch?v=fiyROQNLhSU))
+    * added 1.8.7 compatibility
+* **2013-03-20**, v0.0.2
+    * added web interface
+    * titles in single quotes
+* **2013-03-19**, v0.0.1 first try
+
+## roadmap
+
+* maybe make sure it works with wordpress exports
+* maybe support other output structures
+
+## license
+
+The MIT License (MIT)
+Copyright (c) 2013 Max Jacobson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+

data/bin/flee_to_md

@@ -0,0 +1,22 @@
+#!/usr/bin/ruby
+
+require File.expand_path('../../lib/flee_to_md', __FILE__)
+require 'optparse'
+
+ARGV.push "-h" if ARGV.length == 0
+optparse = OptionParser.new do |opts|
+  opts.on( '-h', '--help', 'Display this screen' ) do
+    puts "run: flee_to_md your_xml_file.xml"
+    exit
+  end
+  opts.on( '-v', '--version', 'Version Information') do
+    puts "flee_to_md v#{Flee_to_md::VERSION}"
+    exit
+  end
+end
+
+optparse.parse!
+
+filename = ARGV[0]
+xml = File.open(filename).read
+folder = Flee_to_md::Blog.new(xml, filename).write

data/flee_to_md.gemspec

@@ -0,0 +1,25 @@
+require File.expand_path("../lib/flee_to_md", __FILE__)
+
+Gem::Specification.new do |flee_to_md|
+  flee_to_md.name          = "flee_to_md"
+  flee_to_md.version       = Flee_to_md::VERSION
+  flee_to_md.date          = Flee_to_md::LAST_UPDATED
+  flee_to_md.summary       = "Helps convert a big xml file (like from squarespace) into separate markdown files (like for statamic)"
+  flee_to_md.description   = File.open("README.md").read
+  flee_to_md.authors       = ["Max Jacobson"]
+  flee_to_md.email         = "max@maxjacobson.net"
+  flee_to_md.files         = Dir[
+                            './*.{md,gemspec}',
+                            './.gitignore',
+                            './bin/*',
+                            './lib/*'
+                          ]
+  flee_to_md.require_paths = ["lib"]
+  flee_to_md.executables   = ["flee_to_md"]
+  flee_to_md.homepage      = "http://rubygems.org/gems/flee_to_md"
+  flee_to_md.license       = "MIT"
+  flee_to_md.required_ruby_version = '>= 1.8.7'
+  flee_to_md.add_runtime_dependency 'nokogiri'
+  flee_to_md.add_runtime_dependency 'kramdown'
+  flee_to_md.add_runtime_dependency 'ruby-progressbar'
+end

data/lib/flee_to_md.rb

@@ -0,0 +1,138 @@
+require 'rubygems'
+require 'nokogiri' # helps parse xml
+require 'kramdown' # markdown implementation
+require 'ruby-progressbar' # for nice output
+require 'open-uri'
+
+module Flee_to_md
+  VERSION = "0.1.0"
+  LAST_UPDATED = "2013-04-10"
+  class Blog
+    attr_accessor :pages
+    def initialize (xml, filename)
+      @filename = filename
+      document = Nokogiri::XML normalize(xml)
+      @pages = Array.new
+      items = document.xpath("//item")
+      prog = ProgressBar.create(:title => "Reading", :total => items.length)
+      items.each_with_index do |item, index|
+        @pages.push Page.new(item)
+        # break if index == 4
+        prog.increment
+      end
+    end
+    def normalize(xml)
+      # iron out some kinks that are causing errors
+      xml.gsub(/data-image/, 'src')
+    end
+    def write
+      foldername = @filename.gsub(/\.xml$/, '')
+      @foldername = foldername
+      i = 1
+      until File.exists?(@foldername) == false
+        @foldername = "#{i}-#{foldername}"
+        i += 1
+      end
+      prog = ProgressBar.create(:title => "Writing", :total => @pages.length)
+      Dir.mkdir @foldername
+      Dir.mkdir "#{@foldername}/attachments"
+      @pages.each do |page|
+        if page.attachment_url != "" # is an attachment
+          extension = /(\.[\w\d]+)$/
+          if page.attachment_url =~ extension
+            filename = "#{page.title}#{page.attachment_url.match(extension)[1]}"
+          else
+            filename = page.title
+          end
+          open(page.attachment_url) do |f|
+            File.open("#{@foldername}/attachments/#{filename}","wb") do |file|
+              file.puts f.read
+            end
+          end
+        else # is not an attachment
+          str = "---\n"
+          str << "title: '#{page.title}'\n"
+          str << "date: #{page.date.strftime "%Y-%m-%d %H:%M:%S %z"}\n"
+          if page.categories.length > 0
+            str << "categories: [#{page.categories.join(", ")}]\n"
+          end
+          if page.tags.length > 0
+            str << "tags: [#{page.tags.join(", ")}]\n"
+          end
+          if page.type == "linkpost"
+            str << "link: #{page.link}\n"
+          end
+          str << "---\n\n"
+          str << page.markdown.gsub(/\\("|'|\[|\]|\:)/,'\1')
+          page_filename = "#{page.date.strftime('%Y-%m-%d')}-#{page.title.downcase.gsub(/\s+/, '-').gsub(/[^-\w\d]/,'')}.md".gsub(/[^\w\d](.md)$/, '.md')
+          File.open("#{@foldername}/#{page_filename}", 'w') { |file| file.write(str) }
+        end
+        prog.increment
+      end
+      puts "Written to #{@foldername}/"
+      return @foldername
+    end
+  end
+
+  class Page
+    attr_accessor :title, :permalink, :link, :date, :html, :markdown,
+      :type, :post_id, :status, :tags, :categories, :link,
+      :attachment_url, :attachment_filename
+    def initialize (item)
+      @title = item.xpath("title").children.to_s
+      @permalink = item.xpath("link").children.to_s
+      date = item.xpath("pubDate").children.to_s
+      @date = pubdate_to_ruby_time(date)
+      @html = item.xpath("content:encoded").children.to_s.slice(9..-4)
+      @markdown = Kramdown::Document.new(@html, :input => 'html').to_kramdown
+      @type = item.xpath("wp:post_type").children.to_s
+      @post_id = item.xpath("wp:post_id").children.to_s
+      @status = item.xpath("wp:status").children.to_s
+      link = item.xpath("wp:postmeta/wp:meta_value").children.to_s
+      if link != ""
+        @type = "linkpost"
+        @link = link
+      end
+      @attachment_url = item.xpath("wp:attachment_url").children.to_s
+      if @attachment_url != ""
+        @attachment_filename = item.xpath("wp:post_name").children.to_s
+        @type = "file"
+      end
+      @categories = []
+      @tags = []
+      metas = item.xpath("category")
+      metas.each do |meta|
+        kind = meta.attributes["domain"].children.to_s # "category" or "post_tag"
+        # nickname = meta.attributes["nicename"].to_s # tag or category name
+        proper_name = meta.children.to_s.slice(9..-4)
+        if kind == "category"
+          @categories.push proper_name
+          # @categories.push({:nickname => nickname, :proper_name => proper_name})
+        elsif kind == "post_tag"
+          @tags.push proper_name
+          # @tags.push({:nickname => nickname, :proper_name => proper_name})
+        end
+      end
+    end
+    def pubdate_to_ruby_time (str)
+      pattern = /\w{3}, (\d{2}) (\w{3}) (\d{4}) (\d{2}):(\d{2}):(\d{2}) (\+\d{4})/
+      match = str.match(pattern)
+      day = match[1]
+      month = match[2]
+      year = match[3]
+      hour = match[4]
+      # weird error: many posts have publish times at 24 oclock.
+      # was causing errors when trying to construct a ruby time object. out of range
+      # what the hell? I'm assuming that means midnight
+      hour = "0" if hour == "24"
+      minute = match[5]
+      second = match[6].to_i
+      offset = match[7].sub(/(\d{2})(\d{2})/,'\1:\2')
+      if RUBY_VERSION.to_f >= 1.9
+        return Time.new year, month, day, hour, minute, second, offset
+      else
+        return Time.local(year, month, day, hour, minute, second)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: flee_to_md
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Max Jacobson
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: kramdown
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |+
+  # flee to md
+
+  helps convert a big xml file (like from squarespace) to markdown files
+
+  ## installation
+
+  `gem install flee_to_md`
+
+  (you'll need [rubygems](https://rubygems.org/pages/download))
+
+  ## usage
+
+  run `flee_to_md {your_xml_file}.xml`
+
+  ## changelog
+
+  * **2013-04-10**, v0.1.0,
+      * repackaged as a rubygem for easier installation / usage
+  * **2013-04-01**, v0.0.6
+      * better support for windows (I think)
+  * **2013-03-23**, v0.0.5
+      * downloads attachments
+  * **2013-03-21**, v0.0.4
+      * removed some of the escaping `\` characters that kramdown was adding
+      * removed the trailing hyphens in some filenames (where the title ends in a -> like character)
+  * **2013-03-20**, v0.0.3
+      * removed web interface ([too slow for (heroku's) love](http://www.youtube.com/watch?v=fiyROQNLhSU))
+      * added 1.8.7 compatibility
+  * **2013-03-20**, v0.0.2
+      * added web interface
+      * titles in single quotes
+  * **2013-03-19**, v0.0.1 first try
+
+  ## roadmap
+
+  * maybe make sure it works with wordpress exports
+  * maybe support other output structures
+
+  ## license
+
+  The MIT License (MIT)
+  Copyright (c) 2013 Max Jacobson
+
+  Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+  The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+email: max@maxjacobson.net
+executables:
+- flee_to_md
+extensions: []
+extra_rdoc_files: []
+files:
+- ./README.md
+- ./flee_to_md.gemspec
+- ./.gitignore
+- ./bin/flee_to_md
+- ./lib/flee_to_md.rb
+- bin/flee_to_md
+homepage: http://rubygems.org/gems/flee_to_md
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.0
+signing_key: 
+specification_version: 4
+summary: Helps convert a big xml file (like from squarespace) into separate markdown
+  files (like for statamic)
+test_files: []