tender_import 0.0.1

data/README.md

@@ -0,0 +1,18 @@
+# Tender Import Scripts
+
+This is a repository of code for producing [Tender import
+archives](https://help.tenderapp.com/faqs/setup-installation/importing),
+for Tender customers who wish to move their discussions from another service.
+
+The methods used to produce an import archive will vary with the service
+and the facilities they provide, such as data dumps or API access. These
+scripts are being made available in the hopes that they'll be useful, but
+it's up to you to review them before running them on your own computer or
+against an existing service. We aren't responsible if your computer blows
+up or you are banned from an existing service for TOS violations, or both.
+
+That said, if you make useful or necessary modifications to a script, or
+produce a script for importing from a new service, please
+[open a Tender discussion](https://help.tenderapp.com/discussions/suggestions#new_topic_form)
+or send a pull request to let us know.
+

data/bin/zendesk2tender

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby -rubygems -Ilib
+#
+# Produce a Tender import archive by collecting tickets and discussions from
+# the ZenDesk API. Requires the ZenDesk subdomain and login credentials.
+#
+# For more info: https://help.tenderapp.com/faqs/setup-installation/importing
+#
+#   Usage:
+#     zendesk2tender -e <email> -p <password> -s <subdomain>
+#
+#   `zendesk2tender --help' displays detailed option info.
+#
+#   Prerequisites:
+#     # Ruby gems
+#     gem install faraday -v "~>0.4.5"
+#     gem install trollop
+#     gem install yajl-ruby
+#     # Python tools (must be in your PATH)
+#     html2text.py: # https://github.com/aaronsw/html2text
+#
+require 'tender_import'
+TenderImport::ZendeskApiImport.run

data/lib/tender_import/archive.rb

@@ -0,0 +1,216 @@
+#
+# Provides a Ruby API for constructing Tender import archives.
+#
+# https://help.tenderapp.com/faqs/setup-installation/importing
+#
+# ## Example
+#
+#     # Currently requires a site name.
+#     archive = TenderImport::Archive.new('tacotown')
+#
+#     # Can add users, categories and discussions to the archive.
+#     archive.add_user :email => 'frank@tacotown.com', :state => 'support'
+#     archive.add_user :email => 'bob@bobfoo.com'
+#
+#     # When you add a category you'll get a handle needed to add discusions.
+#     category = archive.add_category :name => 'Tacos'
+#
+#     # Discussions must have at least one comment.
+#     archive.add_discussion category, :title => 'your tacos',
+#       :author_email => 'bob@bobfoo.com',
+#       :comments => [{
+#         :author_email => 'bob@bobfoo.com',
+#         :body => 'They are not so good.'
+#       }, {
+#         :author_email => 'frank@tacotown.com',
+#         :body => 'You have terrible taste in tacos. Good day, sir.'
+#       }]
+#
+#     # By default, files are written as you add them, so this will just
+#     # assemble a gzipped tar archive from those files.
+#     filename = archive.write_archive
+#     puts "your import file is #{filename}"
+#
+#     # If any errors are reported, some records were not included in the archive.
+#     if !archive.report.empty?
+#       puts "Problems reported: ", *archive.report
+#     end
+#
+require 'yajl'
+require 'fileutils'
+class TenderImport::Archive
+  class Error < StandardError; end
+  include FileUtils
+  attr_reader :site, :report, :stats, :buffer
+
+  # Options:
+  #
+  # buffer:: When true, don't flush to disk until the end. Defaults to false.
+  #
+  def initialize site_name, options = {}
+    @site = site_name
+    @export_dir = ".#{site_name}-export-#{$$}"
+    @report = []
+    @import = {}
+    @stats = {}
+    @buffer = options.key?(:buffer) ? !!options[:buffer] : false
+    @category_counter = Hash.new(0)
+  end
+
+  # Returns the params on success, nil on failure
+  def add_user params
+    validate_and_store :user, {:state => 'user'}.merge(params)
+  end
+  
+  # Returns a handle needed for adding discussions
+  def add_category params
+    cat = validate_and_store :category, params
+    cat ? category_key(cat) : nil
+  end
+  
+  def add_discussion category_key, params
+    raise Error, "add_discussion: missing category key" if category_key.nil?
+    validate_and_store :discussion, params, :key => category_key
+  end
+
+  def category_key cat
+    "category:#{category_id cat}".downcase
+  end
+
+  def category_id cat
+    cat[:name].gsub(/\W+/,'_').downcase
+  end
+
+  def categories
+    @import[:category]
+  end
+
+  def discussions category_key
+    raise Error, "discussions: missing category key" if category_key.nil?
+    @import[category_key] || []
+  end
+  
+  def users
+    @import[:user]
+  end
+
+  def write_archive
+    write_users if users
+    write_categories_and_discussions if categories
+    export_file = "export_#{site}.tgz"
+    system "tar -zcf #{export_file} -C #{export_dir} ."
+    system "rm -rf #{export_dir}"
+    return export_file
+  end
+
+  def write_user user
+    return unless user
+    mkdir_p export_dir('users')
+    File.open(File.join(export_dir('users'), "#{user[:email].gsub(/\W+/,'_')}.json"), "w") do |file|
+      file.puts Yajl::Encoder.encode(user)
+    end
+  end
+
+  def write_users
+    users.each do |u|
+      write_user u
+    end
+  end
+
+  def write_category c
+    mkdir_p export_dir('categories')
+    File.open(File.join(export_dir('categories'), "#{category_id(c)}.json"), "w") do |file|
+      file.puts Yajl::Encoder.encode(c)
+    end
+  end
+
+  def write_categories_and_discussions
+    categories.each do |c|
+      write_category c
+      write_discussions c
+    end
+  end
+
+  def write_discussion category_id, discussion
+    @category_counter[category_id] += 1
+    dir = File.join(export_dir('categories'), category_id)
+    mkdir_p dir
+    File.open(File.join(dir, "#{@category_counter[category_id]}.json"), "w") do |file|
+      file.puts Yajl::Encoder.encode(discussion)
+    end
+  end
+
+  def write_discussions category
+    discussions(category_key(category)).each do |d|
+      write_discussion category_id(category), d
+    end
+  end
+
+  protected
+
+  def validate_and_store *args
+    type, params, options = args
+    options ||= {}
+    key = options[:key] || type
+    @import[key] ||= []
+    if valid? type, params
+      if buffer
+        # save in memory and flush to disk at the end
+        @import[key] << params
+      else
+        # write immediately instead of storing in memory
+        write *args
+      end
+      @stats[key] ||= 0
+      @stats[key] += 1
+      params
+    else
+      @stats["invalid:#{key}"] ||= 0
+      @stats["invalid:#{key}"] += 1
+      nil
+    end
+  end
+
+  def write type, params, options = {}
+    case type
+    when :discussion
+      # ughh
+      write_discussion options[:key].split(':',2)[1], params
+    when :category
+      write_category params
+    when :user
+      write_user params
+    end
+  end
+
+  def valid? type, params
+    problems = []
+    # XXX this is not really enough validation, also it's ugly as fuck
+    if type == :user && (params[:email].nil? || params[:email].empty?)
+      problems << "Missing email in user data: #{params.inspect}."
+    end
+    if type == :user && !%w[user support].include?(params[:state])
+      problems << "Invalid state in user data: #{params.inspect}."
+    end
+    if type == :category && (params[:name].nil? || params[:name].empty?)
+      problems << "Missing name in category data: #{params.inspect}."
+    end
+    if type == :discussion && (params[:author_email].nil? || params[:author_email].empty?)
+      problems << "Missing author_email in discussion data: #{params.inspect}."
+    end
+    if type == :discussion && (params[:comments].nil? || params[:comments].any? {|c| c[:author_email].nil? || c[:author_email].empty?})
+      problems << "Missing comments and authors in discussion data: #{params.inspect}."
+    end
+    if problems.empty?
+      true
+    else
+      @report += problems
+      false
+    end
+  end
+
+  def export_dir subdir=nil
+    subdir.nil? ? @export_dir : File.join(@export_dir, subdir.to_s)
+  end
+
+end

data/lib/tender_import/version.rb

@@ -0,0 +1,4 @@
+module TenderImport
+  VERSION = '0.0.1'
+end
+

data/lib/tender_import/zendesk_api_import.rb

@@ -0,0 +1,307 @@
+require 'yajl'
+require 'faraday'
+require 'trollop'
+require 'fileutils'
+require 'logger'
+
+# Produce a Tender import archive from a ZenDesk site using the ZenDesk API.
+class TenderImport::ZendeskApiImport
+  class Error < StandardError; end
+  class ResponseJSON < Faraday::Response::Middleware
+    def parse(body)
+      Yajl::Parser.parse(body)
+    end
+  end
+
+  module Log # {{{
+    attr_reader :logger
+    def log string
+      logger.info "#{to_s}: #{string}"
+    end
+
+    def debug string
+      logger.debug "#{to_s}: #{string}"
+    end
+  end # }}}
+
+  class Client # {{{
+    include Log
+    attr_reader :opts, :conn, :subdomain
+
+    # If no options are provided they will be obtained from the command-line.
+    #
+    # The options are subdomain, email and password.
+    #
+    # There is also an optional logger option, for use with Ruby/Rails
+    def initialize(options = nil)
+      @opts = options || command_line_options
+      @subdomain = opts[:subdomain]
+      @logger = opts[:logger] || Logger.new(STDOUT).tap {|l| l.level = Logger::INFO}
+      @conn = Faraday::Connection.new("http://#{subdomain}.zendesk.com") do |b|
+        b.adapter :net_http
+        b.use ResponseJSON
+      end
+      conn.basic_auth(opts[:email], opts[:password])
+    end
+
+    def to_s
+      "#{self.class.name} (#{subdomain})"
+    end
+
+    # API helpers # {{{
+    def user user_id
+      fetch_resource("users/#{user_id}.json")
+    end
+
+    def users
+      fetch_paginated_resources("users.json?page=%d")
+    end
+
+    def forums
+      fetch_resource("forums.json")
+    end
+
+    def entries forum_id
+      fetch_paginated_resources("forums/#{forum_id}/entries.json?page=%d")
+    end
+
+    def posts entry_id
+      fetch_paginated_resources("entries/#{entry_id}/posts.json?page=%d", 'posts')
+    end
+
+    def open_tickets
+      fetch_paginated_resources("search.json?query=type:ticket+status:open+status:pending+status:new&page=%d")
+    end
+    # }}}
+    
+    protected # {{{
+
+    # Fetch every page of a given resource. Must provide a sprintf format string
+    # with a single integer for the page specification.
+    #
+    # Example: "users.json?page=%d"
+    #
+    # In some cases the desired data is not in the top level of the payload. In
+    # that case specify resource_key to pull the data from that key.
+    def fetch_resource resource_url, resource_key = nil
+      debug "fetching #{resource_url}"
+      loop do
+        response = conn.get(resource_url)
+        if response.success?
+          return resource_key ? response.body[resource_key] : response.body
+        elsif response.status == 503
+          log "got a 503 (API throttle), waiting 30 seconds..."
+          sleep 30
+        else
+          raise Error, "failed to get resource #{resource_format}: #{response.inspect}"
+        end
+      end
+    end
+
+    # Fetch every page of a given resource. Must provide a sprintf format string
+    # with a single integer for the page specification.
+    #
+    # Example: "users.json?page=%d"
+    #
+    # In some cases the desired data is not in the top level of the payload. In
+    # that case specify resource_key to pull the data from that key.
+    def fetch_paginated_resources resource_format, resource_key = nil
+      resources = []
+      page = 1
+      loop do
+        resource = fetch_resource(resource_format % page, resource_key)
+        break if resource.empty?
+        page += 1
+        resources += resource
+      end
+      resources
+    end
+
+    def command_line_options
+      options = Trollop::options do
+        banner <<-EOM
+      Usage:
+        #{$0} -e <email> -p <password> -s <subdomain>
+
+      Prerequisites:
+        # Ruby gems
+        gem install faraday -v "~>0.4.5"
+        gem install trollop
+        gem install yajl-ruby
+        # Python tools (must be in your PATH)
+        html2text.py: http://www.aaronsw.com/2002/html2text/
+
+      Options:
+        EOM
+        opt :email,       "user email address", :type => String
+        opt :password,    "user password",      :type => String
+        opt :subdomain,   "subdomain",          :type => String
+      end
+
+      [:email, :password, :subdomain ].each do |option|
+        Trollop::die option, "is required" if options[option].nil?
+      end
+      return options
+    end
+    # }}}
+
+  end # }}}
+
+  class Exporter # {{{
+    attr_reader :logger, :client
+    include Log
+    include FileUtils
+
+    def initialize client
+      @client = client
+      @author_email = {}
+      @logger = client.logger
+      @archive = TenderImport::Archive.new(client.subdomain)
+      if `which html2text.py`.empty?
+        raise Error, 'missing prerequisite: html2text.py is not in your PATH'
+      end
+    end
+
+    def to_s
+      "#{self.class.name} (#{client.subdomain})"
+    end
+
+    def stats
+      @archive.stats
+    end
+
+    def report
+      @archive.report
+    end
+
+    def export_users # {{{
+      log 'exporting users'
+      client.users.each do |user|
+        @author_email[user['id'].to_s] = user['email']
+        log "exporting user #{user['email']}"
+        @archive.add_user \
+          :name => user['name'],
+          :email => user['email'],
+          :created_at => user['created_at'],
+          :updated_at => user['updated_at'],
+          :state => (user['roles'].to_i == 0 ? 'user' : 'support')
+      end
+    end # }}}
+
+    def export_categories # {{{
+      log 'exporting categories'
+      client.forums.each do |forum|
+        log "exporting category #{forum['name']}"
+        category = @archive.add_category \
+            :name => forum['name'],
+            :summary => forum['description']
+        export_discussions(forum['id'], category)
+      end
+    end # }}}
+
+    def export_tickets # {{{
+      log "exporting open tickets"
+      tickets = client.open_tickets
+      if tickets.size > 0
+        # create category for tickets
+        log "creating ticket category"
+        category = @archive.add_category \
+          :name => 'Tickets',
+          :summary => 'Imported from ZenDesk.'
+        # export tickets into new category
+        tickets.each do |ticket|
+          comments = ticket['comments'].map do |post|
+            {
+              :body => post['value'],
+              :author_email => author_email(post['author_id']),
+              :created_at => post['created_at'],
+              :updated_at => post['updated_at'],
+            }
+          end
+          log "exporting ticket #{ticket['nice_id']}"
+          @archive.add_discussion category, 
+            :title        => ticket['subject'],
+            :author_email => author_email(ticket['submitter_id']),
+            :created_at   => ticket['created_at'],
+            :updated_at   => ticket['updated_at'],
+            :comments     => comments
+        end
+      end
+    end # }}}
+
+    def export_discussions forum_id, category # {{{
+      client.entries(forum_id).each do |entry|
+        comments = client.posts(entry['id']).map do |post|
+          dump_body post, post['body']
+          {
+            :body => load_body(entry),
+            :author_email => author_email(post['user_id']),
+            :created_at => post['created_at'],
+            :updated_at => post['updated_at'],
+          }
+        end
+        dump_body entry, entry['body']
+        log "exporting discussion #{entry['title']}"
+        @archive.add_discussion category,
+          :title    => entry['title'],
+          :author_email => author_email(entry['submitter_id']),
+          :comments => [{
+            :body => load_body(entry),
+            :author_email => author_email(entry['submitter_id']),
+            :created_at => entry['created_at'],
+            :updated_at => entry['updated_at'],
+          }] + comments
+        rm "tmp/#{entry['id']}_body.html"
+      end
+    end # }}}
+  
+    def create_archive # {{{
+      export_file = @archive.write_archive
+      log "created #{export_file}"
+    end # }}}
+
+    protected
+
+    def author_email user_id
+      # the cache should be populated during export_users but we'll attempt
+      # to fetch unrecognized ids just in case
+      @author_email[user_id.to_s] ||= (client.user(user_id)['email'] rescue nil)
+    end
+
+    def dump_body entry, body
+      File.open(File.join("tmp", "#{entry['id']}_body.html"), "w") do |file|
+        file.write(body)
+      end
+    end
+
+    def load_body entry
+      `html2text.py /$PWD/tmp/#{entry['id']}_body.html`
+    end
+
+  end # }}}
+
+  # Produce a complete import archive either from API or command line options.
+  def self.run options=nil
+    begin
+      client = Client.new options
+      exporter = Exporter.new client
+      exporter.export_users
+      exporter.export_categories
+      exporter.export_tickets
+      exporter.create_archive
+    rescue Error => e
+      puts "FAILED WITH AN ERROR"
+      puts e.to_s
+      exit 1
+    ensure
+      if exporter
+        puts "RESULTS"
+        puts exporter.stats.inspect
+        puts exporter.report.join("\n")
+      end
+    end
+  end
+
+end
+
+# vi:foldmethod=marker

data/lib/tender_import.rb

@@ -0,0 +1,4 @@
+module TenderImport
+  autoload :Archive, 'tender_import/archive'
+  autoload :ZendeskApiImport, 'tender_import/zendesk_api_import'
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification 
+name: tender_import
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Zack Hobson
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-03 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: faraday
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: trollop
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: yajl-ruby
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: |
+  These are tools written in Ruby to support importing data into Tender.
+  
+  For more information:
+  
+  https://help.tenderapp.com/kb/setup-installation/importing
+
+email: zack@zackhobson.com
+executables: 
+- zendesk2tender
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- lib/tender_import/archive.rb
+- lib/tender_import/version.rb
+- lib/tender_import/zendesk_api_import.rb
+- lib/tender_import.rb
+- bin/zendesk2tender
+has_rdoc: true
+homepage: https://help.tenderapp.com/kb/setup-installation/importing
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.1
+signing_key: 
+specification_version: 3
+summary: Tools for producing Tender import archives.
+test_files: []
+