zendeskdumper 0.0.0 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 793dbc9da00104153184bcee327a99e11fd0c4c4
-  data.tar.gz: a7f86221af7f332c3d68d832e32c6cc721525c90
+  metadata.gz: 337035df0e670d759100d5031ebd1b65c1585cc5
+  data.tar.gz: 59e5ff9c8a89cafdc04ba926ac37381135ddab10
 SHA512:
-  metadata.gz: 7e25519285a8e4c56e2d4e9708b5f45db8f6f07cba0b795507e024c8a005cd60c5be7f7068ead15935c6645b50d011f9273ea88109254f525c5ba605a2c9ce63
-  data.tar.gz: 20028948fabf596fc9f121123ba7ccdcfc7243fa47bb5fa48d66affa944ba1cb17bf74e150fcc24edef2c2af54c4878cd5f0ca141b93aaeabe001900b39f5290
+  metadata.gz: 197f764b04756f3ea00f909132c4f3579415bb132da29904d2ce2d505741fa558d08947c1764d35416d0d0c75a1df2d91816019fbc9eea7cf6fa439a39a6786f
+  data.tar.gz: ce7cf9fac323826ba1fb85fa09988055c16654ba80203b7d4a5b0a36c92884631bdc25c8d8f3c08d0456ccae541a277c9c8fe11b1959d9f12b142ab7a7805d94

data/bin/zendeskdumper

@@ -0,0 +1,83 @@
+#!/usr/bin/env ruby
+# Copyright © 2017 Taylor C. Richberger <taywee@gmx.com>
+# This code is released under the license described in the LICENSE file
+
+require 'logger'
+require 'optparse'
+require 'pathname'
+require 'set'
+require 'yaml'
+
+require 'zendeskdumper'
+require 'zendeskdumper/meta'
+
+
+def main!
+  options = {directory: Pathname.new('.')}
+
+  opts = OptionParser.new
+  def opts.abort(output, code: 1)
+      STDERR.puts "ERROR: #{output}"
+      STDERR.puts self
+      exit code
+  end
+
+  opts.banner = 'Usage: zendeskdumper [options] DOMAIN'
+  opts.version = ZenDeskDumper::Meta::VERSION
+
+  opts.on_tail('-h', '--help', 'Show this help message') do
+    puts opts
+    exit
+  end
+  opts.on_tail('-V', '--version', 'Show the version') do
+    puts opts.version
+    exit
+  end
+
+  opts.on('-u', '--user USERNAME:PASSWORD', 'Specify username and password') do |user|
+    unless user.include? ':'
+      opts.abort '-u, --user needs at least one colon'
+    end
+
+    options.update([:username, :password].zip(user.split(':', 2)).to_h)
+  end
+  opts.on('-c', '--credentials FILE', 'Credentials YAML file with username and password fields') do |filename|
+    set = Set[:username, :password]
+    # Should only update username and password
+    options.update(YAML.load_file(filename).select {|key| set.include? key})
+  end
+  opts.on('-d', '--directory DIR', 'Directory to dump the files (default: .)') do |dir|
+    options[:directory] = Pathname.new(dir)
+  end
+  opts.parse!
+
+  if ARGV.empty?
+    opts.abort 'ERROR: DOMAIN is mandatory'
+  end
+
+  unless options.key? :username and options.key? :password 
+    opts.abort 'Credentials must be specified with either -u or -c, and must specify username and password'
+  end
+
+  domain = ARGV.pop
+
+  logger = Logger.new(STDERR)
+
+  ZenDeskDumper::Dumper.new(
+    username: options[:username],
+    password: options[:password],
+    domain: domain,
+  ).dump do |filename, content|
+    path = options[:directory] + filename
+    dir = path.dirname
+    unless dir.directory?
+      logger.info "creating #{dir}"
+      dir.mkpath
+    end
+    logger.info "dumping #{path}"
+    File::write(path, content)
+  end
+end
+
+
+main! if __FILE__ == $0

data/lib/zendeskdumper.rb

@@ -0,0 +1,212 @@
+require 'json'
+require 'net/http'
+require 'pathname'
+
+module ZenDeskDumper
+  # A special dumper class.  The "dump" method will use a passed-in block to
+  # generate, that way it can actually be used to directly write into an archive
+  # or something of the sort if desired.  All path names are kept relative
+  class Dumper
+    attr_accessor :domain, :username, :password
+
+    def initialize(domain:, username:, password:)
+      @logger = Logger.new(STDERR)
+
+      @domain = domain
+      @username = username
+      @password = password
+
+      @threadlimit = 50
+    end
+
+    # Main method to call for this type; runs the full dumper.  Uses a passed-in
+    # block to generate all files.
+    def dump(&block)
+      @logger.info 'pulling users'
+      dump_users(&block)
+      @logger.info 'pulling organizations'
+      dump_organizations(&block)
+      @logger.info 'pulling tickets'
+      dump_tickets(&block)
+    end
+
+    # Run a list of items, yield to the block, and join on the thread limit (and
+    # before finishing
+    def checkthreads(list)
+      threads = []
+      list.each do |item|
+        threads << Thread.new do
+          yield item
+        end
+        if threads.size >= @threadlimit
+          threads.each(&:join)
+          threads = []
+        end
+      end
+      threads.each(&:join)
+    end
+
+    # Get method which runs requests and optionally sleeps if necessary based on
+    # ratelimiting
+    def get(uri)
+      request = Net::HTTP::Get.new uri
+      request.basic_auth(@username, @password)
+      response = Net::HTTP::start(uri.host, uri.port, use_ssl: true) do |http|
+        http.request request
+      end
+      if Integer(response.code) == 429
+        # Make sure to allow some decent grace period
+        time = Integer(response['Retry-After']) + 60
+        @logger.debug "Hitting rate limiting, sleeping for #{time} seconds"
+        sleep time
+        return get uri
+      end
+      # Raise errors when necessary
+      response.value
+      response
+    end
+
+    # Get an attachment file, following redirects, and return its body
+    def get_attachment(uri, limit=10)
+      raise ArgumentError, 'too many redirects' if limit == 0
+
+      request = Net::HTTP::Get.new uri
+      @logger.debug "getting attachment from uri #{uri}"
+      request.basic_auth(@username, @password)
+      response = Net::HTTP::start(uri.host, uri.port, use_ssl: true) do |http|
+        http.request request
+      end
+      if response.is_a? Net::HTTPRedirection
+        @logger.debug "redirecting to #{response['location']}"
+        return get_attachment URI(response['location']), limit - 1
+      end
+      response.value
+      response.body
+    end
+
+    # get all pages from a paged api and yield their parsed JSON body into a block
+    def getpages(uri)
+      pagenumber = 1
+      loop do
+        response = get uri
+        body = JSON.parse(response.body)
+        yield body, pagenumber
+        next_page = body['next_page']
+        break if next_page.nil?
+        # If the next_page was the same as this one, it will loop forever
+        next_uri = URI(next_page)
+        break if uri == next_uri
+
+        uri = next_uri
+        pagenumber += 1
+      end
+    end
+
+    def dumppages(uri, basepath, formatname)
+      getpages uri do |page, pagenum|
+        path = basepath + formatname % pagenum
+        yield path, JSON.fast_generate(page)
+      end
+    end
+
+    def dump_users(&block)
+      uri = URI('https:/api/v2/users.json')
+      uri.hostname = @domain
+      getpages uri do |page|
+        checkthreads(page['users']) do |user|
+          begin
+            dump_user(user['id'], &block)
+          rescue Net::HTTPServerException, Net::HTTPFatalError
+            @logger.error "could not find user #{user}"
+          end
+        end
+      end
+    end
+
+    def dump_user(id, &block)
+      # Get user meta details
+      uri = URI("https:/api/v2/users/#{id}.json")
+      uri.hostname = @domain
+      response = get uri
+      basepath = Pathname.new('users') + id.to_s
+      yield basepath + 'user.json', response.body
+
+      # Get user groups
+      uri = URI("https:/api/v2/users/#{id}/groups.json")
+      uri.hostname = @domain
+      dumppages(uri, basepath, 'groups-%03d.json', &block)
+    end
+
+    def dump_organizations(&block)
+      uri = URI('https:/api/v2/organizations.json')
+      uri.hostname = @domain
+      getpages uri do |page|
+        checkthreads(page['organizations']) do |organization|
+          begin
+            dump_organization(organization['id'], &block)
+          rescue Net::HTTPServerException, Net::HTTPFatalError
+            @logger.error "could not find organization #{organization}"
+          end
+        end
+      end
+    end
+
+    def dump_organization(id, &block)
+      # get organization meta details
+      uri = URI("https:/api/v2/organizations/#{id}.json")
+      uri.hostname = @domain
+      response = get uri
+      basepath = Pathname.new('organizations') + id.to_s
+      yield basepath + 'organization.json', response.body
+
+      # Get users
+      uri = URI("https:/api/v2/organizations/#{id}/users.json")
+      uri.hostname = @domain
+      dumppages(uri, basepath, 'users-%03d.json', &block)
+    end
+
+    def dump_tickets(&block)
+      uri = URI('https:/api/v2/incremental/tickets.json?start_time=0')
+      uri.hostname = @domain
+      getpages uri do |page|
+        checkthreads(page['tickets']) do |ticket|
+          begin
+            dump_ticket(ticket['id'], &block)
+          rescue Net::HTTPServerException, Net::HTTPFatalError
+            @logger.error "could not find ticket #{ticket}"
+          end
+        end
+      end
+    end
+
+    def dump_ticket(id)
+      # Yield base ticket details
+      uri = URI("https:/api/v2/tickets/#{id}.json")
+      uri.hostname = @domain
+      response = get uri
+      basepath = Pathname.new('tickets') + id.to_s
+      yield basepath + 'ticket.json', response.body
+
+      # Yield out all comments
+      uri = URI("https:/api/v2/tickets/#{id}/comments.json")
+      uri.hostname = @domain
+      getpages uri do |page, pagenum|
+        path = basepath + 'comments-%03d.json' % pagenum
+        yield path, JSON.fast_generate(page)
+        page['comments'].each do |comment|
+          comment['attachments'].each do |attachment|
+            basepath = Pathname.new('attachments') + attachment['id'].to_s
+            @logger.debug "getting attachment json file"
+            yield basepath + 'attachment.json', JSON.fast_generate(attachment)
+            begin
+              @logger.debug "getting attachment file #{basepath + 'files' + attachment['file_name']}"
+              yield basepath + 'files' + attachment['file_name'], get_attachment(URI(attachment['content_url']))
+            rescue Net::HTTPServerException, Net::HTTPFatalError
+              @logger.error "could not find attachment file #{attachment}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/zendeskdumper/meta.rb

@@ -0,0 +1,20 @@
+# This is simply here in case we want any of this data available from within the
+# running library, especially for generating the useable command line
+module ZenDeskDumper
+  module Meta
+    NAME = 'zendeskdumper'
+    VERSION = '0.1.0'
+    DATE = '2018-01-02'
+    SUMMARY = 'A simple dumper for a ZenDesk domain.'
+    DESCRIPTION = 'A simple dumper for a ZenDesk domain.  Attempts to pull all users, tickets, comments, and attachments'
+    AUTHORS = ['Taylor C. Richberger']
+    EMAIL = 'tcr@absolute-performance.com'
+    FILES = [
+      'bin/zendeskdumper',
+      'lib/zendeskdumper.rb',
+      'lib/zendeskdumper/meta.rb',
+    ]
+    HOMEPAGE = 'https://rubygems.org/gem/zendeskdumper'
+    LICENSE = 'MIT'
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: zendeskdumper
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.0
 platform: ruby
 authors:
 - Taylor C. Richberger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-28 00:00:00.000000000 Z
+date: 2018-01-02 00:00:00.000000000 Z
 dependencies: []
 description: A simple dumper for a ZenDesk domain.  Attempts to pull all users, tickets,
   comments, and attachments
@@ -17,7 +17,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- bin/zendeskdumper
 - lib/zendeskdumper.rb
+- lib/zendeskdumper/meta.rb
 homepage: https://rubygems.org/gem/zendeskdumper
 licenses:
 - MIT