sp-seutils 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: fbad290d1c8944202370f5dc03228c9952a3c664bd61b8f061e6e0121cc75ea6
+  data.tar.gz: a81cb77c92d5bb2009e5c5d40053a61b5aed034b72eaa2d4ae8e58def4f64be6
+SHA512:
+  metadata.gz: f79bcd35c7dc7ffe8bbc3e5a95094d6407aa8530c273a4dca360ed9bddbe714510495168dd569019dc5ab29860e5c3f69623249e35c3bb317de4c9197c455e71
+  data.tar.gz: 4b53eca404537d0ffb8c34bd294b6b1fada511f5f45869703b4026bc0da03e24afb0021e824531f1f63cdc81a0e6795c337c6d5102e68da3b2782fbcdf911451

data/lib/sp-seutils.rb

@@ -0,0 +1,2347 @@
+#!/usr/bin/env ruby
+
+# idn-utils.rb
+#
+# Utilities for IdentityNow management
+
+class SeUtils
+
+  require 'net/http'
+  require 'net/https'
+  require 'uri'
+  require 'rest-client'
+  # uncomment the following if you want detailed logging of REST calls
+  # RestClient.log = 'stdout'
+  RestClient.log = '/dev/null'
+  require 'digest'
+  require 'json'
+  require 'base64'
+
+  def self.cc_url(url)
+    base_url = url.split(".identitynow.com").first+".api.identitynow.com"
+    api = "/cc/api"+url.split("/api").last
+    api = url.split("/api").last if api.split("/").include? "beta"
+    api = url.split("/api").last if api.split("/").include? "v2"
+    api = url.split("/api").last if api.split("/").include? "v3"
+    return base_url+api
+  end
+
+  #-----------------------------------------------------------------------------#
+  # API GET UTILS
+  # Methods to assist in running API GET calls
+  #-----------------------------------------------------------------------------#
+
+  #-----------------------------------------------------------------------------#
+  # run an API GET method, pass in the api
+  # org needs to be a full url like 'https://myorg.identitynow.com'
+  # api_user and api_key are not the same as user/password
+  # returns JSON
+  #
+  def self.api_get(org,api,user,password,client_id,client_secret,token=nil)
+    api = "api/"+api
+    url = cc_url(URI.join(org, api).to_s)
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    do_retry = true
+    loop do
+      break unless do_retry
+      begin
+        response = RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' } ) do |r_response, request, result, &block|
+          rest_result = handle_rest_response(r_response, request, result, &block)
+          do_retry = false
+        end
+        do_retry = response.nil?
+        puts "Received nil response, rertrying..." if response.nil?
+      rescue RestClient::Exceptions::OpenTimeout => e
+        e.response
+        puts "Timed out. Retrying..."
+        do_retry = true
+      rescue RestClient::Exceptions::ReadTimeout => e
+        e.response
+        puts "Timed out. Retrying..."
+        do_retry = true
+      rescue SocketError => e
+        puts "SocketError. Retrying..."
+        do_retry = true
+      rescue RestClient::InternalServerError => e
+        puts "RestClient::InternalServerError Retrying..."
+        do_retry = true
+      rescue RestClient::GatewayTimeout => e
+        puts "RestClient::GatewayTimeout Retrying..."
+        do_retry = true
+      end
+      sleep 5 if do_retry # going to retry
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # run an API GET method, pass in the api
+  # org needs to be a full url like 'https://myorg.identitynow.com'
+  # returns JSON
+  #
+  def self.api_get_new(org,api,client_id,client_secret,token=nil)
+    api = "api/"+api
+    url = cc_url(URI.join(org, api).to_s)
+    token = token || idn_oauth_2(org,client_id,client_secret)
+    rest_result = nil
+    do_retry = true
+    loop do
+      break unless do_retry
+      begin
+        response = RestClient.get(url, {
+          'Authorization'=>"Bearer #{token}", 
+          "Content-Type"=>"application/json",
+          "Accept"=>'*/*',
+          "Host"=>URI.parse(org).host,
+          "Accept-Encoding" =>'gzip, deflate',
+          "Connection" => 'keep-alive',
+          "cache-control" => 'no-cache'
+        } ) do |r_response, request, result, &block|
+          rest_result = handle_rest_response(r_response, request, result, &block)
+          do_retry = false
+        end
+        do_retry = response.nil?
+        puts "Received nil response, rertrying..." if response.nil?
+      rescue RestClient::Exceptions::OpenTimeout => e
+        e.response
+        puts "Timed out. Retrying..."
+        do_retry = true
+      rescue RestClient::Exceptions::ReadTimeout => e
+        e.response
+        puts "Timed out. Retrying..."
+        do_retry = true
+      rescue SocketError => e
+        puts "SocketError. Retrying..."
+        do_retry = true
+      rescue RestClient::InternalServerError => e
+        puts "RestClient::InternalServerError Retrying..."
+        do_retry = true
+      rescue RestClient::GatewayTimeout => e
+        puts "RestClient::GatewayTimeout Retrying..."
+        do_retry = true
+      end
+      sleep 5 if do_retry # going to retry
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Get a list of apps
+  # returns JSON
+  # filter can select which apps to return
+  # filter=available is all apps that the logged-in user can add
+  # filter=org is the org-wide list
+  # filter=added is the ones on the launchpad for the logged-in user
+  #
+  def self.api_app_list(org,user,password,client_id,client_secret,filter='org',token=nil)
+    api = "api/app/list"
+    url = cc_url(URI.join(org, api).to_s+"?filter=#{filter}")
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get a list of requestable apps
+  # returns JSON
+  #
+  def self.api_requestable_apps(org,user,password,client_id,client_secret,token=nil)
+    api = "api/v2/identity/apps"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get an identity object
+  # returns JSON
+  #
+  def self.api_identity(org,user,password,client_id,client_secret,id,token=nil)
+    api = "api/v2/identity/#{id}"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get a list of identities
+  # returns JSON
+  # include alias (string) or id (integer) as arg to return only the identity for that one user
+  #
+  def self.api_identities(org,user,password,client_id,client_secret,arg=nil,token=nil)
+    api = "api/v2/identities/"
+    # have to paginate in chunks of 250
+    rest_result = []
+    i = 0
+    loop do
+      url = URI.join(org, api).to_s+arg.to_s+"?limit=250&offset="+i.to_s
+      # puts "executing API GET #{url}"
+      token = token || idn_oauth(org,user,password,client_id,client_secret)
+      page = nil
+      RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+        page = handle_rest_response(response, request, result, &block)
+      end
+      i += JSON.parse(page).length
+      rest_result += JSON.parse(page)
+      break if JSON.parse(page).length != 250
+    end
+    return JSON.generate(rest_result)
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get a list of access profiles
+  # returns JSON
+  #
+  def self.api_accessprofile_list(org,user,password,client_id,client_secret,token=nil)
+    api = URI.escape("api/v2/access-profiles")
+    rest_result = []
+    i = 0
+    loop do
+      url = URI.join(org, api).to_s+"?limit=250&offset="+i.to_s
+      token = token || idn_oauth(org,user,password,client_id,client_secret)
+      page = nil
+      RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+        page = handle_rest_response(response, request, result, &block)
+      end
+      i += JSON.parse(page).length
+      rest_result += JSON.parse(page)
+      break if JSON.parse(page).length != 250
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get an access profile by id
+  # returns JSON
+  #
+  def self.api_accessprofile_get(org,user,password,client_id,client_secret,id,token=nil)
+    api = URI.escape("api/v2/access-profiles/#{id}")
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get governance groups
+  # returns JSON
+  #
+  def self.api_workgroups_get(org,user,password,client_id,client_secret,id=nil,token=nil)
+    api = URI.escape("api/v2/workgroups/#{id}") # hopefully no more than 250 governance groups
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get an app's config
+  # Returns the JSON config data from an app so you can import it elsewhere
+  # Exactly the same output as app/update POST
+  #
+  def self.api_app_get(org,user,password,client_id,client_secret,id,token=nil)
+    api = "api/app/get/"+id.to_s
+    url = URI.join(org, api).to_s
+    return if id.nil? or id == ""
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # List all identities
+  # returns JSON object wth a total, and then an array of "items"
+  #
+  def self.api_identity_list(org,user,password,client_id,client_secret,token=nil)
+    api = "api/user/list"
+    url = cc_url(URI.join(org, api).to_s)
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get an identity
+  # Returns the JSON object for an identity cube
+  #
+  def self.api_user_details(org,user,password,client_id,client_secret,id,token=nil)
+    api = "api/user/details/"+id.to_s
+    url = cc_url(URI.join(org, api).to_s)
+    return if id.nil? or id == ""
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get uncorrelated accounts from a source
+  # 
+  #
+  def self.api_source_getuncorrelatedaccounts(org,user,password,client_id,client_secret,source_id,token=nil)
+    api = "api/source/getUncorrelatedAccounts?id="+source_id
+    url = cc_url(URI.join(org, api).to_s)
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get a list of all connectors
+  # 
+  def self.api_connector_list(org,user,pass,client_id,client_secret,token)
+    api = "api/connector/list"
+    url = cc_url(URI.join(org, api).to_s)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}",'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # List sources (beta)
+  # 
+  def self.api_source_get(org,client_id,client_secret,user,pass,source_ext_id,token=nil)
+    # if source_ext_id is nil, it will return a list
+    api = "/beta/sources/"
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    url = URI.join(base_url, api).to_s
+    url = URI.join(base_url, api).to_s+"/"+source_ext_id.to_s unless source_ext_id.nil?
+    token = token || idn_oauth_3(org,user,pass,client_id,client_secret) 
+    rest_result = nil
+    redirected_url = nil
+    response = RestClient::Request.new({
+      method: :get,
+      url: url,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Content-Type' => 'application/json',
+                    'cache-control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create a source (beta)
+  # 
+  def self.api_source_post(org,client_id,client_secret,user,pass,args,token=nil)
+    return nil if args.nil?
+    ["id", "created", "modified","accountCorrelationConfig","schemas","passwordPolicies",
+    "backUpAttributes", "managerCorrelationRule"].each{|i| args.delete i}
+    ["cloudExternalId", "healthCheckTimeout", "cloudCacheUpdate","acctAggregationEnd",
+    "acctAggregationStart", "hasFullAggregationCompleted", "source-config", "user[]",
+    "cloudDisplayName", "md5"].each{|i| args["connectorAttributes"].delete i}
+    body = JSON.generate(args)
+    api = "/beta/sources/"
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    url = URI.join(base_url, api).to_s
+    token = token || idn_oauth_3(org,user,pass,client_id,client_secret) 
+    rest_result = nil
+    redirected_url = nil
+    response = RestClient::Request.new({
+      method: :post,
+      url: url,
+      payload: body,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Content-Type' => 'application/json',
+                    'cache-control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update a source (beta)
+  # 
+  def self.api_source_put(org,client_id,client_secret,user,pass,source_ext_id,args,token=nil)
+    return "{}" if args.nil?
+    return "{}" unless args.is_a? Hash
+    # have to actually go get the object bundle first, because if we pass in incomplete list, 
+    # it tends to delete stuff...
+    source_json = JSON.parse(api_source_get(org,client_id,client_secret,user,pass,source_ext_id,token))
+    source_json.each do |k,v|
+      if v.is_a? Hash
+        v.each do |l,w|
+          source_json[k][l] = "true" if w.is_a? TrueClass
+          source_json[k][l] = "false" if w.is_a? FalseClass
+        end
+      end
+      source_json[k] = "true" if v.is_a? TrueClass
+      source_json[k] = "false" if v.is_a? FalseClass
+    end
+    source_json["connectorAttributes"]["cloudExternalId"] = source_json["connectorAttributes"]["cloudExternalId"].to_s
+
+    # shouldn't be updating these things... we'll keep the ones from the GET
+    ["type","connector","connectorClass","created", "modified"].each{|i| args.delete i}
+    ["cloudExternalId", "healthCheckTimeout", "cloudCacheUpdate","acctAggregationEnd",
+    "acctAggregationStart", "hasFullAggregationCompleted", "user[]",
+    "cloudDisplayName", "md5"].each{|i| args["connectorAttributes"].delete i} if args.keys.include? "connectorAttributes"
+
+    args.each do |k,v|
+      if v.is_a? Hash
+        args[k].each{|l,w| source_json[k][l] = w}
+      else
+        source_json[k] = v
+      end
+    end
+
+    body = JSON.generate(source_json)
+    api = "/beta/sources/"
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    url = URI.join(base_url, api, source_ext_id).to_s
+    token = token || idn_oauth_3(org,user,pass,client_id,client_secret) 
+    rest_result = nil
+    redirected_url = nil
+    response = RestClient::Request.new({
+      method: :put,
+      url: url,
+      payload: body,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Content-Type' => 'application/json',
+                    'cache-control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Patch a source (beta)
+  # args is a an array of hashes, must be in specific format:
+  # [{:op=>"replace", :path=>"/connectorAttributes/password",:value=>"MyPassword!"}]
+  # 
+  def self.api_source_patch(org,client_id,client_secret,user,pass,source_ext_id,args,token=nil)
+    return nil if args.nil?
+    args.each do |arg|
+      arg.keys.each{|k|arg.delete k unless [:op,:path,:value].include? k.to_sym}
+      arg[:op] = "replace"
+    end
+    body = JSON.generate(args)
+    api = "/beta/sources/"
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    url = URI.join(base_url, api, source_ext_id).to_s
+    token = token || idn_oauth_3(org,client_id,client_secret,user,pass) 
+    rest_result = nil
+    redirected_url = nil
+    response = RestClient::Request.new({
+      method: :patch,
+      url: url,
+      payload: body,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Content-Type' => 'application/json-patch+json',
+                    'cache-control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Get accounts from a source
+  # 
+  def self.api_source_getaccounts(org,user,password,client_id,client_secret,source_id,token=nil)
+    api = "api/source/getAccounts"
+
+    body = JSON.generate({:id=>source_id})
+    url = cc_url(URI.join(org, api).to_s)
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+
+    rest_result = nil
+    redirected_url = nil
+    response = RestClient::Request.new({
+      method: :get,
+      url: url,
+      payload: body,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Cache-Control' => 'no-cache',
+                    'Content-Type' => 'application/json',
+                    'cache-control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get accounts CSV file
+  # 
+  #
+  def self.api_source_getaccountscsv(org,user,password,client_id,client_secret,source_id,token=nil)
+    puts "api/source/getAccountsCsv is deprecated. Using runAccountsExportReport"
+    csv = api_source_export_accounts(org,user,password,client_id,client_secret,source_id,token=nil)
+    return csv
+  end
+
+  def self.identity_map(org,user,pass,client_id,client_secret,token)
+    token = token || idn_oauth_2(org,client_id,client_secret)
+    users = {}
+    i = 0
+    nil_users = 0
+    loop do
+      page = JSON.parse(api_get(org, "user/list?start=#{i}&limit=250",user,pass,client_id,client_secret,token))["items"]
+      i += page.length
+      page.each do |u|
+        users[u["name"].split(" ").join(".")] = u["externalId"] unless u.nil?
+        @idn_users << u unless u.nil?
+        nil_users+=1 if u.nil?
+      end
+      break if page.length != 250
+    end
+    puts "There were #{nil_users} nil users!" unless nil_users == 0
+    return users
+  end
+
+  def self.source_map(org,user,pass,client_id,client_secret,token)
+    token = token || idn_oauth_3(org,user,pass,client_id,client_secret)
+    sources = {}
+    result = JSON.parse(api_get_new(org,"source/list",client_id,client_secret,token))
+    result.each{|s| sources[s["name"]] = s["externalId"]}
+
+    t_entitlements = []
+    entitlements = {}
+    sources.each do |source_name,ext_id|
+      entitlement_bundle = []
+      api = "api/entitlement/list"
+      t_entitlements << Thread.new do
+        page_size = 250
+        offset = 0
+        rest_result = nil
+        loop do
+          args = "?CISApplicationId=#{ext_id}&offset=#{offset}&limit=#{page_size}"
+          url = URI.join(org, api+args).to_s # might have to paginate using args above
+          
+          redirected_url = nil
+          response = RestClient::Request.new({
+            method: :get,
+            url: url,
+            timeout: nil,
+            headers: {'Authorization' => "Bearer #{token}", 
+                          "Accept"=>"*/*" ,
+                          'Cache-Control' => 'no-cache',
+                          'Content-Type' => 'application/json',
+                          'cache-control' => 'no-cache'
+                          }
+             }).execute do |response, request, result, &block|
+            rest_result = handle_rest_response(response, request, result, &block)
+          end
+          puts "result = "+JSON.parse(rest_result).inspect if @debug
+          entitlement_bundle += JSON.parse(rest_result)["items"]
+          break if JSON.parse(rest_result).length < page_size
+          offset += page_size
+        end
+        t_ent_items = []
+        entitlement_bundle.each do |ent|
+          t_ent_items << Thread.new{ entitlements[ent["displayableName"]] = ent["id"]}
+        end
+        t_ent_items.each(&:join)
+      end
+    end
+    t_entitlements.each(&:join)
+    return {"applications" => sources, "entitlements" => entitlements}
+  end
+
+
+  def self.entitlement_map(org,user,pass,client_id,client_secret,token)
+    token = token || idn_oauth(org,user,pass,client_id,client_secret)
+    entitlements = {}
+    sources = {}
+    result = JSON.parse(api_get(org, "source/list",user,pass,client_id,client_secret,token))
+    result.each{|s| sources[s["name"]] = s["externalId"]}
+    result.each{|s| entitlements[s["name"]] = {"entitlements"=>{}}}
+
+    t_entitlements = []
+    sources.each do |source_name,ext_id|
+      t_entitlements << Thread.new do
+        result = api_get(org, "entitlement/list/?CISApplicationId=#{ext_id}",user,pass,client_id,client_secret,token)
+        entitlement_bundle = JSON.parse(result)["items"]
+        t_ent_items = []
+        entitlement_bundle.each do |ent|
+          t_ent_items << Thread.new{ entitlements[source_name]["entitlements"][ent["displayableName"]] = ent["id"]}
+        end
+        t_ent_items.each(&:join)
+      end
+    end
+    t_entitlements.each(&:join)
+    return entitlements
+  end
+
+  #-----------------------------------------------------------------------------#
+  # API POST UTILS
+  # Methods to assist in running API POST calls
+  #-----------------------------------------------------------------------------#
+
+  #-----------------------------------------------------------------------------#
+  # Run the specified api as POST
+  # doesn't work in most cases because IdN's POST methods almost always require
+  # specialized or unique data configuration
+  #
+  def self.api_post(org,api,user,password,client_id,client_secret,data=nil,token=nil)
+    api = "api/"+api
+    url = cc_url(URI.join(org, api).to_s)
+    rest_result = nil
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    RestClient.post(url,  data, 
+                          {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck', "Accept"=>"*/*" 
+                          }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  def self.get_recommendations(org,client_id,client_secret,token,data)
+    iai_base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    api = "beta/recommendations/request"
+
+    body = JSON.generate(data)
+    url = URI.join(iai_base_url, api).to_s
+    token = token || idn_oauth_2(org,client_id,client_secret)
+
+    rest_result = nil
+    redirected_url = nil
+    response = RestClient::Request.new({
+      method: :post,
+      url: url,
+      payload: body,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Cache-Control' => 'no-cache',
+                    'Content-Type' => 'application/json',
+                    'cache-control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Search for identities
+  #
+  def self.api_search_identities(org,client_id,client_secret,user,pass,query,token=nil)
+
+    body = nil
+    if query.is_a? Array # we passed in a list of identities
+      n = []
+      query.each{|display_name| n << "attributes.displayName:"+display_name}
+      body = JSON.generate({"query"=> {"query"=>n.join(" OR "), "includeNested"=>false}})
+    elsif query.is_a? Hash
+      # assume the query is correctly formatted, dunno why we have a double query key, so add it
+      query = JSON.parse(query) unless query.is_a? Hash
+      body = JSON.generate({"query"=>query}) if query.keys.include? "query"
+      body = JSON.generate({"query"=>{"query"=>query}}) unless query.keys.include? "query"
+    end
+
+
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    api = "beta/search/identities"
+    
+    token = token || idn_oauth_3(org,user,pass,client_id,client_secret)
+
+    result = []
+    page_size = 50
+    offset = 0
+
+    loop do
+      rest_result = []
+      args = "?offset=#{offset}&limit=#{page_size}"
+      url = URI.join(base_url, api+args).to_s # might have to paginate using args above
+      
+      redirected_url = nil
+      response = RestClient::Request.new({
+        method: :post,
+        url: url,
+        payload: body,
+        timeout: nil,
+        headers: {'Authorization' => "Bearer #{token}", 
+                      "Accept"=>"*/*" ,
+                      'Cache-Control' => 'no-cache',
+                      'Content-Type' => 'application/json',
+                      'cache-control' => 'no-cache'
+                      }
+         }).execute do |response, request, result, &block|
+        rest_result = handle_rest_response(response, request, result, &block)
+      end
+      puts "result = "+JSON.parse(rest_result).inspect if @debug
+      result += JSON.parse(rest_result)
+      break if JSON.parse(rest_result).length < page_size
+      offset += page_size
+    end
+    return JSON.generate(result)
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get details of a single identity
+  #
+  def self.api_search_identities_id(org,client_id,client_secret,user,pass,id,token=nil)
+
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    api = "beta/search"
+    
+    token = token || idn_oauth_3(org,client_id,client_secret,user,pass)
+
+    url = URI.join(base_url, api).to_s + "/identities/" + id.to_s
+    rest_result = nil
+    redirected_url = nil
+    response = RestClient::Request.new({
+      method: :get,
+      url: url,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Cache-Control' => 'no-cache',
+                    'Content-Type' => 'application/json',
+                    'cache-control' => 'no-cache',
+                    "Host"=>URI.parse(org).host,
+                    "Accept-Encoding" =>'gzip, deflate'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Create a VA cluster
+  #
+  def self.api_cluster_create(org,user,password,client_id,client_secret,cluster_name,token=nil)
+
+    url = URI.join(org, "api/cluster/create").to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, {:name => cluster_name},
+              {'Authorization' => "Bearer #{token}",'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create a direct-connect source on the target system
+  #
+  def self.api_source_create_direct(org,user,password,client_id,client_secret,args,token=nil)
+    api = "api/source/create"
+    args.each{|k,v|args[k.to_sym] = v}
+    url = cc_url(URI.join(org, api).to_s)
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, {:name => args[:name], 
+              :description => args[:description], 
+              :applicationTemplate => args[:type], 
+              :serviceDefinitionName => args[:type], 
+              :sourceType => args[:source_type],
+              :serviceType => 'app' },
+              {'Authorization' => "Bearer #{token}",'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Clone an entitlement
+  # You have to do this whenever you create an App Access Profile
+  # args are a hash of {:ids => <array of entitlement IDs>, :CISApplicationId => <Source External ID>
+  # 
+  def self.api_entitlement_clone(org,user,password,client_id,client_secret,args,token=nil)
+    # this call uses "ids" NOT as a comma-separated list, but as a list of "ids=<id>"
+    ids = []
+    args[:ids].each{|id|ids << "&ids="+id}
+    args.delete :ids
+    data = hash_to_args(args)+ids.join
+    api = "api/entitlement/clone"
+    url = URI.join(org, api).to_s
+    rest_result = nil
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    args = 
+    RestClient.post(url,  data, 
+                          {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck', "Accept"=>"*/*" 
+                          }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    sleep 2
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create an access profile on the source
+  # the "data" is a hash that has required elements: 
+  # :name :sourceId :entitlements :description
+  # :entitlements is an array of the entitlement IDs
+  #
+  def self.api_accessprofile_create(org,user,password,client_id,client_secret,data,token=nil)
+    api = "api/v2/access-profiles/"
+    if data[:entitlements].nil? or data[:entitlements].length == 0
+      puts "Can't create Access Profile '#{data[:name]}' without entitlements"
+      return
+    end
+    org = org.split('.').first
+    data[:entitlements] = Array(data[:entitlements].reject(&:empty?))
+    data[:description] = data[:name] if data[:description].nil?
+    id = data[:id]
+    data.delete(:id)
+    payload = JSON.generate(data)
+    url = URI.join("#{org}.identitynow.com", api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    do_retry = true
+    loop do
+      break unless do_retry
+      begin
+        response = RestClient.post(url, payload,
+              {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'application/json',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |r_response, request, result, &block|
+          rest_result = handle_rest_response(r_response,request,result,&block)
+          do_retry = false
+        end
+        do_retry = response.nil?
+        puts "Received nil response, rertrying..." if response.nil?
+      rescue RestClient::Exceptions::OpenTimeout => e
+        e.response
+        puts "Timed out. Retrying..."
+        do_retry = true
+      rescue RestClient::Exceptions::ReadTimeout => e
+        e.response
+        puts "Timed out. Retrying..."
+        do_retry = true
+      rescue SocketError => e
+        puts "SocketError. Retrying..."
+        do_retry = true
+      rescue RestClient::InternalServerError => e
+        puts "RestClient::InternalServerError Retrying..."
+        do_retry = true
+      rescue RestClient::GatewayTimeout => e
+        puts "RestClient::GatewayTimeout Retrying..."
+        do_retry = true
+      end
+      sleep 5 if do_retry
+    end
+    sleep 2
+    return rest_result
+  end
+
+  def self.api_accessprofile_bulkdelete(org,user,password,client_id,client_secret,data,token=nil)
+    api = "api/v2/access-profiles/bulk-delete/"
+    payload = JSON.generate(data)
+    url = URI.join(org,api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, payload,
+              {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'application/json',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    puts "result = "+JSON.parse(rest_result).inspect if @debug
+    sleep 2
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create an access profile on an app
+  # the "data" is a hash that has required elements: 
+  # :name :sourceId :entitlements :description
+  # :entitlements is an array of the entitlement IDs
+  #
+  def self.api_app_createaccessprofile(org,user,password,client_id,client_secret,data,token=nil)
+    puts "Don't use this method api_app_createaccessprofile, creating on source instead with api_accessprofile_create"
+    return api_accessprofile_create(org, user, password, api_user ,api_key, data, token)
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update an access profile
+  # the "data" is a hash that has required elements: 
+  # :name :sourceId :entitlements :description :id
+  # :entitlements is an array of the entitlement IDs for the AP, and :id is the
+  # id of the target access profile to update
+  #
+  def self.api_accessprofile_update(org,user,password,client_id,client_secret,data,token=nil)
+    api = "api/v2/access-profiles/"
+    org = org.split('.').first
+    data[:entitlements] = Array(data[:entitlements].reject(&:empty?))
+    data[:description] = data[:name] if data[:description].nil?
+    id = data[:id]
+    data.delete(:id)
+    json_data = JSON.generate data
+    payload = json_data
+    url = URI.join("#{org}.identitynow.com", api, id).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.patch(url, payload,
+              {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'application/json',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    sleep 2
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Post an access request
+  #
+  # data is a hash including:
+  # :user_id - id of the user who is making the access request
+  # :app_id - id of the application being requested
+  # :ap_id - id of the access profile being requested
+
+  def self.api_accessrequest_make(org,user,password,client_id,client_secret,data,basic_auth,token=nil)
+    api = "/api/v2/users/#{data[:user_id]}/launchers?appId=#{data[:app_id]}&accessProfileId=#{data[:ap_id]}"
+    org = org.split('.').first
+    url = URI.join("#{org}.identitynow.com", api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, nil, headers = {'Authorization' => "Basic #{basic_auth}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'application/json',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    sleep 2
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Access Request approvals
+  #
+  # data is a hash including:
+  # :request_id - id of access request to approve (or nil)
+  # :action - approve or reject (or nil)
+  # data is REQUIRED but can be nil
+  # basic auth probably won't work except for getting a list of all approvals
+
+  # /api/v2/approvals/<id>/approve (or reject)
+
+  def self.api_accessrequest_approval(org,user,password,client_id,client_secret,data,basic_auth,token=nil)
+    if data.nil?
+      api = "/api/v2/approvals/"
+    else
+      api = "/api/v2/approvals/#{data[:request_id]}/#{data[:action]}"
+    end
+    org = org.split('.').first
+    
+    url = URI.join("#{org}.identitynow.com", api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, nil, headers = {'Authorization' => "Basic #{basic_auth}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'application/json',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    sleep 2
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create a custom connector
+  #
+  def self.api_connector_create(org,user,pass,client_id,client_secret,args,token)
+    api = "api/connector/create"
+    url = URI.join(org, api).to_s
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, args,
+                         {'Authorization' => "Bearer #{token}",'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Discover Schema
+  #
+  def self.api_source_discoverschema(org,user,password,client_id,client_secret,id,token=nil)
+    api = "/cc/api/source/discoverSchema"
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    token = token || idn_oauth_3(org,client_id,client_secret,user,pass) 
+    url = URI.join(base_url,api).to_s+"/"+id.to_s
+    redirected_url = nil
+    rest_result = nil
+    response = RestClient::Request.new({
+      method: :post,
+      url: url,
+      path: api+"/"+id.to_s,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Cache-Control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Set schema (beta)
+  # 
+  def self.set_schema(org,user,pass,clent_id,client_secret,source_id,schema,token=nil)
+    return nil if schema.nil?
+    # first figure out if a schema already exists of this type, because it's PUT vs. POST
+
+    api = "/beta/sources/#{source_id}/schemas/"
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    url = URI.join(base_url, api).to_s
+    token = token || idn_oauth_3(org,user,pass,client_id,client_secret) 
+    rest_result = nil
+    redirected_url = nil
+    response = RestClient::Request.new({
+      method: :get,
+      url: url,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Content-Type' => 'application/json',
+                    'cache-control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+
+    existing_schemas = JSON.parse(rest_result)
+    found_existing = false
+    existing_schema_id = nil
+    existing_schemas.each do |s|
+      if s["nativeObjectType"] == schema["nativeObjectType"]
+        schema["id"] = s["id"]
+        existing_schema_id = s["id"]
+        found_existing = true
+      end
+    end
+    body = JSON.generate(schema)
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    
+    if !found_existing
+      api = "/beta/sources/#{source_id}/schemas/"
+      url = URI.join(base_url, api).to_s
+      token = token || idn_oauth_3(org,user,pass,client_id,client_secret) 
+      rest_result = nil
+      redirected_url = nil
+      response = RestClient::Request.new({
+        method: :post,
+        url: url,
+        payload: body,
+        timeout: nil,
+        headers: {'Authorization' => "Bearer #{token}", 
+                      "Accept"=>"*/*" ,
+                      'Content-Type' => 'application/json',
+                      'cache-control' => 'no-cache'
+                      }
+         }).execute do |response, request, result, &block|
+        rest_result = handle_rest_response(response, request, result, &block)
+      end
+    else
+      api = "/beta/sources/#{source_id}/schemas/#{existing_schema_id}"
+      url = URI.join(base_url, api).to_s
+      token = token || idn_oauth_3(org,user,pass,client_id,client_secret) 
+      rest_result = nil
+      redirected_url = nil
+      response = RestClient::Request.new({
+        method: :put,
+        url: url,
+        payload: body,
+        timeout: nil,
+        headers: {'Authorization' => "Bearer #{token}", 
+                      "Accept"=>"*/*" ,
+                      'Content-Type' => 'application/json',
+                      'cache-control' => 'no-cache'
+                      }
+         }).execute do |response, request, result, &block|
+        rest_result = handle_rest_response(response, request, result, &block)
+      end
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Import source xml file
+  #
+  def self.api_source_import(org,user,password,client_id,client_secret,id,filename,token=nil)
+    api = "/cc/api/source/import"
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    token = token || idn_oauth_3(org,client_id,client_secret,user,pass) 
+    url = URI.join(base_url,api).to_s+"/"+id.to_s
+    redirected_url = nil
+    rest_result = nil
+    response = RestClient::Request.new({
+      method: :post,
+      url: url,
+      path: api+"/"+id.to_s,
+      :payload => { :file => File.new(filename, 'rb') },  
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'cache-control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Export connector zip file
+  #
+  def self.api_connector_export(org,user,pass,client_id,client_secret,connector_id,token)
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    api = "api/connector/export/"+connector_id.to_s
+    url = URI.join(org, api).to_s
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    zip_data = rest_result
+    
+    zip_filename = "#{connector_id}.zip"
+    puts "Writing connector file: "+zip_filename
+    pwd = Dir.pwd
+    out_file = File.open(File.join(pwd, zip_filename), 'w')
+    out_file.write(zip_data)
+    out_file.close
+
+    return zip_data
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Import connector zip file
+  #
+  def self.api_connector_import(org,user,password,client_id,client_secret,id,filename,token=nil)
+    api = "api/connector/import/#{id.to_s}"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, {:file => File.new(filename)},
+                         {'Authorization' => "Bearer #{token}",'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Export source flat file
+  #
+  def self.api_source_export_accounts(org,user,password,client_id,client_secret,source_id,token=nil)
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+
+    json_source = JSON.parse(api_get(org, "source/get/#{source_id}", user, password, api_user, api_key, token))
+    # if json_source["sourceType"] == "DELIMITED_FILE" 
+    #   source_csv = api_get(org, "source/exportAccountFeed/?id=#{source_id}", user, password, api_user, api_key, @token)
+    #   return source_csv
+    # end
+
+    # we will have to 
+    # 1. runAccountsExportReport
+    # 2. get the task id
+    # 3. poll until the task is finished
+    # 4. download the file
+    # 5. unzip the file
+    # 6. read the unzipped file
+    # 7. delete the fields
+    # 8. return the csv
+
+    t = Time.now.to_i
+
+    args = {:sourceId => source_id, :reportName => "Accounts Report"}
+
+    api = "api/source/runAccountsExportReport/"+source_id.to_s
+    url = URI.join(org, api).to_s
+    # puts "executing API POST #{url}"
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, args, {'Authorization' => "Bearer #{token}",
+                                'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    puts "result = "+JSON.parse(rest_result).inspect if @debug
+
+    require "shellwords"
+    task = JSON.parse(rest_result)
+    zip_filename = Shellwords.escape(task["name"]+".zip")
+    zip_filename = "accounts.zip"
+
+    api = "api/taskResult/get/?id="+task["id"]
+    loop do
+      sleep 2
+      result = JSON.parse(api_get(org,api,user,password,client_id,client_secret,@token))
+      break if result["progress"] == "Completed"
+    end
+
+    api = "api/report/get/"+task["id"].to_s+"?format=csv&name=source_"+source_id.to_s+"_accounts"
+    url = URI.join(org, api).to_s
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    zip_data = rest_result
+
+    pwd = Dir.pwd
+    tmp_dir = File.join(pwd,"_zipfile_temp_#{t}")
+    Dir.mkdir tmp_dir unless File.exist? tmp_dir 
+
+    out_file = File.open(File.join(tmp_dir, zip_filename), 'w')
+
+    out_file.write(zip_data)
+    out_file.close
+
+    require 'zip'
+    file = File.join(tmp_dir, zip_filename)
+
+    csv_filename = ""
+
+    Zip::File.open(file) do |zip_file|
+      zip_file.each do |f|
+        fpath = File.join(tmp_dir, f.name)
+        result = zip_file.extract(f, fpath)
+        csv_filename = f.name
+      end
+    end
+
+    accounts_csv = File.read(File.join(tmp_dir, csv_filename))
+    puts "Exported accounts flat file: "+csv_filename
+
+    require 'fileutils'
+    FileUtils.rm_r tmp_dir
+
+    return accounts_csv
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Import source flat file
+  #
+  def self.api_source_loadaccounts(org,user,password,client_id,client_secret,id,filename,token=nil)
+    api = "api/source/loadAccounts/"
+    url = URI.join(org, api, id).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, {:file => File.new(filename)},
+                         {'Authorization' => "Bearer #{token}",'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update the config of a source from the output of api/sources/list for that
+  # source. config_json is the config data. Deletes keys that can't be updated.
+  # I found this list by trial and error and it's almost definitely not complete.
+  # -JK
+  #
+  def self.api_service_update(org,user,password,client_id,client_secret,id,config_json,token=nil)
+    puts "calling api/source/update instead of [requested] api/service/update."
+    api = "api/source/update/"
+    url = URI.join(org, api, id).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    data = config_json.first
+    # have to delete some stuff from the data that will not import on a target source.
+    # this list will have to be constantly updated when each new source type is encountered
+    # because our APIs are totally not user-friendly
+    a = ["id",  "version", "lastUpdated", "appCount", "userCount", "sourceConnected", 
+      "connectedApps", "connector_secret", "connector_kid", "health", "externalId", 
+      "passwordPolicy", "passwordPolicyName", "passwordSyncGroup", 
+      "connectorCluster", 
+      "connectorClusterGmtOffset", "authenticationCookie", "certificate","metadata","entityID",
+      "forceNameID", "hideAcsUrl", "hideAttributeMap", "hideEntityID", "acsUrl", "applicationTemplate",
+      "encryptAssertion", "connector_password", "connector_user", "attributeMap",
+      "domainName", "enableSAML", "nameIdFormat", "none_supported", "passwordReplay_supported", 
+      "proxy_supported", "samlBinding", "saml_supported", "wsfed_supported"]
+      # for whatever reason, icon is not getting set
+      # eventually need to do passwordPolicy and passwordSyncGroup
+      # attributeMap should be put back eventually, it's a bug due to switching from OpenDJ
+    a.each{|i|data.delete i}
+    args = hash_to_args data # it comes in as an array with a hash first
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, args,
+                          {'Authorization' => "Bearer #{token}",'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    puts "result = "+JSON.parse(rest_result).inspect if @debug
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update properties of a source
+  #
+  def self.api_source_update(org,user,password,client_id,client_secret,args,token=nil)
+    api = "api/source/update/"
+
+    url = URI.join(org, api, args[:id].to_s).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    do_retry = true
+    loop do
+      break unless do_retry
+      RestClient.post(url, args,
+                          {'Authorization' => "Bearer #{token}",'usage' => 'create', 'content_type' => 'text/plain',
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+        rest_result = handle_rest_response(response,request,result,&block)
+      end
+      do_retry = ([503,504].include? rest_result.code.to_i)
+      sleep 5 if do_retry # going to retry
+      puts "Retrying source update due temporary error." if do_retry
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update the so-called "Create Profile" (account profile) on the specified 
+  # source
+  #
+  def self.api_accountprofile_bulkupdate(org,user,password,client_id,client_secret,id,account_profile_json,token=nil)
+
+    api = "api/accountProfile/bulkUpdate/"
+    url = cc_url(URI.join(org, api, id).to_s).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    payload = JSON.generate(JSON.parse(account_profile_json)).to_s if account_profile_json.is_a? String
+    payload = JSON.generate(account_profile_json).to_s if account_profile_json.is_a? Hash
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, payload,
+                          {'Authorization' => "Bearer #{token}",'usage' => 'create', 
+                          'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate',
+                          'Accept-Language' => 'en-US,en;q=0.8', 'Cache-Control' => 'no-cache',
+                          'Connection' => 'keep-alive', 'Content-Type' => 'text/plain;charset=UTF-8',
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update the so-called "Create Profile" (account profile) on the specified 
+  # source
+  #
+  def self.api_accountprofile_update(org,user,password,client_id,client_secret,id,account_profile_json,token=nil)
+    return nil if account_profile_json.nil?
+    api = "api/accountProfile/update/"
+    url = cc_url(URI.join(org, api, id).to_s)+"?usage=Create"
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    payload = JSON.generate(account_profile_json) # if account_profile_json.is_a? Hash
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, payload,
+                          {'Authorization' => "Bearer #{token}", 
+                          'Accept' => '*/*', 
+                          "Content-Type" => "application/json",
+                          'Cache-Control' => 'no-cache',
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Reset a source
+  #
+  def self.api_source_reset(org,user,password,client_id,client_secret,id,query,token=nil)
+    api = "api/source/reset/"
+    url = cc_url(URI.join(org, api, id, query).to_s).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, nil,
+                         {'Authorization' => "Bearer #{token}",'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Import entitlement CSV file
+  #
+  def self.api_source_import_entitlementscsv(org,user,password,client_id,client_secret,id,filename,token=nil)
+    api = "api/source/importEntitlementsCsv/"
+    url = URI.join(org, api, id).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, {:file => File.new(filename)},
+                         {'Authorization' => "Bearer #{token}",'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Aggregate a source with optimization disabled
+  #
+  def self.api_aggregate_source(org,user,password,client_id,client_secret,id,token=nil)
+    api = "api/source/loadAccounts/"
+    url = cc_url(URI.join(org, api, id).to_s+"?disableOptimization=true")
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    redirected_url = nil
+    response = RestClient::Request.new({
+      method: :post,
+      url: url,
+      payload: nil,
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Cache-Control' => 'no-cache',
+                    'Content-Type' => 'application/json',
+                    'cache-control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Create an app
+  # args needs to be a hash including :name and :description
+  #
+  def self.api_app_create(org,user,password,client_id,client_secret,args,token=nil)
+    api = "api/app/create"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, {:name => args[:name], 
+                          :description => args[:description]},
+                          {'Authorization' => "Bearer #{token}",'usage' => 'create', 'content_type' => 'text/plain',
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update an app
+  # Also returns the JSON config data from an app so you can import it elsewhere
+  # Use this to set most of the infiormation for an app.
+  #
+  def self.api_app_update(org,user,password,client_id,client_secret,args,token=nil)
+    api = "api/app/update/"
+    id = args[:id]
+    url = URI.join(org, api, id.to_s).to_s
+    token_time = Time.now.to_i
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    token_monitor = Thread.new do
+      loop do
+        sleep 60
+        if Time.now.to_i - token_time > 300 # the token may expire
+          token = idn_oauth(org,user,password,client_id,client_secret)
+        end
+      end
+    end
+    redirected_url = nil
+    rest_result = nil
+    app_name = args[:name]
+
+    # if they only passed in the id, they probably wanted api/app/get
+    app_json = api_app_get(org,user,password,client_id,client_secret,args[:id],token)
+    current = JSON.parse(app_json) unless app_json.nil?
+    return JSON.generate(current) if args.keys == [:id]
+    # Updating an app can't be done all at once. The UI does it in multiple steps, 
+    # and the API seems to be unable to handle it any other way.
+    # Going to have to partition the dataset and perform multiple REST calls
+
+    # Remove unnecessary stuff...
+    [:appId, :serviceId, :serviceAppId, :health, :dateCreated, :lastUpdated, 
+      :applicationTemplate, :externalId, :launcherCount, :enabledSamlTimestamp, 
+      :app_accounts, :appAccounts, :authenticationCookie, 
+      :scriptName, :appProfiles, :accountServiceExternalId,
+      :passwordServiceName, :accountServiceName,
+      :passwordServiceUseForPasswordManagement,
+      :privateApp, :status, :usageAnalytics,:supportsEntitlementAggregation, 
+      :defaultAccessProfile, :accountServicePolicyName].each{|i|args.delete i}
+
+
+  # TODO: translate/fix this. actually the burden should be on the source.
+    args.delete :accountServicePolicies
+
+    args.each{|k,v| args.delete k if v.nil?}
+    args.each{|k,v| args.delete k if v == ""}
+
+    compare = {}
+
+    # fix the usageCertText so that it will parse correctly
+    args[:usageCertText] = args[:usageCertText].gsub('&', 'and') unless args[:usageCertText].nil?
+
+    args[:service] = current["service"] if args[:service].nil?
+    args.delete :accountServiceId if args[:accountServiceId].to_s == "-1"
+
+    equal = []
+    unequal_keys = [:id, :controlType, :accountServiceId]
+
+    wait_for_identity_refresh(org,user,password,client_id,client_secret,token)
+    rest_result = nil
+    source = nil
+    if args.keys.include? :accountServiceId
+      source = {:id => args[:id], :accountServiceId => args[:accountServiceId].to_s, 
+                :accountServiceMatchAllAccounts => args[:accountServiceMatchAllAccounts],
+                :provisionRequestEnabled => args[:provisionRequestEnabled],
+                :app_accounts => args[:app_accounts],
+                :appCenterEnabled => args[:appCenterEnabled],
+                :controlType => args[:controlType] }
+      source[:accountServiceMatchAllAccounts] = false if args[:accountServiceMatchAllAccounts].nil?
+      source.each{|k,v| source.delete k if v.nil?}
+      source.each{|k,v| source.delete k if v == ""}
+    end
+
+    owner = nil
+    if args.keys.include? :owner
+      unless (args[:owner] == []) || (args[:owner].nil?)
+        owner = { :id=>args[:id], :ownerId=>args[:owner]["id"]}
+      end
+    end
+
+    access_profiles = nil
+    if args.keys.include? :accessProfileIds
+      unless args[:accessProfileIds].nil?
+        access_profiles = { :id=>args[:id], :accessProfileIds => args[:accessProfileIds]}
+      end
+      args.delete :accessProfileIds
+    end
+
+    [source, owner, args].each do |rest_data| 
+      next if rest_data.nil?
+      RestClient.post(url, rest_data,
+                      {'Authorization' => "Bearer #{token}",'usage' => 'create', 'content_type' => 'application/json',
+                      'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+        rest_result = handle_rest_response(response,request,result,&block)
+      end
+      puts "result = "+JSON.parse(rest_result).inspect if @debug
+      wait_for_identity_refresh(org,user,password,client_id,client_secret,token)
+    end
+
+    unless access_profiles.nil?
+      # RestClient has a bug that results in an array in the data field being posted wrong, so we can't use it to set access profiles
+      # Using curl instead. hackish and non-portable. But nothing else works.
+      headers = "-H \'Authorization: Bearer #{token}\' -H \'usage: create\' -H \'Content-Type: application/json\' -H \'X-CSRF-Token: nocheck\'"
+      curl_payload = JSON.generate(access_profiles)
+      puts "Due to an issue with RestClient on passing arrays in JSON data, using curl..."
+      cmd = "curl -X POST #{headers} -d \'#{curl_payload}\' #{url}"
+      puts "\n"+cmd+"\n\n"
+      result = `#{cmd}`+"\n\n"
+      puts result
+      puts "\n\nUnable to find Access Profiles matching id(s): #{access_pofiles[:accessProfileIds].join(",")} 
+            for source ID: #{source[:accountServiceId]}" if result.split.include? "\"NotFoundException:" # no error handling for curl!
+      wait_for_identity_refresh(org,user,password,client_id,client_secret,token)
+    end
+    # Get the current state of the app to return
+    result = JSON.parse(api_app_get(org,user,password,client_id,client_secret,args[:id],token))
+    return result
+  end
+
+  def self.source_name_id_extid(org, user, pass, api_user, api_key)
+    sources_list = JSON.parse(api_get(org, "source/list", user, pass, api_user, api_key, @token))
+    name_id_extid = {}
+    sources_list.each do |i|
+      next if i["name"] == "IdN Admins" # skip this source, which is flat file and unique to each IdN org
+      id = i["id"].to_i
+      ext_id = i["externalId"]
+      name = i["name"]
+      name_id_extid[name] = {:id => id, :ext_id => ext_id}
+    end
+    return name_id_extid
+  end
+
+  def self.entitlement_id_transform(org, user, pass, api_user, api_key)
+    entitlement_maps = {}
+    source_dir_contents = Dir.entries(@sources_dir)[2..-1]
+    sources = {}
+    entitlement_maps = {}
+    # get the names of all of the sources
+    a = []
+    source_dir_contents.each{|f| a << f.split(File.extname(f)).first.split('_')[2]}
+    a.each{|i|sources[i] = {}}
+
+    source_dir_contents.each do |f|
+      f.chomp!
+      f_base = f.split(File.extname(f)).first
+      type = f_base.split('_')[1]
+      name = f_base.split('_')[2]
+      entitlement_maps[name] = File.join(@sources_dir, f) if type == "ENTITLEMENTMAP"
+    end
+    all_entitlement_map = {}
+    entitlement_maps.each do |name, f|
+      j = JSON.parse(File.read(entitlement_maps[name]))
+      j.each do |ent|
+        all_entitlement_map[ent["id"]] = {  :source_id => ent["applicationName"].split("[source-").last[0..-2], 
+                                            :source_name => name,
+                                            :name =>ent["displayName"] }
+        all_entitlement_map[ent["id"]][:name] = ent["displayableName"] if ent["displayName"].nil? # could be a bug?
+      end
+    end
+    sources_list = JSON.parse(api_get(org, "source/list", user, pass, api_user, api_key, @token))
+    ext_ids = {}
+    sources_list.each do |i|
+      next if i["name"] == "IdN Admins" # skip this source, which is flat file and unique to each IdN org
+      id = i["id"].to_i
+      ext_id = i["externalId"]
+      ext_ids[id] = ext_id
+    end
+    target_entitlements = []
+    threads = []
+    ext_ids.each do |id, ext_id|
+      threads << Thread.new do
+        app_entitlements = []
+        result = api_get(org, "entitlement/list/?CISApplicationId=#{ext_id}", user, pass, api_user, api_key, @token)
+        entitlement_bundle = JSON.parse(result)["items"]
+        entitlement_bundle.each do |ent|
+          entitlement = {}
+          entitlement["applicationName"] = ent["applicationName"]
+          entitlement["displayName"] = ent["displayName"]
+          entitlement["displayableName"] = ent["displayableName"]
+          entitlement["id"] = ent["id"]
+          target_entitlements << entitlement
+        end
+      end
+    end
+    threads.each(&:join)
+    id_transform = {}
+    # old => new
+    all_entitlement_map.each do |id, h|
+      new_id = nil
+      target_entitlements.each do |ent|
+        next if ent["id"].nil? # we can't use it anyway
+        target_source_name = ent["applicationName"].split(" [source-").first
+        next unless target_source_name == h[:source_name]
+        target_name = ent["displayName"]
+        target_name = ent["displayableName"] if ent["displayName"].nil? # I think this is a bug... in the API code. -JK
+        new_id = ent["id"] if target_name == h[:name]
+      end
+      id_transform[id] = new_id
+    end
+    return id_transform
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Import an app
+  # args is a hash, :id and :xml which is the filename
+  # You cannot import a pre-installed app!
+  #
+  def self.api_app_import(org,user,password,client_id,client_secret,args,token=nil)
+    # required:  "id","xml"
+    api = "api/app/import/"
+    url = URI.join(org, api, args[:id].to_s).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    filename = args[:xml]
+    RestClient.post(url, { :xml => args[:xml]},
+                          {'Authorization' => "Bearer #{token}",'usage' => 'create', 'content_type' => 'multipart/form-data',
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Import an custom icon
+  # args is a hash, :id and :image which is the filename of the png file
+  # file has to be a 80x80 png preferably
+  # if you are downloading it from the link you get from /api/source/get then it's the "70.png" in the specified location
+  #
+  def self.api_source_upload_custom_icon(org,user,password,client_id,client_secret,args,token=nil)
+    # required:  "id","xml"
+    api = "api/source/uploadCustomIcon/"
+    url = URI.join(org, api, args[:id].to_s).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    filename = args[:image]
+    RestClient.post(url, { :image => args[:image]},
+                          {'Authorization' => "Bearer #{token}",'usage' => 'create', 'content_type' => 'text/plain',
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Import an custom connector jar file
+  # args is a hash, "id" and "file" which is the filename of the jar file
+  #
+  def self.api_source_upload_connector_file(org,client_id,client_secret,user,pass,args,token=nil)
+    api = "/cc/api/source/uploadConnectorFile"
+    base_url = org.split(".identitynow.com").first+".api.identitynow.com"
+    token = token || idn_oauth_3(org,client_id,client_secret,user,pass) 
+    args.each{|k,v|args[k.to_s] = v} #inconsistently using symbols and strings
+    url = URI.join(base_url,api).to_s+"/"+args["id"].to_s
+    redirected_url = nil
+    rest_result = nil
+    filename = args["file"]
+    response = RestClient::Request.new({
+      method: :post,
+      url: url,
+      path: api+"/"+args["id"].to_s,
+      :payload => {
+        :multipart => true,
+        :file => File.new(filename, 'rb')
+      },  
+      timeout: nil,
+      headers: {'Authorization' => "Bearer #{token}", 
+                    "Accept"=>"*/*" ,
+                    'Cache-Control' => 'no-cache'
+                    }
+       }).execute do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Create an Identity Profile
+  #
+  def self.api_profile_create(org,user,password,client_id,client_secret,args,token=nil)
+    api = "api/profile/create"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    payload = {:name => args[:name], :sourceId => args[:source_id]}
+    RestClient.post(url, payload,
+              {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'text/plain;charset=UTF-8',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create an Identity Attribute
+  #
+  def self.api_identityattribute_create(org,user,password,client_id,client_secret,args,token=nil)
+    # args is going to be a hash we have to JSON
+    api = "api/identityAttribute/create"
+    url = URI.join(org, api).to_s
+    ["attributeSyncTargets", "extendedNumber", "targets"].each{|i| args.delete i}
+    payload = JSON.generate(args)
+
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, payload,
+              {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'text/plain;charset=UTF-8',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update an Identity Profile
+  # TODO: 
+  # This probably has lots of dependencies that are unresolved
+  # even if it did work, which it doesn't as of 10/12/15 -JK
+  def self.api_profile_update(org,user,password,client_id,client_secret,id,data_hash,token=nil)
+    api = "api/profile/update/"
+    url = URI.join(org, api, id.to_s).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    payload = JSON.generate(data_hash)
+    RestClient.post(url, payload,
+              {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'text/plain;charset=UTF-8',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create a Transform (seaspray)
+  #
+  def self.api_transform_create(org,user,password,client_id,client_secret,args,token=nil)
+    return if ["ISO3166 Country Format", "Remove Diacritical Marks"].include? args[:id]
+    api = "api/transform/create"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    payload = JSON.generate(args) # should be a hash
+    RestClient.post(url, payload,
+              {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'text/plain;charset=UTF-8',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update a transform
+  #
+  def self.api_transform_update(org,user,password,client_id,client_secret,args,token=nil)
+    return if ["ISO3166 Country Format", "Remove Diacritical Marks"].include? args[:id]
+    api = "api/transform/update"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    payload = JSON.generate(args)
+    RestClient.post(url, payload,
+              {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'text/plain;charset=UTF-8',
+                'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create a governance group (workgroup)
+  # 
+  def self.api_workgroup_create(org,user,password,client_id,client_secret,args,token=nil)
+    # args is a hash
+    ["connectionCount", "created", "id", "memberCount", "modified"].each{|k| args.delete k}
+    api = "api/v2/workgroups"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    payload = JSON.generate(args).to_s
+    RestClient.post(url, payload,
+                          {'Authorization' => "Bearer #{token}",'usage' => 'create', 'Content-Type' => 'application/json',
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create a role
+  # 
+  def self.api_role_create(org,user,password,client_id,client_secret,name,token=nil)
+    api = "api/role/create"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.post(url, {:name => name},
+                          {'Authorization' => "Bearer #{token}",'usage' => 'create', 'Content-Type' => 'application/x-www-form-urlencoded',
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update a role
+  # 
+  def self.api_role_update(org,user,password,client_id,client_secret,id,args,token=nil)
+    api = "api/role/update/"
+    url = URI.join(org, api, id).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    args.delete :name
+    args["selector"].each{|k,v| args["selector"].delete k if v.nil?}
+    payload = JSON.generate(args).to_s
+    RestClient.post(url, payload,
+                  {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                    'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'text/plain;charset=UTF-8',
+                    'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                    'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Create a Password Policy
+  # 
+  def self.api_password_policy_create(org,user,password,client_id,client_secret,args,token=nil)
+    api = "api/passwordPolicy/create/"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    args.delete :id
+    args.delete 
+    payload = JSON.generate(args).to_s
+    RestClient.post(url, payload,
+                  {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                    'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'text/plain;charset=UTF-8',
+                    'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                    'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Update a Password Policy
+  # 
+  def self.api_password_policy_set(org,user,password,client_id,client_secret,id,args,token=nil)
+    api = "api/passwordPolicy/set"
+    url = URI.join(org, api, id).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    args.delete :id
+    # :connectedServices may refer to sources that are not set up ... should process these out of the 
+    # args before making the call
+    payload = JSON.generate(args).to_s
+    RestClient.post(url, payload,
+                  {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck',
+                    'Accept-Language' => 'en-US,en;q=0.8', 'Content-Type' => 'text/plain;charset=UTF-8',
+                    'Accept' => '*/*', 'Accept-Encoding' => 'gzip, deflate', 'Cache-Control' => 'no-cache',
+                    'Connection' => 'keep-alive' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Certification API UTILS
+  #-----------------------------------------------------------------------------#
+
+  #-----------------------------------------------------------------------------#
+  # List all certification campaigns
+  #
+  def self.api_campaign_list(org,user,password,client_id,client_secret,token=nil)
+    api = "api/campaign/list"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create a certification campaigns
+  # args requires data that doesn't come easily or at all out of the get/list methods
+  #
+  def self.api_campaign_create(org,user,password,client_id,client_secret,args,token=nil)
+
+    new_args = "?"
+    args.each do |k,v|
+      new_args = new_args+k.to_s+"="+v.to_s+"&" unless k == :sourceIds # sourceIds will be an array
+      if k == :sourceIds  # must be converted to multiple keys with each one value
+        s = ""
+        args[k].each {|id| s = s+"sourceIds="+id.to_s+"&"}
+        new_args = new_args+s
+      end
+    end
+
+    new_args.chop!
+
+    args = URI.escape(new_args)
+
+    api = "api/campaign/create"
+    url = URI.join(org, api, args).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+
+    RestClient.post(url, "", {'Authorization' => "Bearer #{token}",
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # List all certifications
+  #
+  def self.api_certification_list(org,user,password,client_id,client_secret,token=nil)
+    api = "api/certification/list"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Get a certification by id
+  #
+  def self.api_certification_get(org,user,password,client_id,client_secret,id,token=nil)
+    api = "api/certification/getCertification"
+    url = URI.join(org, api).to_s + "?certificationId=#{id}"
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Complete an assigned task
+  #
+  def self.api_task_complete(org,user,password,client_id,client_secret,id,token=nil)
+    api = "api/task/complete/"+id.to_s
+
+    # args = {:id => id}
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+
+    RestClient.post(url, nil, {'Authorization' => "Bearer #{token}",
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # API ACCOUNT STUFF
+  # CRUD accounts
+  #-----------------------------------------------------------------------------#
+
+  #-----------------------------------------------------------------------------#
+  # Get a list of accounts on a source
+  # returns JSON
+  # include alias (string) or id (integer) as arg to return only the identity for that one user
+  #
+  def self.api_accounts_get(org,user,password,client_id,client_secret,source_id,token=nil)
+    api = "api/v2/accounts/"
+    url = URI.join(org, api).to_s+"?sourceId="+source_id.to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Create an account on a FF source
+  # returns JSON
+  # include source id (integer) and the body as a hash of attributes
+  #
+  def self.api_accounts_post(org,user,password,client_id,client_secret,source_id,args,token=nil)
+    api = "api/v2/accounts/"
+    url = URI.join(org, api).to_s+"?sourceId="+source_id.to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    RestClient.post(url, args, {'Authorization' => "Bearer #{token}",
+                          'X-CSRF-Token' => 'nocheck' }) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return rest_result
+  end
+
+  #-----------------------------------------------------------------------------#
+  # API MISC UTILS
+  # Methods to assist in running API calls besides POST and GET
+  #-----------------------------------------------------------------------------#
+
+  def self.get_api_key(org,user,password,client_id,client_secret)
+    api = "api/client/create?type=CLI"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret)
+    rest_result = nil
+    RestClient.post(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' } ) do |response, request, result, &block|
+      rest_result = handle_rest_response(response, request, result, &block)
+    end
+    return rest_result
+  end
+
+  def self.hash_password(user,password)
+    # generates a password hash for oauth token authentication
+    user_hash = Digest::SHA256.hexdigest(user)
+    password_hash = Digest::SHA256.hexdigest([password,user_hash].join)
+    return password_hash
+  end
+
+  def self.idn_oauth(org,user,password,client_id,client_secret)
+    return idn_oauth_3(org,user,password,client_id,client_secret)
+  end
+
+  def self.idn_oauth_2(org,client_id,client_secret)
+    org = org.split(".identitynow.com").first+".api.identitynow.com"
+    api = "/oauth/token"
+
+    args = "?grant_type=client_credentials&client_id=#{client_id}&client_secret=#{client_secret}"
+    url = URI.join(org, api, args).to_s
+    rest_result = nil
+    RestClient.post(url, nil, {'X-CSRF-Token'=>'nocheck'}) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    return token
+  end
+
+  def self.idn_oauth_3(org,user,password,client_id,client_secret)
+    user = user.downcase
+    pass_hash = hash_password(user,password)
+    org = org.split(".identitynow.com").first+".api.identitynow.com"
+    api = "/oauth/token"
+    resource = RestClient::Resource.new(org, client_id, client_secret)
+    args = "?grant_type=password&client_id=#{client_id}&client_secret=#{client_secret}&username=#{user}&password=#{pass_hash}"
+    url = URI.join(org, api, args).to_s
+    rest_result = nil
+    RestClient.post(url, nil, {'X-CSRF-Token'=>'nocheck'}) do |response, request, result, &block|
+      rest_result = handle_rest_response(response,request,result,&block)
+    end
+    token = JSON.parse(rest_result)["access_token"] unless rest_result.nil?
+    return token
+  end
+
+  #-----------------------------------------------------------------------------#
+  # Get status of pending identity updates
+  # this is what displays the "blue bar"
+  #
+  def self.identity_update_pending(org,user,password,client_id,client_secret,token=nil)
+    api = "api/org/getPendingIdentityTasks"
+    url = URI.join(org, api).to_s
+    token = token || idn_oauth(org,user,password,client_id,client_secret) 
+    redirected_url = nil
+    rest_result = nil
+
+    do_retry = true
+    loop do
+      break unless do_retry
+      begin
+        response = RestClient.get(url, {'Authorization' => "Bearer #{token}", 'X-CSRF-Token' => 'nocheck' }  ) do |r_response, request, result, &block|
+          rest_result = handle_rest_response(r_response, request, result, &block)
+          do_retry = false
+        end
+        do_retry = response.nil?
+        puts "Received nil response, rertrying..." if response.nil?
+      rescue RestClient::Exceptions::OpenTimeout => e
+        e.response
+        puts "Timed out. Retrying..."
+        do_retry = true
+      rescue RestClient::Exceptions::ReadTimeout => e
+        e.response
+        puts "Timed out. Retrying..."
+        do_retry = true
+      rescue SocketError => e
+        puts "SocketError. Retrying..."
+        do_retry = true
+      rescue RestClient::InternalServerError => e
+        puts "RestClient::InternalServerError Retrying..."
+        do_retry = true
+      end
+    end
+    return rest_result
+  end
+
+
+  #-----------------------------------------------------------------------------#
+  # Wait until identity updates are completed
+  # this is what displays the "blue bar"
+  #
+  def self.wait_for_identity_refresh(org,user,password,client_id,client_secret,token=nil)
+    puts "Waiting for Identity refresh to complete."
+    token_time = Time.now.to_i
+    token = token || idn_oauth_3(org,user,password,client_id,client_secret)
+    token_monitor = Thread.new do
+      loop do
+        sleep 60
+        if Time.now.to_i - token_time > 300 # the token may expire
+          token = idn_oauth_3(org,user,password,client_id,client_secret)
+          token_time = Time.now.to_i
+        end
+      end
+    end
+    t = 0
+    loop do
+      result = JSON.parse(identity_update_pending(org,user,password,client_id,client_secret,token))
+      break if result == []
+      break if !result.first["completed"].nil?
+      print "\r"+t.to_s+" seconds..."
+      sleep 10
+      t += 10
+      # break if t == 18000 # if we are waiting 300 minutes, there's a problem
+    end
+    puts "\n"
+    puts "result = "+JSON.parse(result).inspect if @debug
+    puts "Waited #{t} seconds for Identity refresh. Resuming."
+  end
+
+  #-----------------------------------------------------------------------------#
+  # wait until a job is complete
+  #
+  def self.wait_for_job(org,user,password,client_id,client_secret,job_id,token=nil)
+    t = 0
+    loop do
+      sleep 1
+      t += 1
+      result = JSON.parse(api_get("/api/job/getStatus/#{job_id}", org, user, password, api_user, api_key, token))
+      break if result.values.include? "Success"
+      break if t == 3600 # if we are waiting 60 minutes, there's might be a problem
+    end
+    puts "result = "+JSON.parse(result).inspect if @debug
+    puts "\n\nTimed out waiting for job #{job_id} to finish.\n\n" if t >= 3600
+    puts "\n\nWaited #{t} seconds for job #{job_id} to finish.\n\n" if t < 3600
+  end
+
+  def self.handle_rest_response(response,request,result,&block)
+   if response.code < 400
+      # puts "Success:\n"+request.url.inspect+"\n\n"
+      response.return!
+    elsif response.code == 400
+      puts "Bad Request:\n"+[request.url.inspect, request.args.inspect, request.raw_response.inspect, response.body].join("\n")
+      response.return!
+    elsif response.code == 403
+      puts "Unauthorized:\n"+[request.url.inspect, request.args.inspect, request.raw_response.inspect, response.body].join("\n")
+      response.return!
+    elsif response.code == 404
+      puts "Resource Not Found:\n"+[request.url.inspect, request.args.inspect, request.raw_response.inspect, response.body].join("\n")
+      response.return!
+    elsif response.code == 500
+      puts "IdentityNow Internal Server Error:\n"+[request.url.inspect, request.args.inspect, request.raw_response.inspect, response.body].join("\n")
+      response.return!
+    elsif response.code == 504
+      puts "Gateway Timeout:\n"+[request.url.inspect, request.args.inspect, request.raw_response.inspect, response.body].join("\n")
+      response.return!
+    else
+      puts "Error:\n"+[request.url.inspect, request.args.inspect, request.raw_response.inspect, response.body].join("\n")
+      response.return!
+    end
+  end
+
+  def self.hash_to_args(hash)
+    # convert a hash to URL args
+    args = "?"
+    hash.each do |k,v|
+      args = args+k.to_s+"="+v.to_s+"&"
+    end
+    args.chop!
+    return URI.escape(args)
+  end
+
+  def self.unescape(s)
+    YAML.load(%Q(---\n"#{s}"\n))
+  end
+
+  def self.data_binary(data)
+    # data had better be string from JSON.generate
+    s = JSON.pretty_generate(JSON.parse(data))
+    return ['--data-binary $\'', s, '\''].join
+  end
+
+  def self.csv_to_json(csv)
+    # convert CSV file to JSON
+    require 'csv'
+    lines = csv.split("\n")
+    a = []
+    keys = []
+    lines.first.split(",").each{|k|keys << k.to_sym}
+    lines[1..-1].each do |row|
+      h = {}
+      fields = row.parse_csv
+      keys.each_with_index{|k,i| h[k] = fields[i]}
+      a << h
+    end
+    result = JSON.generate(a)
+    return result
+  end
+
+  def self.rest_value_array(a)
+    return a if (a == []) || (a.nil?)
+    s = ["["]
+    a.each{ |i| s << i+','}
+    s.last.chop!
+    s << "]"
+    a = s.join
+    return a
+  end
+
+  def self.extract_zip(file)
+    require 'zip'
+    destination = file.split('.').first
+    FileUtils.mkdir_p(destination)
+    output_file = ""
+    Zip::File.open(file) do |zip_file|
+      zip_file.each do |f|
+        fpath = File.join(destination, f.name)
+        result = zip_file.extract(f, fpath)
+        output_file = result.keys.first
+      end
+    end
+    return output_file
+  end
+
+end #class