aspera-cli 4.0.0.pre3 → 4.2.1

data/docs/test_env.conf

@@ -5,8 +5,8 @@ default:
   config: cli_default
   aoc: tst_aoc1
   faspex: tst_faspex
-  faspex5: tst_faspex5
-  shares: tst_shares
+  faspex5: tst_faspex5_web
+  shares: tst_shares_1
   shares2: tst_shares2
   node: tst_node
   server: tst_server
@@ -19,7 +19,6 @@ default:
 cli_default:
   interactive: your value here
   smtp: your value here
-  ascp_path: your value here
 local_user:
   ssh_keys: your value here
 smtp_config:
@@ -48,10 +47,24 @@ tst_node_faspex:
   url: your value here
   username: your value here
   password: your value here
-tst_faspex5:
+tst_faspex5_boot:
   url: your value here
+  auth: your value here
   username: your value here
   password: your value here
+tst_faspex5_web:
+  url: your value here
+  auth: your value here
+  redirect_uri: your value here
+  client_id: your value here
+  client_secret: your value here
+tst_faspex5:
+  url: your value here
+  auth: your value here
+  client_id: your value here
+  client_secret: your value here
+  private_key: your value here
+  username: your value here
 tst_shares:
   url: your value here
   username: your value here
@@ -94,6 +107,7 @@ tst_ak_preview:
   url: your value here
   username: your value here
   password: your value here
+  mimemagic: your value here
 tst_node_preview:
   url: your value here
   username: your value here
@@ -110,6 +124,8 @@ misc:
   faspex_publink_send_to_fxuser: your value here
   faspex_publink_send_to_dropbox: your value here
   shares_upload: your value here
+  console_smart_id: your value here
+  console_smart_file: your value here
   orch_workflow_id: your value here
   file_dcm: your value here
   file_pdf: your value here
@@ -122,6 +138,7 @@ misc:
   aoc_publink_recv_from_aocuser: your value here
   aoc_publink_send_shd_inbox: your value here
   aoc_publink_send_aoc_user: your value here
+  aoc_publink_send_use_pass: your value here
   aoc_publink_folder: your value here
   aoc_shbx_ws: your value here
   aoc_shbx_name: your value here
@@ -137,4 +154,5 @@ misc:
   email_internal: your value here
   email_external: your value here
   aoc_org: your value here
-  aoc_user: your value here
+  aoc_user_email: your value here
+  http_gw_fqdn_port: your value here

data/docs/transfer_spec.html

@@ -28,7 +28,7 @@ arg: related ascp argument or env var suffix (PASS for ASPERA_SCP_PASS)
 </p>
 <p>
 UNDER CONSTRUCTION<br/>
-<a href="https://developer.ibm.com/api/view/aspera-prod:ibm-aspera:title-IBM_Aspera#id90944">Documentation&rarr;Node API&rarr;/opt/transfers</a><br/>
+<a href="https://developer.ibm.com/apis/catalog/?search=aspera">Aspera API Documentation</a>&rarr;Node API&rarr;/opt/transfers<br/>
 </p>
 
 <table>

data/examples/aoc.rb

@@ -4,11 +4,22 @@ require 'aspera/log'
 
 Aspera::Log.instance.level=:debug
 
+if ! ARGV.length.eql?(3)
+  Aspera::Log.log.error("wrong number of args: #{ARGV.length}")
+  Aspera::Log.log.error("Usage: #{$0} <aoc URL> <aoc username> <aoc private key content>")
+  Aspera::Log.log.error("Example: #{$0} https://myorg.ibmaspera.com john@example.com $(cat /home/john/my_key.pem)")
+  Process.exit(1)
+end
+
+aoc_url=ARGV[0]
+aoc_user=ARGV[1]
+aoc_key_value=ARGV[2]
+
 aocapi=Aspera::AoC.new(
-url: 'https://myorg.ibmaspera.com',
+url: aoc_url,
 auth: :jwt,
-private_key: File.read('path/to_your_private_key.pem'),
-username: 'my.email@example.com',
+private_key: aoc_key_value,
+username: aoc_user,
 scope: 'user:all',
 subpath: 'api/v1')
 

data/examples/faspex4.rb

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+require 'aspera/rest'
+require 'aspera/log'
+require 'aspera/fasp/local'
+require 'aspera/cli/listener/line_dump'
+require 'aspera/cli/extended_value'
+
+# set high log level for the example
+Aspera::Log.instance.level=:debug
+
+# set folder where SDK is installed
+# (if ascp is not there, the lib will try to find in usual locations)
+Aspera::Fasp::Installation.instance.folder='.'
+
+if ! ARGV.length.eql?(3)
+  Aspera::Log.log.error("wrong number of args: #{ARGV.length}")
+  Aspera::Log.log.error("Usage: #{$0} <faspex URL> <faspex username> <faspex password>")
+  Aspera::Log.log.error("Example: #{$0} https://faspex.com/aspera/faspex john p@sSw0rd")
+  Process.exit(1)
+end
+
+faspex_url=ARGV[0]
+faspex_user=ARGV[1]
+faspex_pass=ARGV[2]
+
+# 1: demo API v3
+#---------------
+
+# create REST API object
+api_v3=Aspera::Rest.new({
+  :base_url => faspex_url,
+  :auth     => {
+  :type     => :basic,
+  :username => faspex_user,
+  :password => faspex_pass
+  }})
+
+api_v3.read('me')
+
+# 2: send a package
+#---------------
+
+# create a sample file to send
+file_to_send='./myfile.bin'
+File.open(file_to_send, "w") {|f| f.write("sample data") }
+# package creation parameters
+package_create_params={'delivery'=>{'title'=>'test package','recipients'=>['aspera.user1@gmail.com'],'sources'=>[{'paths'=>[file_to_send]}]}}
+pkg_created=api_v3.create('send',package_create_params)[:data]
+# get transfer specification
+transfer_spec=pkg_created['xfer_sessions'].first
+# set paths of files to send
+transfer_spec['paths']=['source'=>file_to_send]
+# get the local agent (i.e. ascp)
+client=Aspera::Fasp::Local.new
+# disable ascp output on stdout to not mix with JSON events
+client.quiet=true
+# start transfer (asynchronous)
+job_id=client.start_transfer(transfer_spec)
+result=client.wait_for_transfers_completion
+#  notify of any transfer error
+result.select{|i|!i.eql?(:success)}.each do |e|
+  Aspera::Log.log.error("A transfer error occured: #{e.message}")
+end
+
+# 1: demo API v4
+#---------------
+api_v4=Aspera::Rest.new({
+  :base_url  => faspex_url+'/api',
+  :auth      => {
+  :type      => :oauth2,
+  :base_url  => faspex_url+'/auth/oauth2',
+  :grant     => :header_userpass,
+  :user_name => faspex_user,
+  :user_pass => faspex_pass,
+  :scope     => 'admin'
+  }})
+
+Aspera::Log.dump('users',api_v4.read('users')[:data])

data/lib/aspera/aoc.rb

@@ -9,6 +9,8 @@ module Aspera
     private
     @@use_standard_ports = true
 
+    API_V1='api/v1'
+
     PRODUCT_NAME='Aspera on Cloud'
     # Production domain of AoC
     PROD_DOMAIN='ibmaspera.com'
@@ -42,11 +44,6 @@ module Aspera
     FILES_APP='files'
     PACKAGES_APP='packages'
 
-    CLIENT_RANDOM={
-      'aspera.global-cli-client' => 'frpmsRsG4mjZ0PlxCgdJlvONqBg4Vlpz_IX7gXmBMAfsgMLy2FO6CXLodKfKAuhqnCqSptLbe_wdmnm9JRuEPO-PpFqpq_Kb',
-      'aspera.drive' => 'UegzQ3LcbLht5dLYAXaR-7ZMnJ6-kwPEXWEXaqLSOMGmtzNA9r6kPFLElqBfq66BfgMabdO96k5sPXV-H8M3vsx9LbGlewF1'
-    }
-
     def self.get_client_info(client_name=CLIENT_APPS.first)
       client_index=CLIENT_APPS.index(client_name)
       raise "no such pre-defined client: #{client_name}" if client_index.nil?
@@ -68,9 +65,14 @@ module Aspera
       return organization,instance_domain
     end
 
+    # base API url depends on domain, which could be "qa.xxx"
+    def self.api_base_url(api_domain=PROD_DOMAIN)
+      return "https://api.#{api_domain}"
+    end
+
     def self.metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
       return Rest.new({
-        :base_url => "https://api.#{api_domain}/metering/v1",
+        :base_url => "#{api_base_url(api_domain)}/metering/v1",
         :headers  => {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
       })
     end
@@ -121,12 +123,12 @@ module Aspera
       raise RuntimeError,'too many redirections'
     end
 
-    # @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath
+    # @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath,:password (for pub link)
     def initialize(opt)
       # access key secrets are provided out of band to get node api access
       # key: access key
       # value: associated secret
-      @secrets={}
+      @key_chain=nil
 
       # init rest params
       aoc_rest_p={:auth=>{:type =>:oauth2}}
@@ -147,12 +149,12 @@ module Aspera
 
       # get org name and domain from url
       organization,instance_domain=self.class.parse_url(opt[:url])
-      # this is the base API url (depends on domain, which could be "qa.xxx")
-      api_base_url="https://api.#{instance_domain}"
-      # base API URL
-      aoc_rest_p[:base_url]="#{api_base_url}/#{opt[:subpath]}"
+      # this is the base API url
+      api_url_base=self.class.api_base_url(instance_domain)
+      # API URL, including subpath (version ...)
+      aoc_rest_p[:base_url]="#{api_url_base}/#{opt[:subpath]}"
       # base auth URL
-      aoc_auth_p[:base_url] = "#{api_base_url}/#{OAUTH_API_SUBPATH}/#{organization}"
+      aoc_auth_p[:base_url] = "#{api_url_base}/#{OAUTH_API_SUBPATH}/#{organization}"
       aoc_auth_p[:client_id]=opt[:client_id]
       aoc_auth_p[:client_secret] = opt[:client_secret]
 
@@ -185,23 +187,19 @@ module Aspera
         aoc_auth_p[:jwt_subject]         = opt[:username]
         aoc_auth_p[:jwt_private_key_obj] = OpenSSL::PKey::RSA.new(private_key_PEM_string)
       when :url_token
+        aoc_auth_p[:password]=opt[:password] unless opt[:password].nil?
         # nothing more
       else raise "ERROR: unsupported auth method: #{aoc_auth_p[:grant]}"
       end
       super(aoc_rest_p)
     end
 
-    def add_secrets(secrets)
-      @secrets.merge!(secrets)
-      Log.log.debug("now secrets:#{secrets}")
+    def key_chain=(keychain)
+      raise "keychain already set" unless @key_chain.nil?
+      @key_chain=keychain
       nil
     end
 
-    def has_secret(ak)
-      Log.log.debug("has key:#{ak} -> #{@secrets.has_key?(ak)}")
-      return @secrets.has_key?(ak)
-    end
-
     # additional transfer spec (tags) for package information
     def self.package_tags(package_info,operation)
       return {'tags'=>{'aspera'=>{'files'=>{
@@ -293,13 +291,14 @@ module Aspera
     # no scope: requires secret
     # if secret provided beforehand: use it
     def get_node_api(node_info,node_scope=nil)
+      # X-Aspera-AccessKey required for bearer token only
       node_rest_params={
         :base_url => node_info['url'],
         :headers  => {'X-Aspera-AccessKey'=>node_info['access_key']},
       }
-      ak_secret=@secrets[node_info['access_key']]
+      ak_secret=@key_chain.get_secret(node_info['access_key'],false)
       if ak_secret.nil? and node_scope.nil?
-        raise 'There must be at least one of: secret, node scope'
+        raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}"
       end
       # if secret provided on command line or if there is no scope
       if !ak_secret.nil? or node_scope.nil?
@@ -312,7 +311,7 @@ module Aspera
         node_rest_params[:auth]=self.params[:auth].clone
         node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],node_scope)
       end
-      return Rest.new(node_rest_params)
+      return Node.new(node_rest_params)
     end
 
     # check that parameter has necessary types
@@ -328,102 +327,82 @@ module Aspera
       return node_info,file_id
     end
 
-    # returns node api and folder_id from soft link
-    def read_asplnk(current_file_info)
-      new_node_api=get_node_api(self.read("nodes/#{current_file_info['target_node_id']}")[:data],SCOPE_NODE_USER)
-      return {:node_api=>new_node_api,:folder_id=>current_file_info['target_id']}
+    # add entry to list if test block is success
+    def process_find_files(entry,path)
+      begin
+        # add to result if match filter
+        @find_state[:found].push(entry.merge({'path'=>path})) if @find_state[:test_block].call(entry)
+        # process link
+        if entry[:type].eql?('link')
+          sub_node_info=self.read("nodes/#{entry['target_node_id']}")[:data]
+          sub_opt={method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
+          get_node_api(sub_node_info,SCOPE_NODE_USER).crawl(self,sub_opt)
+        end
+      rescue => e
+        Log.log.error("#{path}: #{e.message}")
+      end
+      # process all folders
+      return true
     end
 
-    # @returns list of file paths that match given regex
     def find_files( top_node_file, test_block )
       top_node_info,top_file_id=check_get_node_file(top_node_file)
       Log.log.debug("find_files: node_info=#{top_node_info}, fileid=#{top_file_id}")
-      result=[]
-      top_node_api=get_node_api(top_node_info,SCOPE_NODE_USER)
-      # initialize loop elements : list of folders to scan
-      # Note: top file id is necessarily a folder
-      items_to_explore=[{:node_api=>top_node_api,:folder_id=>top_file_id,:path=>''}]
-
-      while !items_to_explore.empty? do
-        current_item = items_to_explore.shift
-        Log.log.debug("searching #{current_item[:path]}".bg_green)
-        # get folder content
-        begin
-          folder_contents = current_item[:node_api].read("files/#{current_item[:folder_id]}/files")[:data]
-        rescue => e
-          Log.log.warn("#{current_item[:path]}: #{e.message}")
-          folder_contents=[]
+      @find_state={found: [], test_block: test_block}
+      get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
+      result=@find_state[:found]
+      @find_state=nil
+      return result
+    end
+
+    def process_resolve_node_file(entry,path)
+      # stop digging here if not in right path
+      return false unless entry['name'].eql?(@resolve_state[:path].first)
+      # ok it matches, so we remove the match
+      @resolve_state[:path].shift
+      case entry['type']
+      when 'file'
+        # file must be terminal
+        raise "#{entry['name']} is a file, expecting folder to find: #{@resolve_state[:path]}" unless @resolve_state[:path].empty?
+        @resolve_state[:result][:file_id]=entry['id']
+      when 'link'
+        @resolve_state[:result][:node_info]=self.read("nodes/#{entry['target_node_id']}")[:data]
+        if @resolve_state[:path].empty?
+          @resolve_state[:result][:file_id]=entry['target_id']
+        else
+          get_node_api(@resolve_state[:result][:node_info],SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
         end
-        # TODO: check if this is a folder or file ?
-        Log.dump(:folder_contents,folder_contents)
-        folder_contents.each do |current_file_info|
-          item_path=File.join(current_item[:path],current_file_info['name'])
-          Log.log.debug("looking #{item_path}".bg_green)
-          begin
-            # does item match ?
-            result.push(current_file_info.merge({'path'=>item_path})) if test_block.call(current_file_info)
-            # does it need further processing ?
-            case current_file_info['type']
-            when 'file'
-              Log.log.debug("testing : #{current_file_info['name']}")
-            when 'folder'
-              items_to_explore.push({:node_api=>current_item[:node_api],:folder_id=>current_file_info['id'],:path=>item_path})
-            when 'link' # .*.asp-lnk
-              items_to_explore.push(read_asplnk(current_file_info).merge({:path=>item_path}))
-            else
-              Log.log.error("unknown folder item type: #{current_file_info['type']}")
-            end
-          rescue => e
-            Log.log.error("#{item_path}: #{e.message}")
-          end
+      when 'folder'
+        if @resolve_state[:path].empty?
+          # found: store
+          @resolve_state[:result][:file_id]=entry['id']
+          return false
         end
+      else
+        Log.log.warn("unknown element type: #{entry['type']}")
       end
-      return result
+      # continue to dig folder
+      return true
     end
 
-    # @return node information (returned by API) and file id, from a "/" based path
+    # @return Array(node_info,file_id)   for the given path
+    # @param top_node_file       Array    [root node,file id]
+    # @param element_path_string String   path of element
     # supports links to secondary nodes
-    # input: Array(root node,file id), String path
-    # output: Array(node_info,file_id)   for the given path
-    def resolve_node_file( top_node_file, element_path_string='' )
-      Log.log.debug("resolve_node_file: top_node_file=#{top_node_file}, path=#{element_path_string}")
-      # initialize loop invariants
-      current_node_info,current_file_id=check_get_node_file(top_node_file)
-      items_to_explore=element_path_string.split(PATH_SEPARATOR).select{|i| !i.empty?}
-
-      while !items_to_explore.empty? do
-        current_item = items_to_explore.shift
-        Log.log.debug "searching #{current_item}".bg_green
-        # get API if changed
-        current_node_api=get_node_api(current_node_info,SCOPE_NODE_USER) if current_node_api.nil?
-        # get folder content
-        folder_contents = current_node_api.read("files/#{current_file_id}/files")
-        Log.dump(:folder_contents,folder_contents)
-        matching_folders = folder_contents[:data].select { |i| i['name'].eql?(current_item)}
-        #Log.log.debug "matching_folders: #{matching_folders}"
-        raise "no such folder: #{current_item} in #{folder_contents[:data].map { |i| i['name']}}" if matching_folders.empty?
-        current_file_info = matching_folders.first
-        # process type of file
-        case current_file_info['type']
-        when 'file'
-          current_file_id=current_file_info['id']
-          # a file shall be terminal
-          if !items_to_explore.empty? then
-            raise "#{current_item} is a file, expecting folder to find: #{items_to_explore}"
-          end
-        when 'link'
-          current_node_info=self.read("nodes/#{current_file_info['target_node_id']}")[:data]
-          current_file_id=current_file_info['target_id']
-          # need to switch node
-          current_node_api=nil
-        when 'folder'
-          current_file_id=current_file_info['id']
-        else
-          Log.log.warn("unknown element type: #{current_file_info['type']}")
-        end
+    def resolve_node_file( top_node_file, element_path_string )
+      top_node_info,top_file_id=check_get_node_file(top_node_file)
+      path_elements=element_path_string.split(PATH_SEPARATOR).select{|i| !i.empty?}
+      result={node_info: top_node_info, file_id: nil}
+      if path_elements.empty?
+        result[:file_id]=top_file_id
+      else
+        @resolve_state={path: path_elements, result: result}
+        get_node_api(top_node_info,SCOPE_NODE_USER).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
+        not_found=@resolve_state[:path]
+        @resolve_state=nil
+        raise "entry not found: #{not_found}" if result[:file_id].nil?
       end
-      Log.log.info("resolve_node_file(#{element_path_string}): file_id=#{current_file_id},node_info=#{current_node_info}")
-      return {node_info: current_node_info, file_id: current_file_id}
+      return result
     end
 
   end # AoC