aspera-cli 4.21.1 → 4.22.0

data/lib/aspera/api/cos_node.rb

@@ -87,8 +87,8 @@ module Aspera
           receiver_client_ids: 'aspera_ats'
         )
         # get delegated token to be placed in rest call header and in transfer tags
-        @storage_credentials['token'][TOKEN_FIELD] = OAuth::Factory.bearer_extract(delegated_oauth.token)
-        @headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
+        @storage_credentials['token'][TOKEN_FIELD] = delegated_oauth.token
+        headers['X-Aspera-Storage-Credentials'] = JSON.generate(@storage_credentials)
       end
     end
   end

data/lib/aspera/api/httpgw.rb

@@ -171,7 +171,7 @@ module Aspera
           cond_var:       ConditionVariable.new
         }
         # start read thread after handshake
-        @ws_read_thread = Thread.new {process_read_thread}
+        @ws_read_thread = Thread.new{process_read_thread}
         @notify_cb&.call(:session_start, session_id: session_id)
         @notify_cb&.call(:session_size, session_id: session_id, info: total_bytes_to_transfer)
         sleep(1)
@@ -309,11 +309,15 @@ module Aspera
         # web socket endpoint: by default use v2 (newer gateways), without base64 encoding
         # is the latest supported? else revert to old api
         if !@upload_version.eql?(API_V1)
-          if !@api_info['endpoints'].any?{|i|i.include?(@upload_version)}
+          if !@api_info['endpoints'].any?{ |i| i.include?(@upload_version)}
             Log.log.warn{"API version #{@upload_version} not supported, reverting to #{API_V1}"}
             @upload_version = API_V1
           end
         end
+        @shared_info = nil
+        @ws_handshake = nil
+        @ws_io = nil
+        @ws_read_thread = nil
       end
 
       private
@@ -373,7 +377,7 @@ module Aspera
             raise "File not found: #{source_path}"
           end
         end
-        transfer_spec['paths'] = files_to_send.map{|i|{'source' => i[:name]}}
+        transfer_spec['paths'] = files_to_send.map{ |i| {'source' => i[:name]}}
         files_to_send.push(total_bytes_to_transfer)
         return files_to_send
       end

data/lib/aspera/api/node.rb

@@ -27,7 +27,7 @@ module Aspera
         :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
         :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
 
-      # node api permissions
+      # Node API permissions
       ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
       HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
       HEADER_X_TOTAL_COUNT = 'X-Total-Count'
@@ -38,7 +38,7 @@ module Aspera
       PATH_SEPARATOR = '/'
 
       # register node special token decoder
-      OAuth::Factory.instance.register_decoder(lambda{|token|Node.decode_bearer_token(token)})
+      OAuth::Factory.instance.register_decoder(lambda{ |token| Node.decode_bearer_token(token)})
 
       # class instance variable, access with accessors on class
       @use_standard_ports = true
@@ -81,6 +81,7 @@ module Aspera
         end
 
         # Create an Aspera Node bearer token
+        # @param access_key [String] Access key identifier
         # @param payload [String] JSON payload to be included in the token
         # @param private_key [OpenSSL::PKey::RSA] Private key to sign the token
         def bearer_token(access_key:, payload:, private_key:)
@@ -106,6 +107,7 @@ module Aspera
           ].join("\n")))
         end
 
+        # Decode an Aspera Node bearer token
         def decode_bearer_token(token)
           return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition(SIGNATURE_DELIMITER).first)
         end
@@ -149,7 +151,7 @@ module Aspera
 
       # Call node API, possibly adding cache control header, as globally specified
       def read_with_cache(subpath, query=nil)
-        headers = {'Accept' => 'application/json'}
+        headers = {'Accept' => Rest::MIME_JSON}
         headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless self.class.use_node_cache
         return call(
           operation: 'GET',
@@ -280,7 +282,7 @@ module Aspera
       end
 
       def refreshed_transfer_token
-        return oauth.token(refresh: true)
+        return oauth.authorization(refresh: true)
       end
 
       # @return part of transfer spec with transport parameters only
@@ -310,12 +312,12 @@ module Aspera
         when :basic
           ak_name = auth_params[:username]
           Aspera.assert(auth_params[:password]){'no secret in node object'}
-          ak_token = Rest.basic_token(auth_params[:username], auth_params[:password])
+          ak_token = Rest.basic_authorization(auth_params[:username], auth_params[:password])
         when :oauth2
           ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
-          # TODO: token_generation_lambda = lambda{|do_refresh|oauth.token(refresh: do_refresh)}
+          # TODO: token_generation_lambda = lambda{|do_refresh|oauth.authorization(refresh: do_refresh)}
           # get bearer token, possibly use cache
-          ak_token = oauth.token
+          ak_token = oauth.authorization
         when :none
           ak_name = params[:headers][HEADER_X_ASPERA_ACCESS_KEY]
           ak_token = params[:headers]['Authorization']
@@ -352,7 +354,7 @@ module Aspera
           # get the transfer user from info on access key
           transfer_spec['remote_user'] = info['transfer_user'] if info['transfer_user']
           # get settings from name.value array to hash key.value
-          settings = info['settings']&.each_with_object({}){|i, h|h[i['name']] = i['value']}
+          settings = info['settings']&.each_with_object({}){ |i, h| h[i['name']] = i['value']}
           # check WSS ports
           Transfer::Spec::WSS_FIELDS.each do |i|
             transfer_spec[i] = settings[i] if settings.key?(i)

data/lib/aspera/ascmd.rb

@@ -104,7 +104,7 @@ module Aspera
         # enclose arguments in double quotes, protect backslash and double quotes
         # ascmd uses space as token separator, and optional quotes ('") or \ to escape
         args.each do |v|
-          command.push(%Q{"#{v.gsub(/["\\]/){|s|"\\#{s}"}}"})
+          command.push(%Q{"#{v.gsub(/["\\]/){ |s| "\\#{s}"}}"})
         end
         command_lines.push(command.join(' '))
       end
@@ -141,7 +141,7 @@ module Aspera
       end
       # raise error as exception
       raise Error.new(result[:errno], result[:errstr], action_sym, arguments) if
-        result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{|i|i[:name]}.sort)
+        result.is_a?(Hash) && (result.keys.sort == TYPES_DESCR[:error][:fields].map{ |i| i[:name]}.sort)
       return result
     end
 
@@ -209,7 +209,7 @@ module Aspera
               result[field_info[:name]] ||= []
               result[field_info[:name]].push(parse(typed_buffer[:buffer], field_info[:is_a], indent_level))
             when :list_tlv_list # field is an array of values in a list of buffers
-              result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{|r|parse(r[:buffer], field_info[:is_a], indent_level)}
+              result[field_info[:name]] = parse(typed_buffer[:buffer], :blist, indent_level).map{ |r| parse(r[:buffer], field_info[:is_a], indent_level)}
             when :list_tlv_restart # field is an array of values, but a new value is started on index 1
               fl = result[field_info[:name]] = []
               parse(typed_buffer[:buffer], :blist, indent_level).map do |tb|

data/lib/aspera/ascp/installation.rb

@@ -10,7 +10,7 @@ require 'aspera/assert'
 require 'aspera/web_server_simple'
 require 'aspera/cli/info'
 require 'aspera/cli/version'
-require 'aspera/products/alpha'
+require 'aspera/products/desktop'
 require 'aspera/products/connect'
 require 'aspera/products/transferd'
 require 'aspera/products/other'
@@ -32,9 +32,6 @@ module Aspera
     # Installation.instance.ascp_path=""
     class Installation
       include Singleton
-      # protobuf generated files from sdk
-      EXT_RUBY_PROTOBUF = '_pb.rb'
-      RB_SDK_SUBFOLDER = 'lib'
       DEFAULT_ASPERA_CONF = <<~END_OF_CONFIG_FILE
         <?xml version='1.0' encoding='UTF-8'?>
         <CONF version="2">
@@ -50,37 +47,29 @@ module Aspera
       EXE_FILES = %i[ascp ascp4 async].freeze
       FILES = %i[transferd ssh_private_dsa ssh_private_rsa aspera_license aspera_conf fallback_certificate fallback_private_key].unshift(*EXE_FILES).freeze
       TRANSFER_SDK_LOCATION_URL = 'https://ibm.biz/sdk_location'
-      FILE_SCHEME_PREFIX = 'file:///'
-      SDK_ARCHIVE_FOLDERS = ['/bin/', '/aspera/'].freeze
       # filename for ascp with optional extension (Windows)
-      private_constant :EXT_RUBY_PROTOBUF, :RB_SDK_SUBFOLDER, :DEFAULT_ASPERA_CONF, :FILES, :TRANSFER_SDK_LOCATION_URL, :FILE_SCHEME_PREFIX
+      private_constant :DEFAULT_ASPERA_CONF, :FILES, :TRANSFER_SDK_LOCATION_URL
       # options for SSH client private key
       CLIENT_SSH_KEY_OPTIONS = %i{dsa_rsa rsa per_client}.freeze
 
-      class << self
-        def transfer_sdk_location_url
-          ENV.fetch('ASCLI_TRANSFER_SDK_LOCATION_URL', TRANSFER_SDK_LOCATION_URL)
-        end
-
-        # Loads YAML from cloud with locations of SDK archives for all platforms
-        # @return location structure
-        def sdk_locations
-          location_url = transfer_sdk_location_url
-          transferd_locations = UriReader.read(location_url)
-          Log.log.debug{"Retrieving SDK locations from #{location_url}"}
-          begin
-            return YAML.load(transferd_locations)
-          rescue Psych::SyntaxError
-            raise "Error when parsing yaml data from: #{location_url}"
-          end
+      # Loads YAML from cloud with locations of SDK archives for all platforms
+      # @return location structure
+      def sdk_locations
+        location_url = @transferd_urls
+        transferd_locations = UriReader.read(location_url)
+        Log.log.debug{"Retrieving SDK locations from #{location_url}"}
+        begin
+          return YAML.load(transferd_locations)
+        rescue Psych::SyntaxError
+          raise "Error when parsing yaml data from: #{location_url}"
         end
       end
 
       # set ascp executable path
       def ascp_path=(v)
         Aspera.assert_type(v, String)
-        Aspera.assert(!v.empty?) {'ascp path cannot be empty: check your config file'}
-        Aspera.assert(File.exist?(v)) {"No such file: [#{v}]"}
+        Aspera.assert(!v.empty?){'ascp path cannot be empty: check your config file'}
+        Aspera.assert(File.exist?(v)){"No such file: [#{v}]"}
         @path_to_ascp = v
       end
 
@@ -100,7 +89,7 @@ module Aspera
           pl = installed_products.first
           raise "no Aspera transfer module or SDK found.\nRefer to the manual or install SDK with command:\nascli conf ascp install" if pl.nil?
         else
-          pl = installed_products.find{|i|i[:name].eql?(product_name)}
+          pl = installed_products.find{ |i| i[:name].eql?(product_name)}
           raise "no such product installed: #{product_name}" if pl.nil?
         end
         self.ascp_path = pl[:ascp_path]
@@ -141,22 +130,21 @@ module Aspera
         when :ssh_private_dsa, :ssh_private_rsa
           # assume last 3 letters are type
           type = k.to_s[-3..-1].to_sym
-          file = check_or_create_sdk_file("aspera_bypass_#{type}.pem") {DataRepository.instance.item(type)}
+          file = check_or_create_sdk_file("aspera_bypass_#{type}.pem"){DataRepository.instance.item(type)}
         when :aspera_license
-          file = check_or_create_sdk_file('aspera-license') {DataRepository.instance.item(:license)}
+          file = check_or_create_sdk_file('aspera-license'){DataRepository.instance.item(:license)}
         when :aspera_conf
-          file = check_or_create_sdk_file('aspera.conf') {DEFAULT_ASPERA_CONF}
+          file = check_or_create_sdk_file('aspera.conf'){DEFAULT_ASPERA_CONF}
         when :fallback_certificate, :fallback_private_key
           file_key = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert_private_key.pem')
           file_cert = File.join(Products::Transferd.sdk_directory, 'aspera_fallback_cert.pem')
           if !File.exist?(file_key) || !File.exist?(file_cert)
             require 'openssl'
             # create new self signed certificate for http fallback
-            cert = OpenSSL::X509::Certificate.new
             private_key = OpenSSL::PKey::RSA.new(4096)
-            WebServerSimple.fill_self_signed_cert(cert, private_key)
-            check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true) {private_key.to_pem}
-            check_or_create_sdk_file('aspera_fallback_cert.pem', force: true) {cert.to_pem}
+            cert = WebServerSimple.self_signed_cert(private_key)
+            check_or_create_sdk_file('aspera_fallback_cert_private_key.pem', force: true){private_key.to_pem}
+            check_or_create_sdk_file('aspera_fallback_cert.pem', force: true){cert.to_pem}
           end
           file = k.eql?(:fallback_certificate) ? file_cert : file_key
         else Aspera.error_unexpected_value(k)
@@ -177,7 +165,7 @@ module Aspera
         Aspera.assert_values(types, CLIENT_SSH_KEY_OPTIONS)
         return case types
                when :dsa_rsa, :rsa
-                 types.to_s.split('_').map{|i|Installation.instance.path("ssh_private_#{i}".to_sym)}
+                 types.to_s.split('_').map{ |i| Installation.instance.path("ssh_private_#{i}".to_sym)}
                when :per_client
                  raise 'Not yet implemented'
                end
@@ -250,19 +238,18 @@ module Aspera
       def ascp_info
         ascp_data = file_paths
         ascp_data.merge!(ascp_pvcl_info)
-        ascp_data['sdk_locations'] = self.class.transfer_sdk_location_url
         ascp_data.merge!(ascp_ssl_info)
         return ascp_data
       end
 
       # @return the url for download of SDK archive for the given platform and version
       def sdk_url_for_platform(platform: nil, version: nil)
-        locations = self.class.sdk_locations
+        locations = sdk_locations
         platform = Environment.architecture if platform.nil?
-        locations = locations.select{|l|l['platform'].eql?(platform)}
+        locations = locations.select{ |l| l['platform'].eql?(platform)}
         raise "No SDK for platform: #{platform}" if locations.empty?
-        version = locations.max_by { |entry| Gem::Version.new(entry['version']) }['version'] if version.nil?
-        info = locations.select{|entry| entry['version'].eql?(version)}
+        version = locations.max_by{ |entry| Gem::Version.new(entry['version'])}['version'] if version.nil?
+        info = locations.select{ |entry| entry['version'].eql?(version)}
         raise "No such version: #{version} for #{platform}" if info.empty?
         return info.first['url']
       end
@@ -298,56 +285,47 @@ module Aspera
         end
       end
 
-      # download aspera SDK or use local file
-      # extracts ascp binary for current system architecture
+      # Retrieves ascp binary for current system architecture from URL or file
       # @param url      [String] URL to SDK archive, or SpecialValues::DEF
-      # @param folder   [String] destination
-      # @param backup   [Bool]
-      # @param with_exe [Bool]
-      # @param &block   [Proc] a lambda that receives a file path from archive and tells detination sub folder, or nil to not extract
+      # @param folder   [String] Destination folder path
+      # @param backup   [Bool]   If destination folder exists, then rename
+      # @param with_exe [Bool]   If false, only retrieves files, but do not generate or restrict access
+      # @param &block   [Proc]   A lambda that receives a file path from archive and tells detination sub folder(end with /) or file, or nil to not extract
       # @return ascp version (from execution)
       def install_sdk(url: nil, version: nil, folder: nil, backup: true, with_exe: true, &block)
         url = sdk_url_for_platform(version: version) if url.nil? || url.eql?('DEF')
         folder = Products::Transferd.sdk_directory if folder.nil?
         subfolder_lambda = block
         if subfolder_lambda.nil?
+          # default files to extract directly to main folder if in selected source folders
           subfolder_lambda = ->(name) do
-            if SDK_ARCHIVE_FOLDERS.any?{|i|name.include?(i)}
-              '/'
-            elsif name.end_with?(EXT_RUBY_PROTOBUF)
-              RB_SDK_SUBFOLDER
-            end
+            Products::Transferd::RUNTIME_FOLDERS.any?{ |i| name.match?(%r{^[^/]*/#{i}/})} ? '/' : nil
           end
         end
-        if url.start_with?('file:')
-          # require specific file scheme: the path part is "relative", or absolute if there are 4 slash
-          raise 'use format: file:///<path>' unless url.start_with?(FILE_SCHEME_PREFIX)
-          sdk_archive_path = url[FILE_SCHEME_PREFIX.length..-1]
-          delete_archive = false
-        else
-          sdk_archive_path = File.join(Dir.tmpdir, File.basename(url))
-          Aspera::Rest.new(base_url: url, redirect_max: 3).call(operation: 'GET', save_to_file: sdk_archive_path)
-          delete_archive = true
-        end
         # rename old install
         if backup && !Dir.empty?(folder)
           Log.log.warn('Previous install exists, renaming folder.')
           File.rename(folder, "#{folder}.#{Time.now.strftime('%Y%m%d%H%M%S')}")
           # TODO: delete old archives ?
         end
+        sdk_archive_path = UriReader.read_as_file(url)
         extract_archive_files(sdk_archive_path) do |entry_name, entry_stream, link_target|
-          subfolder = subfolder_lambda.call(entry_name)
-          next if subfolder.nil?
-          dest_folder = File.join(folder, subfolder)
+          dest_folder = subfolder_lambda.call(entry_name)
+          next if dest_folder.nil?
+          dest_folder = File.join(folder, dest_folder)
+          if dest_folder.end_with?('/')
+            dest_file = File.join(dest_folder, File.basename(entry_name))
+          else
+            dest_file = dest_folder
+            dest_folder = File.dirname(dest_file)
+          end
           FileUtils.mkdir_p(dest_folder)
-          new_file = File.join(dest_folder, File.basename(entry_name))
           if link_target.nil?
-            File.open(new_file, 'wb') { |output_stream|IO.copy_stream(entry_stream, output_stream)}
+            File.open(dest_file, 'wb'){ |output_stream| IO.copy_stream(entry_stream, output_stream)}
           else
-            File.symlink(link_target, new_file)
+            File.symlink(link_target, dest_file)
           end
         end
-        File.unlink(sdk_archive_path) rescue nil if delete_archive # Windows may give error
         return unless with_exe
         # ensure license file are generated so that ascp invocation for version works
         path(:aspera_license)
@@ -360,16 +338,18 @@ module Aspera
           Environment.restrict_file_access(exe_path, mode: 0o755) if File.exist?(exe_path)
         end
         sdk_ascp_version = get_ascp_version(sdk_ascp_path)
-        sdk_daemon_path = Products::Transferd.transferd_path
-        Log.log.warn{"No #{sdk_daemon_path} in SDK archive"} unless File.exist?(sdk_daemon_path)
-        Environment.restrict_file_access(sdk_daemon_path, mode: 0o755) if File.exist?(sdk_daemon_path)
-        transferd_version = get_exe_version(sdk_daemon_path, 'version')
+        transferd_exe_path = Products::Transferd.transferd_path
+        Log.log.warn{"No #{transferd_exe_path} in SDK archive"} unless File.exist?(transferd_exe_path)
+        Environment.restrict_file_access(transferd_exe_path, mode: 0o755) if File.exist?(transferd_exe_path)
+        transferd_version = get_exe_version(transferd_exe_path, 'version')
         sdk_name = 'IBM Aspera Transfer SDK'
         sdk_version = transferd_version || sdk_ascp_version
         File.write(File.join(folder, Products::Other::INFO_META_FILE), "<product><name>#{sdk_name}</name><version>#{sdk_version}</version></product>")
         return sdk_name, sdk_version
       end
 
+      attr_accessor :transferd_urls
+
       private
 
       # policy for product selection
@@ -379,6 +359,7 @@ module Aspera
         @path_to_ascp = nil
         @sdk_dir = nil
         @found_products = nil
+        @transferd_urls = TRANSFER_SDK_LOCATION_URL
       end
 
       public
@@ -392,7 +373,7 @@ module Aspera
           # :run_root  O only for Connect Client, location of http port file
           # :sub_bin   O subfolder with executables, default : bin
           scan_locations = Products::Transferd.locations.concat(
-            Products::Alpha.locations,
+            Products::Desktop.locations,
             Products::Connect.locations,
             Products::Other::LOCATION_ON_THIS_OS
           )

data/lib/aspera/ascp/management.rb

@@ -227,7 +227,7 @@ module Aspera
           # TODO: translate enhanced to capitalized ?
           data
             .keys
-            .map{|k|"#{k.capitalize}: #{data[k]}"}
+            .map{ |k| "#{k.capitalize}: #{data[k]}"}
             .unshift(MGT_HEADER)
             .push('', '')
             .join("\n")

data/lib/aspera/assert.rb

@@ -14,7 +14,7 @@ module Aspera
       message = 'assertion failed'
       info = yield if block_given?
       message = "#{message}: #{info}" if info
-      message = "#{message}: #{caller.find{|call|!call.start_with?(__FILE__)}}"
+      message = "#{message}: #{caller.find{ |call| !call.start_with?(__FILE__)}}"
       raise exception_class, message
     end
 
@@ -42,9 +42,18 @@ module Aspera
       raise InternalError, "unreachable line reached: #{caller(2..2).first}"
     end
 
-    # the value is not one of the expected values
+    # The value is not one of the expected values
+    # @param value the wrong value
+    # @param exception_class exception to raise
+    # @param block additional description in front
     def error_unexpected_value(value, exception_class: InternalError)
       raise exception_class, "#{block_given? ? "#{yield}: " : nil}unexpected value: #{value.inspect}"
     end
+
+    def require_method!(name)
+      define_method(name) do |*_args|
+        raise NotImplementedError, "#{self.class} must implement the #{name} method"
+      end
+    end
   end
 end

data/lib/aspera/cli/error.rb

@@ -4,11 +4,11 @@ module Aspera
   module Cli
     # CLI base exception
     class Error < StandardError; end
-
     # raised when an unexpected argument is provided
     class BadArgument < Error; end
+    class NoSuchElement < Error; end
 
-    class NoSuchIdentifier < Error
+    class BadIdentifier < Error
       def initialize(res_type, res_id)
         super("#{res_type} with identifier #{res_id} not found")
       end

data/lib/aspera/cli/extended_value.rb

@@ -21,6 +21,13 @@ module Aspera
       MARKER_END = ':'
       MARKER_IN_END = '@'
 
+      # special handlers stop processing of handlers on right
+      # extend includes processing of other handlers in itself
+      # val keeps the value intact
+      SPECIAL_HANDLERS = %i[extend val].freeze
+
+      private_constant :MARKER_START, :MARKER_END, :MARKER_IN_END, :SPECIAL_HANDLERS
+
       class << self
         # decode comma separated table text
         def decode_csvt(value)
@@ -49,25 +56,25 @@ module Aspera
         # base handlers
         # other handlers can be set using set_handler, e.g. `preset` is reader in config plugin
         @handlers = {
-          val:    lambda{|v|v},
-          base64: lambda{|v|Base64.decode64(v)},
-          csvt:   lambda{|v|ExtendedValue.decode_csvt(v)},
-          env:    lambda{|v|ENV.fetch(v, nil)},
-          file:   lambda{|v|File.read(File.expand_path(v))},
-          uri:    lambda{|v|UriReader.read(v)},
-          json:   lambda{|v|JSON.parse(v)},
-          lines:  lambda{|v|v.split("\n")},
-          list:   lambda{|v|v[1..-1].split(v[0])},
-          none:   lambda{|v|ExtendedValue.assert_no_value(v, :none); nil}, # rubocop:disable Style/Semicolon
-          path:   lambda{|v|File.expand_path(v)},
-          re:     lambda{|v|Regexp.new(v, Regexp::MULTILINE)},
-          ruby:   lambda{|v|Environment.secure_eval(v, __FILE__, __LINE__)},
-          secret: lambda{|v|prompt = v.empty? ? 'secret' : v; $stdin.getpass("#{prompt}> ")}, # rubocop:disable Style/Semicolon
-          stdin:  lambda{|v|ExtendedValue.assert_no_value(v, :stdin); $stdin.read}, # rubocop:disable Style/Semicolon
-          stdbin: lambda{|v|ExtendedValue.assert_no_value(v, :stdin); $stdin.binmode.read}, # rubocop:disable Style/Semicolon
-          yaml:   lambda{|v|YAML.load(v)},
-          zlib:   lambda{|v|Zlib::Inflate.inflate(v)},
-          extend: lambda{|v|ExtendedValue.instance.evaluate_all(v)}
+          val:    lambda{ |v| v},
+          base64: lambda{ |v| Base64.decode64(v)},
+          csvt:   lambda{ |v| ExtendedValue.decode_csvt(v)},
+          env:    lambda{ |v| ENV.fetch(v, nil)},
+          file:   lambda{ |v| File.read(File.expand_path(v))},
+          uri:    lambda{ |v| UriReader.read(v)},
+          json:   lambda{ |v| JSON_parse(v)},
+          lines:  lambda{ |v| v.split("\n")},
+          list:   lambda{ |v| v[1..-1].split(v[0])},
+          none:   lambda{ |v| ExtendedValue.assert_no_value(v, :none); nil}, # rubocop:disable Style/Semicolon
+          path:   lambda{ |v| File.expand_path(v)},
+          re:     lambda{ |v| Regexp.new(v, Regexp::MULTILINE)},
+          ruby:   lambda{ |v| Environment.secure_eval(v, __FILE__, __LINE__)},
+          secret: lambda{ |v| prompt = v.empty? ? 'secret' : v; $stdin.getpass("#{prompt}> ")}, # rubocop:disable Style/Semicolon
+          stdin:  lambda{ |v| ExtendedValue.assert_no_value(v, :stdin); $stdin.read}, # rubocop:disable Style/Semicolon
+          stdbin: lambda{ |v| ExtendedValue.assert_no_value(v, :stdbin); $stdin.binmode.read}, # rubocop:disable Style/Semicolon
+          yaml:   lambda{ |v| YAML.load(v)},
+          zlib:   lambda{ |v| Zlib::Inflate.inflate(v)},
+          extend: lambda{ |v| ExtendedValue.instance.evaluate_all(v)}
         }
         @default_decoder = nil
       end
@@ -77,6 +84,25 @@ module Aspera
         "#{MARKER_START}(#{modifiers.join('|')})#{MARKER_END}"
       end
 
+      # JSON Parser, with more information on error location
+      def JSON_parse(v)
+        JSON.parse(v)
+      rescue JSON::ParserError => e
+        m = /at line (\d+) column (\d+)/.match(e.message)
+        raise if m.nil?
+        line = m[1].to_i - 1
+        column = m[2].to_i - 1
+        lines = v.lines
+        raise if line >= lines.size
+        error_line = lines[line].chomp
+        context_col_beg = [column - 10, 0].max
+        context_col_end = [column + 10, error_line.length].min
+        context = error_line[context_col_beg...context_col_end]
+        cursor_pos = column - context_col_beg
+        pointer = ' ' * cursor_pos + '^'.blink
+        raise BadArgument, "#{e.message}\n#{context}\n#{pointer}"
+      end
+
       public
 
       def default_decoder=(value)
@@ -107,8 +133,7 @@ module Aspera
           handler = m[1].to_sym
           handlers_reversed.unshift(handler)
           value = m[2]
-          # stop processing if handler is extend (it will be processed later)
-          break if handler.eql?(:extend)
+          break if SPECIAL_HANDLERS.include?(handler)
         end
         Log.log.trace1{"evaluating: #{handlers_reversed}, value: #{value}"}
         handlers_reversed.each do |handler|