aspera-cli 4.20.0 → 4.21.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +41 -3
  4. data/CONTRIBUTING.md +69 -142
  5. data/README.md +687 -461
  6. data/bin/ascli +5 -14
  7. data/bin/asession +3 -5
  8. data/examples/get_proto_file.rb +4 -3
  9. data/examples/proxy.pac +20 -20
  10. data/lib/aspera/agent/base.rb +2 -0
  11. data/lib/aspera/agent/connect.rb +20 -2
  12. data/lib/aspera/agent/{alpha.rb → desktop.rb} +12 -18
  13. data/lib/aspera/agent/direct.rb +30 -31
  14. data/lib/aspera/agent/node.rb +1 -11
  15. data/lib/aspera/agent/{trsdk.rb → transferd.rb} +37 -51
  16. data/lib/aspera/api/alee.rb +1 -1
  17. data/lib/aspera/api/aoc.rb +13 -8
  18. data/lib/aspera/api/cos_node.rb +1 -1
  19. data/lib/aspera/api/node.rb +49 -32
  20. data/lib/aspera/ascp/installation.rb +98 -77
  21. data/lib/aspera/ascp/management.rb +27 -6
  22. data/lib/aspera/cli/extended_value.rb +9 -3
  23. data/lib/aspera/cli/formatter.rb +155 -154
  24. data/lib/aspera/cli/info.rb +2 -1
  25. data/lib/aspera/cli/main.rb +12 -0
  26. data/lib/aspera/cli/manager.rb +4 -4
  27. data/lib/aspera/cli/plugin.rb +2 -2
  28. data/lib/aspera/cli/plugins/aoc.rb +134 -73
  29. data/lib/aspera/cli/plugins/config.rb +114 -83
  30. data/lib/aspera/cli/plugins/cos.rb +1 -0
  31. data/lib/aspera/cli/plugins/faspex.rb +4 -2
  32. data/lib/aspera/cli/plugins/faspex5.rb +29 -14
  33. data/lib/aspera/cli/plugins/node.rb +51 -41
  34. data/lib/aspera/cli/transfer_progress.rb +2 -0
  35. data/lib/aspera/cli/version.rb +1 -1
  36. data/lib/aspera/command_line_builder.rb +1 -1
  37. data/lib/aspera/coverage.rb +5 -3
  38. data/lib/aspera/environment.rb +59 -16
  39. data/lib/aspera/faspex_postproc.rb +3 -5
  40. data/lib/aspera/hash_ext.rb +2 -12
  41. data/lib/aspera/node_simulator.rb +230 -112
  42. data/lib/aspera/oauth/base.rb +40 -48
  43. data/lib/aspera/oauth/factory.rb +41 -2
  44. data/lib/aspera/oauth/jwt.rb +4 -1
  45. data/lib/aspera/persistency_action_once.rb +1 -1
  46. data/lib/aspera/persistency_folder.rb +20 -2
  47. data/lib/aspera/preview/generator.rb +13 -10
  48. data/lib/aspera/preview/options.rb +2 -2
  49. data/lib/aspera/preview/terminal.rb +1 -1
  50. data/lib/aspera/preview/utils.rb +11 -6
  51. data/lib/aspera/products/connect.rb +82 -0
  52. data/lib/aspera/products/desktop.rb +30 -0
  53. data/lib/aspera/products/other.rb +82 -0
  54. data/lib/aspera/products/transferd.rb +61 -0
  55. data/lib/aspera/rest.rb +22 -17
  56. data/lib/aspera/secret_hider.rb +9 -2
  57. data/lib/aspera/ssh.rb +31 -24
  58. data/lib/aspera/temp_file_manager.rb +5 -4
  59. data/lib/aspera/transfer/parameters.rb +2 -1
  60. data/lib/aspera/transfer/spec.yaml +22 -20
  61. data/lib/aspera/transfer/sync.rb +1 -5
  62. data/lib/aspera/transfer/uri.rb +2 -2
  63. data/lib/aspera/uri_reader.rb +18 -1
  64. data/lib/transferd_pb.rb +86 -0
  65. data/lib/transferd_services_pb.rb +84 -0
  66. data.tar.gz.sig +0 -0
  67. metadata +13 -166
  68. metadata.gz.sig +0 -0
  69. data/examples/build_exec +0 -74
  70. data/examples/build_exec_rubyc +0 -40
  71. data/lib/aspera/ascp/products.rb +0 -168
  72. data/lib/transfer_pb.rb +0 -84
  73. data/lib/transfer_services_pb.rb +0 -82
@@ -66,14 +66,14 @@ module Aspera
66
66
  base_url = "#{base_url}.#{Api::AoC::SAAS_DOMAIN_PROD}" unless base_url.include?('.')
67
67
  # AoC is only https
68
68
  return nil unless base_url.start_with?('https://')
69
- res_http = Rest.new(base_url: base_url, redirect_max: 10).call(operation: 'GET')[:http]
70
- # Any AoC is on this domain
71
- return nil unless res_http.uri.host.end_with?(Api::AoC::SAAS_DOMAIN_PROD)
72
- Log.log.debug{"AoC Main page: #{res_http.body.include?(Api::AoC::PRODUCT_NAME)}"}
73
- base_url = res_http.uri.to_s if res_http.uri.path.include?('/public')
69
+ res_http = Rest.new(base_url: base_url, redirect_max: 0).call(operation: 'GET', subpath: 'auth/ping', return_error: true)[:http]
70
+ return nil if res_http['Location'].nil?
71
+ redirect_uri = URI.parse(res_http['Location'])
72
+ od = Api::AoC.split_org_domain(URI.parse(base_url))
73
+ return nil unless redirect_uri.path.end_with?("oauth2/#{od[:organization]}/login")
74
74
  # either in standard domain, or product name in page
75
75
  return {
76
- version: 'SaaS',
76
+ version: Api::AoC.saas_url?(base_url) ? 'SaaS' : 'Self-managed',
77
77
  url: base_url
78
78
  }
79
79
  end
@@ -92,8 +92,6 @@ module Aspera
92
92
  # set vars to look like object
93
93
  options = object.options
94
94
  formatter = object.formatter
95
- options.declare(:use_generic_client, 'Wizard: AoC: use global or org specific jwt client id', values: :bool, default: true)
96
- options.parse_options!
97
95
  instance_url = options.get_option(:url, mandatory: true)
98
96
  pub_link_info = Api::AoC.link_info(instance_url)
99
97
  if !pub_link_info[:token].nil?
@@ -108,6 +106,8 @@ module Aspera
108
106
  test_args: 'organization'
109
107
  }
110
108
  end
109
+ options.declare(:use_generic_client, 'Wizard: AoC: use global or org specific jwt client id', values: :bool, default: Api::AoC.saas_url?(instance_url))
110
+ options.parse_options!
111
111
  # make username mandatory for jwt, this triggers interactive input
112
112
  wiz_username = options.get_option(:username, mandatory: true)
113
113
  raise "Username shall be an email in AoC: #{wiz_username}" if !(wiz_username =~ /\A[\w+\-.]+@[a-z\d\-.]+\.[a-z]+\z/i)
@@ -133,15 +133,15 @@ module Aspera
133
133
  formatter.display_status('Please login to your Aspera on Cloud instance.'.red)
134
134
  formatter.display_status('Navigate to: 𓃑 → Admin → Integrations → API Clients')
135
135
  formatter.display_status('Check or create in integration:')
136
- formatter.display_status("- name: #{@info[:name]}")
136
+ formatter.display_status('- name: cli')
137
137
  formatter.display_status("- redirect uri: #{REDIRECT_LOCALHOST}")
138
138
  formatter.display_status('- origin: localhost')
139
139
  formatter.display_status('Use the generated client id and secret in the following prompts.'.red)
140
140
  end
141
- Environment.instance.open_uri("#{instance_url}/admin/api-clients")
141
+ Environment.instance.open_uri("#{instance_url}/admin/integrations/api-clients")
142
142
  options.get_option(:client_id, mandatory: true)
143
143
  options.get_option(:client_secret, mandatory: true)
144
- use_browser_authentication = true
144
+ # use_browser_authentication = true
145
145
  end
146
146
  if use_browser_authentication
147
147
  formatter.display_status('We will use web authentication to bootstrap.')
@@ -282,13 +282,17 @@ module Aspera
282
282
  end
283
283
 
284
284
  # list all entities, given additional, default and user's queries
285
+ # @param resource_class_path path to query on API
286
+ # @param fields fields to display
287
+ # @param base_query a query applied always
288
+ # @param default_query default query unless overriden by user
285
289
  def result_list(resource_class_path, fields: nil, base_query: {}, default_query: {})
286
290
  Aspera.assert_type(base_query, Hash)
287
291
  Aspera.assert_type(default_query, Hash)
288
292
  user_query = query_read_delete(default: default_query)
289
293
  # caller may add specific modifications or checks
290
294
  yield(user_query) if block_given?
291
- return {type: :object_list, fields: fields}.merge(api_read_all(resource_class_path, base_query.merge(user_query)))
295
+ return {type: :object_list, fields: fields}.merge(api_read_all(resource_class_path, base_query.merge(user_query).compact))
292
296
  end
293
297
 
294
298
  def resolve_dropbox_name_default_ws_id(query)
@@ -344,7 +348,7 @@ module Aspera
344
348
  transfer.agent_instance = Agent::Node.new(
345
349
  url: client_apfid[:api].base_url,
346
350
  username: client_apfid[:api].app_info[:node_info]['access_key'],
347
- password: client_apfid[:api].oauth.token,
351
+ password: client_apfid[:api].oauth.authorization,
348
352
  root_id: client_apfid[:file_id]
349
353
  )
350
354
  # additional node to node TS info
@@ -411,8 +415,8 @@ module Aspera
411
415
  when :operation then default_fields = nil
412
416
  when :short_link then default_fields.push('short_url', 'data.url_token_data.purpose')
413
417
  when :user then default_fields.push('name', 'email')
414
- when :group_membership then default_fields.push(*%w[group_id member_type member_id])
415
- when :workspace_membership then default_fields.push(*%w[workspace_id member_type member_id])
418
+ when :group_membership then default_fields.push('group_id', 'member_type', 'member_id')
419
+ when :workspace_membership then default_fields.push('workspace_id', 'member_type', 'member_id')
416
420
  end
417
421
  return result_list(resource_class_path, fields: default_fields, default_query: default_query)
418
422
  when :show
@@ -469,60 +473,111 @@ module Aspera
469
473
  when :subscription
470
474
  org = aoc_api.read('organization')
471
475
  bss_graphql = api_from_options('bss/platform/graphql')
472
- # cspell:disable
473
- graphql_query = <<-GRAPHQL
474
- query ($organization_id: ID!) {
475
- aoc (organization_id: $organization_id) {
476
- bssSubscription {
477
- aocVersion
478
- endDate
479
- startDate
480
- termMonths
481
- plan
482
- trial
483
- termType
484
- aocOrganizations {
485
- id
486
- }
487
- additionalStorageVolumeGb
488
- additionalEgressVolumeGb
489
- term {
490
- startDate
491
- endDate
492
- transferVolumeGb
493
- egressVolumeGb
494
- storageVolumeGb
495
- transferVolumeOffsetGb
496
- }
497
- paygoRate {
498
- transferRate
499
- storageRate
500
- currency
501
- }
502
- aocPlanData {
503
- tier
504
- trial
505
- workspaces { max }
506
- users {
507
- planAmount
508
- max
476
+ command_subscription = options.get_next_command(%i[account usage])
477
+ case command_subscription
478
+ when :account
479
+ # cspell:disable
480
+ graphql_query = <<-GRAPHQL
481
+ query ($organization_id: ID!) {
482
+ aoc (organization_id: $organization_id) {
483
+ bssSubscription {
484
+ aocVersion
485
+ endDate
486
+ startDate
487
+ termMonths
488
+ plan
489
+ trial
490
+ termType
491
+ aocOrganizations {
492
+ id
493
+ }
494
+ additionalStorageVolumeGb
495
+ additionalEgressVolumeGb
496
+ term {
497
+ startDate
498
+ endDate
499
+ transferVolumeGb
500
+ egressVolumeGb
501
+ storageVolumeGb
502
+ transferVolumeOffsetGb
503
+ }
504
+ paygoRate {
505
+ transferRate
506
+ storageRate
507
+ currency
508
+ }
509
+ aocPlanData {
510
+ tier
511
+ trial
512
+ workspaces { max }
513
+ users {
514
+ planAmount
515
+ max
516
+ }
517
+ samlIntegration
518
+ activity
519
+ sharedInboxes
520
+ uniqueUrls
521
+ support
522
+ watermarking
523
+ byok
524
+ automation { planAmount, max }
525
+ }
526
+ }
527
+ }
509
528
  }
510
- samlIntegration
511
- activity
512
- sharedInboxes
513
- uniqueUrls
514
- support
515
- watermarking
516
- byok
517
- automation { planAmount, max }
518
- }
519
- }
520
- }
521
- }
522
- GRAPHQL
523
- # cspell:enable
524
- result = bss_graphql.create(nil, {query: graphql_query, variables: {organization_id: org['id']}})['data']
525
- return {type: :single_object, data: result['aoc']['bssSubscription']}
529
+ GRAPHQL
530
+ # cspell:enable
531
+ result = bss_graphql.create(nil, {query: graphql_query, variables: {organization_id: org['id']}})['data']
532
+ return {type: :single_object, data: result['aoc']['bssSubscription']}
533
+ when :usage
534
+ # cspell:disable
535
+ graphql_query = <<-GRAPHQL
536
+ query ($organization_id: ID!, $startDate: Date!, $endDate: Date!, $aggregate: TransferUsageAggregateOption!) {
537
+ aoc (organization_id: $organization_id) {
538
+ bssSubscription {
539
+ aocOrganizations { id }
540
+ additionalStorageVolumeGb
541
+ additionalEgressVolumeGb
542
+ aocPlanData {
543
+ tier
544
+ trial
545
+ }
546
+ term {
547
+ transferVolumeGb
548
+ egressVolumeGb
549
+ storageVolumeGb
550
+ startDate
551
+ endDate
552
+ transferVolumeOffsetGb
553
+ }
554
+ termMonths
555
+ transferUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { mbTotal }
556
+ egressUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { usageMb }
557
+ }
558
+ subscriptionEntitlements {
559
+ id
560
+ transferUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { mbTotal }
561
+ egressUsages (startDate: $startDate, endDate: $endDate, aggregate: $aggregate) { usageMb }
562
+ }
563
+ }
564
+ }
565
+ GRAPHQL
566
+ aggregate = options.get_next_argument('aggregation', accept_list: %i[ALL MONTHLY], default: :ALL)
567
+ today = Date.today
568
+ start_date = options.get_next_argument('start date', mandatory: false, default: today.prev_year.strftime('%Y-%m-%d'))
569
+ end_date = options.get_next_argument('end date', mandatory: false, default: today.strftime('%Y-%m-%d'))
570
+ # cspell:enable
571
+ result = bss_graphql.create(
572
+ nil,
573
+ {query: graphql_query,
574
+ variables: {
575
+ organization_id: org['id'],
576
+ aggregate: aggregate,
577
+ startDate: start_date,
578
+ endDate: end_date}})['data']
579
+ return {type: :single_object, data: result['aoc']}
580
+ end
526
581
  when :ats
527
582
  ats_api = Rest.new(**aoc_api.params.deep_merge({
528
583
  base_url: "#{aoc_api.base_url}/admin/ats/pub/v1",
@@ -686,7 +741,7 @@ module Aspera
686
741
  when :servers
687
742
  return {type: :object_list, data: Rest.new(base_url: "#{Api::AoC.api_base_url}/#{Api::AoC::API_V1}").read('servers')}
688
743
  when :bearer_token
689
- return {type: :text, data: aoc_api.oauth.token}
744
+ return {type: :text, data: aoc_api.oauth.authorization}
690
745
  when :organization
691
746
  return { type: :single_object, data: aoc_api.read('organization') }
692
747
  when :tier_restrictions
@@ -801,6 +856,12 @@ module Aspera
801
856
  else
802
857
  ids_to_download = [ids_to_download] unless ids_to_download.is_a?(Array)
803
858
  end
859
+ file_list =
860
+ begin
861
+ transfer.source_list.map{|i|{'source'=>i}}
862
+ rescue Cli::BadArgument
863
+ [{'source' => '.'}]
864
+ end
804
865
  # list here
805
866
  result_transfer = []
806
867
  formatter.display_status("found #{ids_to_download.length} package(s).")
@@ -816,7 +877,7 @@ module Aspera
816
877
  package_node_api.transfer_spec_gen4(
817
878
  package_info['contents_file_id'],
818
879
  Transfer::Spec::DIRECTION_RECEIVE,
819
- {'paths'=> [{'source' => '.'}]}),
880
+ {'paths'=> file_list}),
820
881
  rest_token: package_node_api)
821
882
  result_transfer.push({'package' => package_id, Main::STATUS_FIELD => statuses})
822
883
  # update skip list only if all transfer sessions completed
@@ -837,14 +898,14 @@ module Aspera
837
898
  resolve_dropbox_name_default_ws_id(query)
838
899
  end
839
900
  when :delete
840
- return do_bulk_operation(command: package_command, descr: 'identifier', values: identifier) do |id|
901
+ return do_bulk_operation(command: package_command, descr: 'identifier', values: instance_identifier) do |id|
841
902
  Aspera.assert_values(id.class, [String, Integer]){'identifier'}
842
903
  aoc_api.delete("packages/#{id}")
843
904
  end
844
905
  when *Node::NODE4_READ_ACTIONS
845
906
  package_id = instance_identifier
846
907
  package_info = aoc_api.read("packages/#{package_id}")
847
- return execute_nodegen4_command(package_command, package_info['node_id'], file_id: package_info['file_id'], scope: Api::Node::SCOPE_USER)
908
+ return execute_nodegen4_command(package_command, package_info['node_id'], file_id: package_info['contents_file_id'], scope: Api::Node::SCOPE_USER)
848
909
  end
849
910
  when :files
850
911
  command_repo = options.get_next_command([:short_link].concat(NODE4_EXT_COMMANDS))
@@ -891,7 +952,7 @@ module Aspera
891
952
  when :automation
892
953
  Log.log.warn('BETA: work under progress')
893
954
  # automation api is not in the same place
894
- automation_api = Rest.new(**aoc_api.params.merge(base_url: aoc_api.base_url.gsub('/api/', '/automation/')))
955
+ automation_api = Rest.new(**aoc_api.params, base_url: aoc_api.base_url.gsub('/api/', '/automation/'))
895
956
  command_automation = options.get_next_command(%i[workflows instances])
896
957
  case command_automation
897
958
  when :instances