eco-helpers 3.0.37 → 3.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -0
- data/CHANGELOG.md +132 -1
- data/lib/eco/api/common/loaders/base.rb +2 -2
- data/lib/eco/api/common/loaders/case_base.rb +2 -0
- data/lib/eco/api/common/loaders/config/block.rb +78 -0
- data/lib/eco/api/common/loaders/config/workflow/mailer.rb +49 -6
- data/lib/eco/api/common/loaders/config.rb +3 -26
- data/lib/eco/api/common/loaders/error_handler.rb +2 -0
- data/lib/eco/api/common/loaders/parser.rb +1 -4
- data/lib/eco/api/common/people/entries.rb +23 -6
- data/lib/eco/api/common/people/entry_factory.rb +1 -1
- data/lib/eco/api/common/people/person_entry.rb +104 -27
- data/lib/eco/api/common/people/person_parser.rb +50 -16
- data/lib/eco/api/common/people/supervisor_helpers.rb +12 -6
- data/lib/eco/api/common/session/base_session.rb +75 -81
- data/lib/eco/api/common/session/environment.rb +70 -70
- data/lib/eco/api/common/session/file_manager.rb +132 -135
- data/lib/eco/api/common/session/helpers/prompt_user.rb +23 -30
- data/lib/eco/api/common/session/helpers.rb +10 -15
- data/lib/eco/api/common/session/logger/cache.rb +94 -96
- data/lib/eco/api/common/session/logger/channels.rb +24 -32
- data/lib/eco/api/common/session/logger/log.rb +38 -46
- data/lib/eco/api/common/session/logger.rb +50 -54
- data/lib/eco/api/common/session/mailer/aws_provider.rb +64 -71
- data/lib/eco/api/common/session/mailer/provider_base.rb +43 -48
- data/lib/eco/api/common/session/mailer/sendgrid_provider.rb +101 -109
- data/lib/eco/api/common/session/mailer.rb +78 -83
- data/lib/eco/api/common/session/s3_uploader.rb +132 -138
- data/lib/eco/api/common/session/sftp.rb +202 -208
- data/lib/eco/api/common.rb +0 -3
- data/lib/eco/api/custom/mailer.rb +4 -2
- data/lib/eco/api/error/handlers.rb +1 -1
- data/lib/eco/api/microcases/people/apply_changes/set_core/core_excluded.rb +8 -2
- data/lib/eco/api/microcases/people/manage/search.rb +2 -2
- data/lib/eco/api/organization/people/similarity.rb +3 -3
- data/lib/eco/api/organization/people.rb +2 -2
- data/lib/eco/api/session/batch/base_policy.rb +42 -27
- data/lib/eco/api/session/batch/launcher/mode_size.rb +6 -1
- data/lib/eco/api/session/batch/launcher.rb +16 -3
- data/lib/eco/api/session/config/api.rb +4 -3
- data/lib/eco/api/session/config/apis/one_off.rb +1 -1
- data/lib/eco/api/session/config/files.rb +13 -12
- data/lib/eco/api/session/config/workflow.rb +1 -373
- data/lib/eco/api/session/config.rb +45 -10
- data/lib/eco/api/session.rb +7 -9
- data/lib/eco/api/usecases/base_case/model.rb +6 -6
- data/lib/eco/api/usecases/base_case.rb +1 -1
- data/lib/eco/api/usecases/cli.rb +1 -1
- data/lib/eco/api/usecases/default/locations/tagtree_extract_case.rb +22 -13
- data/lib/eco/api/usecases/default_cases/to_csv_case.rb +4 -1
- data/lib/eco/api/usecases/graphql/helpers/location/tags_remap.rb +6 -3
- data/lib/eco/api/usecases/graphql/samples/location/command/dsl.rb +2 -2
- data/lib/eco/api/usecases/lib/base/env.rb +21 -23
- data/lib/eco/api/usecases/lib/files/file_pattern.rb +41 -14
- data/lib/eco/api/usecases/lib/files/input_file.rb +110 -0
- data/lib/eco/api/usecases/lib/files/sftp.rb +5 -2
- data/lib/eco/api/usecases/lib/files.rb +1 -0
- data/lib/eco/api/usecases/lib/locations/base.rb +23 -0
- data/lib/eco/api/usecases/lib/locations/mapping.rb +94 -0
- data/lib/eco/api/usecases/lib/locations.rb +7 -0
- data/lib/eco/api/usecases/lib/people/base.rb +20 -0
- data/lib/eco/api/usecases/lib/people.rb +6 -0
- data/lib/eco/api/usecases/lib.rb +2 -0
- data/lib/eco/api/usecases/ooze_cases.rb +1 -1
- data/lib/eco/api/usecases/ooze_samples/ooze_base_case.rb +1 -1
- data/lib/eco/api/usecases/ooze_samples/ooze_from_doc_case.rb +3 -3
- data/lib/eco/api/usecases/ooze_samples/ooze_run_base_case.rb +4 -1
- data/lib/eco/api/usecases/ooze_samples/ooze_update_case.rb +1 -1
- data/lib/eco/api/usecases/ooze_samples/register_export_case.rb +6 -2
- data/lib/eco/api/usecases/ooze_samples/register_migration_case.rb +7 -6
- data/lib/eco/api/usecases/ooze_samples/register_update_case.rb +14 -4
- data/lib/eco/api/usecases/ooze_samples/target_oozes_update_case.rb +11 -8
- data/lib/eco/api/usecases/samples/drivers/sftp_sample.rb +2 -0
- data/lib/eco/api/usecases/samples/drivers/url_pull_sample.rb +8 -2
- data/lib/eco/api/usecases/service/sftp/with_target_config.rb +0 -33
- data/lib/eco/api/usecases/service/sftp.rb +7 -1
- data/lib/eco/api/usecases/use_case.rb +3 -2
- data/lib/eco/api/usecases/workflow.rb +5 -0
- data/lib/eco/api/usecases.rb +12 -5
- data/lib/eco/cli/scripting/args_helpers.rb +1 -9
- data/lib/eco/cli_default/options.rb +98 -68
- data/lib/eco/cli_default/workflow/end.rb +22 -0
- data/lib/eco/cli_default/workflow/launch_jobs.rb +14 -0
- data/lib/eco/cli_default/workflow/load/data.rb +27 -0
- data/lib/eco/cli_default/workflow/load/input.rb +28 -0
- data/lib/eco/cli_default/workflow/load.rb +13 -0
- data/lib/eco/cli_default/workflow/options.rb +18 -0
- data/lib/eco/cli_default/workflow/post_launch.rb +65 -0
- data/lib/eco/cli_default/workflow/report.rb +17 -0
- data/lib/eco/cli_default/workflow/rescued_exception.rb +21 -0
- data/lib/eco/cli_default/workflow/usecases.rb +23 -0
- data/lib/eco/cli_default/workflow.rb +24 -180
- data/lib/eco/data/count_trace.rb +51 -0
- data/lib/eco/data/files/content.rb +39 -0
- data/lib/eco/data/files/directory.rb +78 -45
- data/lib/eco/data/files/encoding.rb +12 -21
- data/lib/eco/data/files/file_pattern.rb +15 -8
- data/lib/eco/data/files/folder.rb +196 -0
- data/lib/eco/data/files/relative_path.rb +54 -0
- data/lib/eco/data/files/timestamp.rb +18 -0
- data/lib/eco/data/files.rb +46 -5
- data/lib/eco/data/fuzzy_match.rb +1 -1
- data/lib/eco/data/hashes/array_diff.rb +11 -5
- data/lib/eco/data/hashes/diff_result/meta.rb +12 -4
- data/lib/eco/data/locations/node_diff/accessors.rb +1 -1
- data/lib/eco/data/mapper.rb +8 -1
- data/lib/eco/data.rb +1 -0
- data/lib/eco/language/auxiliar_logger.rb +6 -11
- data/lib/eco/language/delegation/delegating_missing.rb +1 -1
- data/lib/eco/language/delegation/delegating_missing_const.rb +1 -1
- data/lib/eco/language/delegation/delegating_missing_on_class.rb +1 -1
- data/lib/eco/language/delegation/for_delegator/delegated_class.rb +1 -1
- data/lib/eco/language/klass/auto_loader.rb +129 -0
- data/lib/eco/language/klass/builder.rb +6 -6
- data/lib/eco/language/klass/const.rb +19 -0
- data/lib/eco/language/klass/helpers_built.rb +3 -1
- data/lib/eco/language/klass/hierarchy.rb +5 -1
- data/lib/eco/language/klass/naming.rb +5 -2
- data/lib/eco/language/klass/resolver.rb +21 -6
- data/lib/eco/language/klass/uid.rb +12 -0
- data/lib/eco/language/klass/when_inherited.rb +30 -6
- data/lib/eco/language/klass.rb +5 -2
- data/lib/eco/language/methods/access_modifier.rb +23 -0
- data/lib/eco/language/methods/instance_method_helpers.rb +6 -1
- data/lib/eco/language/methods.rb +1 -0
- data/lib/eco/language/models/hierarchy.rb +41 -0
- data/lib/eco/language/models/workflow.rb +385 -0
- data/lib/eco/language/models.rb +2 -1
- data/lib/eco/version.rb +1 -1
- metadata +31 -7
- data/lib/eco/api/common/class_auto_loader.rb +0 -114
- data/lib/eco/api/common/class_helpers.rb +0 -9
- data/lib/eco/api/common/class_hierarchy.rb +0 -45
- data/lib/eco/data/files/helpers.rb +0 -152
- data/lib/eco/language/models/class_helpers.rb +0 -136
@@ -1,143 +1,137 @@
|
|
1
|
-
module Eco
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
File.open(file, "rb") do |f|
|
59
|
-
upload(key, f)
|
60
|
-
end
|
61
|
-
end.compact
|
62
|
-
end
|
63
|
-
|
64
|
-
# @param path [String] a full path to a S3 object
|
65
|
-
# @return [String] `link` to the S3 object on console
|
66
|
-
def link(path)
|
67
|
-
return path.map { |pth| link(pth) } if path.is_a?(Enumerable)
|
68
|
-
return unless path.is_a?(String)
|
69
|
-
|
70
|
-
s3_path_str = "https://s3.console.aws.amazon.com/s3/object"
|
71
|
-
s3_params = "region=#{fetch_region}&tab=overview"
|
72
|
-
"#{s3_path_str}/#{path.sub('s3://', '')}?#{s3_params}"
|
73
|
-
end
|
74
|
-
|
75
|
-
private
|
76
|
-
|
77
|
-
def full_path(obj)
|
78
|
-
"s3://#{bucket.name}/#{obj.key}" if obj
|
79
|
-
end
|
80
|
-
|
81
|
-
def bucket
|
82
|
-
require 'aws-sdk-s3'
|
83
|
-
begin
|
84
|
-
@bucket ||= Aws::S3::Resource.new(
|
85
|
-
access_key_id: fetch_access_key_id,
|
86
|
-
secret_access_key: fetch_secret_access_key,
|
87
|
-
region: fetch_region
|
88
|
-
).bucket(fetch_bucket)
|
89
|
-
rescue StandardError => err
|
90
|
-
log(:error) {
|
91
|
-
"Trying to upload to S3 with wrong configuration: #{err}"
|
92
|
-
}
|
93
|
-
end
|
94
|
-
@bucket
|
95
|
-
end
|
96
|
-
|
97
|
-
def session_path
|
98
|
-
["scripts", @prefix, @timestamp].join("/")
|
99
|
-
end
|
100
|
-
|
101
|
-
def new_s3_object(filename)
|
102
|
-
file_path = [session_path, filename].join("/")
|
103
|
-
bucket&.object(file_path)
|
104
|
-
end
|
105
|
-
|
106
|
-
def log_upload(obj)
|
107
|
-
yield
|
108
|
-
|
109
|
-
log(:debug) { "Uploaded #{full_path(obj)}" }
|
110
|
-
end
|
111
|
-
|
112
|
-
def logger
|
113
|
-
@enviro&.logger || super
|
114
|
-
end
|
115
|
-
|
116
|
-
def config
|
117
|
-
@enviro.config || {}
|
118
|
-
end
|
119
|
-
|
120
|
-
def fetch_bucket
|
121
|
-
config.s3storage.bucket_name
|
122
|
-
end
|
123
|
-
|
124
|
-
def fetch_prefix
|
125
|
-
config.s3storage.prefix
|
126
|
-
end
|
127
|
-
|
128
|
-
def fetch_access_key_id
|
129
|
-
config.s3storage.access_key_id || ENV['AWS_ACCESS_KEY_ID']
|
130
|
-
end
|
131
|
-
|
132
|
-
def fetch_secret_access_key
|
133
|
-
config.s3storage.secret_access_key || ENV['AWS_SECRET_ACCESS_KEY']
|
134
|
-
end
|
135
|
-
|
136
|
-
def fetch_region
|
137
|
-
config.s3storage.region || ENV['AWS_REGION']
|
138
|
-
end
|
1
|
+
module Eco::API::Common::Session
|
2
|
+
class S3Uploader
|
3
|
+
include Eco::Language::AuxiliarLogger
|
4
|
+
|
5
|
+
attr_reader :prefix
|
6
|
+
|
7
|
+
# @param enviro [Eco::API::Common::Session::Environment]
|
8
|
+
def initialize(enviro:)
|
9
|
+
msg = "Required Environment object (enviro:). Given: #{enviro.class}"
|
10
|
+
raise msg if enviro && !enviro.is_a?(Eco::API::Common::Session::Environment)
|
11
|
+
|
12
|
+
@enviro = enviro
|
13
|
+
@prefix = fetch_prefix
|
14
|
+
@timestamp = Time.now.iso8601
|
15
|
+
end
|
16
|
+
|
17
|
+
# Uploads `content` to S3 as `filename`
|
18
|
+
# @param filename [String] the name of the object to be created on S3
|
19
|
+
# @param content [String] that to be uploaded
|
20
|
+
# @return [String] S3 path to the uploaded `filename` object
|
21
|
+
def upload(filename, content)
|
22
|
+
if (obj = new_s3_object(filename))
|
23
|
+
log_upload(obj) do
|
24
|
+
obj.put(body: content)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
full_path(obj)
|
29
|
+
end
|
30
|
+
|
31
|
+
# Uploads a single file
|
32
|
+
# @param path [String] the target file to be uploaded
|
33
|
+
# @return [String] S3 path to the uploaded `path` file
|
34
|
+
def upload_file(path)
|
35
|
+
return unless File.exist?(path)
|
36
|
+
|
37
|
+
File.open(path, "rb") do |f|
|
38
|
+
upload(File.basename(path), f)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
# @note it will skip subfolders
|
43
|
+
# @param path [String] the target directory to be uploaded
|
44
|
+
# @param recurse [Boolean] deepen in the folder structure? (`false`: default)
|
45
|
+
# @return [Array<String>] S3 paths to all the uploaded files of `path` directory
|
46
|
+
def upload_directory(path, recurse: false)
|
47
|
+
path = File.expand_path(path)
|
48
|
+
prefix = File.expand_path(File.join(path, ".."))
|
49
|
+
wildcard = recurse ? "**/*" : "*"
|
50
|
+
|
51
|
+
Dir.glob(File.join(path, wildcard)).sort.map do |file|
|
52
|
+
next unless File.file?(file) # Skip directories
|
53
|
+
key = file.sub(prefix, "").gsub("\\", "/").sub(/^\/+/, "")
|
54
|
+
|
55
|
+
File.open(file, "rb") do |f|
|
56
|
+
upload(key, f)
|
139
57
|
end
|
58
|
+
end.compact
|
59
|
+
end
|
60
|
+
|
61
|
+
# @param path [String] a full path to a S3 object
|
62
|
+
# @return [String] `link` to the S3 object on console
|
63
|
+
def link(path)
|
64
|
+
return path.map { |pth| link(pth) } if path.is_a?(Enumerable)
|
65
|
+
return unless path.is_a?(String)
|
66
|
+
|
67
|
+
s3_path_str = "https://s3.console.aws.amazon.com/s3/object"
|
68
|
+
s3_params = "region=#{fetch_region}&tab=overview"
|
69
|
+
"#{s3_path_str}/#{path.sub('s3://', '')}?#{s3_params}"
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
|
74
|
+
def full_path(obj)
|
75
|
+
"s3://#{bucket.name}/#{obj.key}" if obj
|
76
|
+
end
|
77
|
+
|
78
|
+
def bucket
|
79
|
+
require 'aws-sdk-s3'
|
80
|
+
begin
|
81
|
+
@bucket ||= Aws::S3::Resource.new(
|
82
|
+
access_key_id: fetch_access_key_id,
|
83
|
+
secret_access_key: fetch_secret_access_key,
|
84
|
+
region: fetch_region
|
85
|
+
).bucket(fetch_bucket)
|
86
|
+
rescue StandardError => err
|
87
|
+
log(:error) {
|
88
|
+
"Trying to upload to S3 with wrong configuration: #{err}"
|
89
|
+
}
|
140
90
|
end
|
91
|
+
@bucket
|
92
|
+
end
|
93
|
+
|
94
|
+
def session_path
|
95
|
+
["scripts", @prefix, @timestamp].join("/")
|
96
|
+
end
|
97
|
+
|
98
|
+
def new_s3_object(filename)
|
99
|
+
file_path = [session_path, filename].join("/")
|
100
|
+
bucket&.object(file_path)
|
101
|
+
end
|
102
|
+
|
103
|
+
def log_upload(obj)
|
104
|
+
yield
|
105
|
+
|
106
|
+
log(:debug) { "Uploaded #{full_path(obj)}" }
|
107
|
+
end
|
108
|
+
|
109
|
+
def logger
|
110
|
+
@enviro&.logger || super
|
111
|
+
end
|
112
|
+
|
113
|
+
def config
|
114
|
+
@enviro.config || {}
|
115
|
+
end
|
116
|
+
|
117
|
+
def fetch_bucket
|
118
|
+
config.s3storage.bucket_name
|
119
|
+
end
|
120
|
+
|
121
|
+
def fetch_prefix
|
122
|
+
config.s3storage.prefix
|
123
|
+
end
|
124
|
+
|
125
|
+
def fetch_access_key_id
|
126
|
+
config.s3storage.access_key_id || ENV['AWS_ACCESS_KEY_ID']
|
127
|
+
end
|
128
|
+
|
129
|
+
def fetch_secret_access_key
|
130
|
+
config.s3storage.secret_access_key || ENV['AWS_SECRET_ACCESS_KEY']
|
131
|
+
end
|
132
|
+
|
133
|
+
def fetch_region
|
134
|
+
config.s3storage.region || ENV['AWS_REGION']
|
141
135
|
end
|
142
136
|
end
|
143
137
|
end
|