multiwoven-integrations 0.3.1 → 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/lib/multiwoven/integrations/rollout.rb +1 -1
  3. data/lib/multiwoven/integrations/source/amazon_s3/client.rb +29 -8
  4. data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json +63 -11
  5. data/lib/multiwoven/integrations.rb +1 -0
  6. data/multiwoven-integrations.gemspec +1 -0
  7. metadata +16 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f9a7e28f8f3f1a559e5bb4feaec6a030201d447e1f1c7e90e8e8ced256274d5c
4
- data.tar.gz: a16b584bbc112998e54b9a51785498411e98cb8156116f66e082afd55c1c4916
3
+ metadata.gz: b4c9d23f9f2c64aa4e25911e6e399b2e316237902d742bb09cdc2fd4cb538666
4
+ data.tar.gz: 3257d08bcdf074497cd26dca09b3d8367039cda5b1c6922dcc8d943a5e0d293d
5
5
  SHA512:
6
- metadata.gz: f2e3b44f30db035ac8b87f1cc2ffbecc34aef53630ff7c5757193affaebc0749c827368c72cdb04444a00c0abc800c6c2b0cef97aa2484b532662292e8b0ccd0
7
- data.tar.gz: 0f2650b28d03b4118dcb86d3533cadf6fec2bcb54c0b54f349e79aa7f276b50c9a574e6b3ef87c2b19dda810b18cefcf00dfb470706753e49f5e35c0a5d858a4
6
+ metadata.gz: 29ba8c2cc87689a8bf51ac6ea0e0ab439e83197cd1aaa81b390928cf088651a827de5ce66361f13a3cada6c8252fbe38b4ec5cd5ebd8f73e6d1ec44f7f2e16a0
7
+ data.tar.gz: c4361564c16167cc5f2c59b24c9ff644c59ea5fd6c09979d17dbcb11adc63f526e82cbc00783c5f23621ce5339e9a8f82f023c516234bea4f98f0bb3dcbbd7a1
data/lib/multiwoven/integrations/rollout.rb CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Multiwoven
4
4
  module Integrations
5
- VERSION = "0.3.1"
5
+ VERSION = "0.3.2"
6
6
 
7
7
  ENABLED_SOURCES = %w[
8
8
  Snowflake
data/lib/multiwoven/integrations/source/amazon_s3/client.rb CHANGED
@@ -8,8 +8,9 @@ module Multiwoven::Integrations::Source
8
8
 
9
9
  def check_connection(connection_config)
10
10
  connection_config = connection_config.with_indifferent_access
11
- client = config_aws(connection_config)
12
- client.get_bucket_policy_status({ bucket: connection_config[:bucket] })
11
+ auth_data = get_auth_data(connection_config)
12
+ client = config_aws(auth_data, connection_config[:region])
13
+ client.get_bucket_location({ bucket: connection_config[:bucket] })
13
14
  ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
14
15
  rescue StandardError => e
15
16
  ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
@@ -17,6 +18,10 @@ module Multiwoven::Integrations::Source
17
18
 
18
19
  def discover(connection_config)
19
20
  connection_config = connection_config.with_indifferent_access
21
+ auth_data = get_auth_data(connection_config)
22
+ connection_config[:access_id] = auth_data.credentials.access_key_id
23
+ connection_config[:secret_access] = auth_data.credentials.secret_access_key
24
+ connection_config[:session_token] = auth_data.credentials.session_token
20
25
  conn = create_connection(connection_config)
21
26
  # If pulling from multiple files, all files must have the same schema
22
27
  path = build_path(connection_config[:path])
@@ -32,6 +37,10 @@ module Multiwoven::Integrations::Source
32
37
 
33
38
  def read(sync_config)
34
39
  connection_config = sync_config.source.connection_specification.with_indifferent_access
40
+ auth_data = get_auth_data(connection_config)
41
+ connection_config[:access_id] = auth_data.credentials.access_key_id
42
+ connection_config[:secret_access] = auth_data.credentials.secret_access_key
43
+ connection_config[:session_token] = auth_data.credentials.session_token
35
44
  conn = create_connection(connection_config)
36
45
  query = sync_config.model.query
37
46
  query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
@@ -47,6 +56,19 @@ module Multiwoven::Integrations::Source
47
56
 
48
57
  private
49
58
 
59
+ def get_auth_data(connection_config)
60
+ if connection_config[:auth_type] == "user"
61
+ Aws::Credentials.new(connection_config[:access_id], connection_config[:secret_access])
62
+ elsif connection_config[:auth_type] == "role"
63
+ sts_client = Aws::STS::Client.new(region: connection_config[:region])
64
+ session_name = "s3-check-connection"
65
+ sts_client.assume_role({
66
+ role_arn: connection_config[:arn],
67
+ role_session_name: session_name
68
+ })
69
+ end
70
+ end
71
+
50
72
  # DuckDB
51
73
  def create_connection(connection_config)
52
74
  conn = DuckDB::Database.open.connect
@@ -56,7 +78,8 @@ module Multiwoven::Integrations::Source
56
78
  TYPE S3,
57
79
  KEY_ID '#{connection_config[:access_id]}',
58
80
  SECRET '#{connection_config[:secret_access]}',
59
- REGION '#{connection_config[:region]}'
81
+ REGION '#{connection_config[:region]}',
82
+ SESSION_TOKEN '#{connection_config[:session_token]}'
60
83
  );
61
84
  "
62
85
  get_results(conn, secret_query)
@@ -111,13 +134,11 @@ module Multiwoven::Integrations::Source
111
134
  end
112
135
 
113
136
  # AWS SDK
114
- def config_aws(config)
115
- config = config.with_indifferent_access
137
+ def config_aws(config, region)
116
138
  Aws.config.update({
117
- region: config[:region],
118
- credentials: Aws::Credentials.new(config[:access_id], config[:secret_access])
139
+ region: region,
140
+ credentials: config
119
141
  })
120
- config.with_indifferent_access
121
142
  Aws::S3::Client.new
122
143
  end
123
144
 
data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json CHANGED
@@ -6,13 +6,51 @@
6
6
  "$schema": "http://json-schema.org/draft-07/schema#",
7
7
  "title": "AmazonS3",
8
8
  "type": "object",
9
- "required": ["region", "bucket", "access_id", "secret_access", "file_type"],
9
+ "if": {
10
+ "properties": {
11
+ "auth_type": {
12
+ "enum": ["user"]
13
+ }
14
+ }
15
+ },
16
+ "then": {
17
+ "required": [
18
+ "auth_type",
19
+ "region",
20
+ "bucket",
21
+ "access_id",
22
+ "secret_access",
23
+ "file_type"
24
+ ]
25
+ },
26
+ "else": {
27
+ "required": [
28
+ "auth_type",
29
+ "region",
30
+ "bucket",
31
+ "arn",
32
+ "file_type"
33
+ ]
34
+ },
10
35
  "properties": {
11
- "region": {
12
- "description": "AWS region",
13
- "examples": ["us-east-2"],
36
+ "auth_type": {
37
+ "title": "Authentication type",
14
38
  "type": "string",
15
- "title": "Region",
39
+ "default": "user",
40
+ "description": "Authenticate either by using an IAM User (Access Key ID & Secret Access Key) or an IAM Role (ARN)",
41
+ "enum": [
42
+ "user",
43
+ "role"
44
+ ],
45
+ "enumNames": [
46
+ "IAM User",
47
+ "IAM Role"
48
+ ],
49
+ "order": 0
50
+ },
51
+ "arn": {
52
+ "type": "string",
53
+ "title": "IAM Role ARN",
16
54
  "order": 1
17
55
  },
18
56
  "access_id": {
@@ -26,26 +64,40 @@
26
64
  "multiwoven_secret": true,
27
65
  "order": 3
28
66
  },
67
+ "region": {
68
+ "description": "AWS region",
69
+ "examples": [
70
+ "us-east-2"
71
+ ],
72
+ "type": "string",
73
+ "title": "Region",
74
+ "order": 4
75
+ },
29
76
  "bucket": {
30
77
  "description": "Bucket Name",
31
78
  "type": "string",
32
79
  "title": "Bucket",
33
- "order": 4
80
+ "order": 5
34
81
  },
35
82
  "path": {
36
83
  "description": "Path to csv or parquet files",
37
- "examples": ["/path/to/files"],
84
+ "examples": [
85
+ "/path/to/files"
86
+ ],
38
87
  "type": "string",
39
88
  "title": "Path",
40
- "order": 5
89
+ "order": 6
41
90
  },
42
91
  "file_type": {
43
92
  "description": "The type of file to read",
44
93
  "type": "string",
45
94
  "title": "File Type",
46
- "enum": ["csv", "parquet"],
47
- "order": 6
95
+ "enum": [
96
+ "csv",
97
+ "parquet"
98
+ ],
99
+ "order": 7
48
100
  }
49
101
  }
50
102
  }
51
- }
103
+ }
data/lib/multiwoven/integrations.rb CHANGED
@@ -30,6 +30,7 @@ require "base64"
30
30
  require "aws-sdk-s3"
31
31
  require "duckdb"
32
32
  require "iterable-api-client"
33
+ require "aws-sdk-sts"
33
34
 
34
35
  # Service
35
36
  require_relative "integrations/config"
data/multiwoven-integrations.gemspec CHANGED
@@ -37,6 +37,7 @@ Gem::Specification.new do |spec|
37
37
  spec.add_runtime_dependency "async-websocket"
38
38
  spec.add_runtime_dependency "aws-sdk-athena"
39
39
  spec.add_runtime_dependency "aws-sdk-s3"
40
+ spec.add_runtime_dependency "aws-sdk-sts"
40
41
  spec.add_runtime_dependency "csv"
41
42
  spec.add_runtime_dependency "dry-schema"
42
43
  spec.add_runtime_dependency "dry-struct"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multiwoven-integrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.1
4
+ version: 0.3.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Subin T P
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-06-26 00:00:00.000000000 Z
11
+ date: 2024-06-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -66,6 +66,20 @@ dependencies:
66
66
  - - ">="
67
67
  - !ruby/object:Gem::Version
68
68
  version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: aws-sdk-sts
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
69
83
  - !ruby/object:Gem::Dependency
70
84
  name: csv
71
85
  requirement: !ruby/object:Gem::Requirement