multiwoven-integrations 0.1.13 → 0.1.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 823511ceb667f340a6a98cf55c054e7926060f259ae02ac9bd64e2667ce24dbf
-  data.tar.gz: a6f2e9a533ab1f2ffecc8c40b7078a182296dad452d30968c6d6bc8de10304bc
+  metadata.gz: b39ab365030240bd8a86273bf11ba5a583c033b794fd355229a26c9e2e18cb08
+  data.tar.gz: 3ebfe60a941654753b9e6874a722d77c43d3def2b91dcccbbf4e58d8d343fbd8
 SHA512:
-  metadata.gz: 9feabb2d1de4402f301a6d2fa1d021973d17bf3d7b262dddd7a18e8a2136de5bba9aeb48247647a5ba698bfc4784ad1b01351042168900ea0b28b76f0696578e
-  data.tar.gz: c114097c52b34c42307c7f1590b23321d21fbc0c9bbb429ee06b17b25dbf31649d522b2753e61c04ce82d81d2b79ea06428058e1dc144c91babad8007c7b560e
+  metadata.gz: ae5e08fedccad34910479b98eff4cde85c0626f9d98cca4b221f6eb5e2d1b982271013c03d13d8fd6bd4ae88ac7bd19090b684c547830de6016e270340ed0d00
+  data.tar.gz: fc5eaa86aaaa89df100d197d2be6dc24e94c5ced57a35670ab70874de8b497a9642f612ef7ff2d67c47874f51984d45de73d0d504ac5aa1982fac659343cb26a

data/lib/multiwoven/integrations/core/base_connector.rb

@@ -15,10 +15,12 @@ module Multiwoven
       end
 
       def meta_data
-        client_meta_data = read_json(META_DATA_PATH).to_h
-        icon_name = client_meta_data.with_indifferent_access["data"]["icon"]
+        client_meta_data = read_json(META_DATA_PATH).deep_symbolize_keys
+        icon_name = client_meta_data[:data][:icon]
         icon_url = "https://raw.githubusercontent.com/Multiwoven/multiwoven-integrations/#{MAIN_BRANCH_SHA}/assets/images/connectors/#{icon_name}"
-        @meta_data ||= client_meta_data.merge!({ "icon": icon_url })
+        client_meta_data[:data][:icon] = icon_url
+
+        @meta_data ||= client_meta_data
       end
 
       def check_connection(_connection_config)

data/lib/multiwoven/integrations/core/source_connector.rb

@@ -5,11 +5,18 @@ module Multiwoven
     class SourceConnector < BaseConnector
       def read(_sync_config)
         raise "Not implemented"
-        # return list of RecordMessage
+        # setup sync configs
+        # call query(connection, query)
       end
 
       private
 
+      # This needs to be implemented as private method
+      # In every source connector. This will be used for model preview
+      def query(connection, query)
+        # return list of RecordMessage
+      end
+
       def batched_query(sql_query, limit, offset)
         offset = offset.to_i
         limit = limit.to_i

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.13"
+    VERSION = "0.1.14"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/source/bigquery/client.rb

@@ -51,13 +51,8 @@ module Multiwoven::Integrations::Source
         query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
 
         bigquery = create_connection(connection_config)
-        records = []
-        results = bigquery.query(query) || []
-        results.each do |row|
-          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
-        end
 
-        records
+        query(bigquery, query)
       rescue StandardError => e
         handle_exception(
           "BIGQUERY:READ:EXCEPTION",
@@ -66,6 +61,17 @@ module Multiwoven::Integrations::Source
         )
       end
 
+      private
+
+      def query(connection, query)
+        records = []
+        results = connection.query(query) || []
+        results.each do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+        records
+      end
+
       def create_connection(connection_config)
         Google::Cloud::Bigquery.new(
           project: connection_config["project_id"],

data/lib/multiwoven/integrations/source/redshift/client.rb

@@ -50,12 +50,8 @@ module Multiwoven::Integrations::Source
         query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
 
         db = create_connection(connection_config)
-        records = db.exec(query) do |result|
-          result.map do |row|
-            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
-          end
-        end
-        records
+
+        query(db, query)
       rescue StandardError => e
         handle_exception(
           "REDSHIFT:READ:EXCEPTION",
@@ -68,6 +64,14 @@ module Multiwoven::Integrations::Source
 
       private
 
+      def query(connection, query)
+        connection.exec(query) do |result|
+          result.map do |row|
+            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+          end
+        end
+      end
+
       def create_connection(connection_config)
         raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
 

data/lib/multiwoven/integrations/source/snowflake/client.rb

@@ -43,12 +43,7 @@ module Multiwoven::Integrations::Source
 
         db = create_connection(connection_config)
 
-        records = []
-        db.fetch(query) do |row|
-          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
-        end
-
-        records
+        query(db, query)
       rescue StandardError => e
         handle_exception(
           "SNOWFLAKE:READ:EXCEPTION",
@@ -59,6 +54,14 @@ module Multiwoven::Integrations::Source
 
       private
 
+      def query(connection, query)
+        records = []
+        connection.fetch(query) do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+        records
+      end
+
       def create_connection(connection_config)
         raise "Unsupported Auth type" if connection_config[:credentials][:auth_type] != "username/password"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.14
 platform: ruby
 authors:
 - Subin T P