multiwoven-integrations 0.1.13 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 823511ceb667f340a6a98cf55c054e7926060f259ae02ac9bd64e2667ce24dbf
-  data.tar.gz: a6f2e9a533ab1f2ffecc8c40b7078a182296dad452d30968c6d6bc8de10304bc
+  metadata.gz: 6ecb2cd75c255a0eafd6eb964568ea33eb9b7256402b2f9644857b76646dce01
+  data.tar.gz: 59489ab31180446d458cb09246319479e637b171b13f82f4deebb4089e28f4cf
 SHA512:
-  metadata.gz: 9feabb2d1de4402f301a6d2fa1d021973d17bf3d7b262dddd7a18e8a2136de5bba9aeb48247647a5ba698bfc4784ad1b01351042168900ea0b28b76f0696578e
-  data.tar.gz: c114097c52b34c42307c7f1590b23321d21fbc0c9bbb429ee06b17b25dbf31649d522b2753e61c04ce82d81d2b79ea06428058e1dc144c91babad8007c7b560e
+  metadata.gz: 98c07060a572704b189ffdb99cf31c70b39eb6b5c12c07daa50a2a572bff73b7d9a4b7ac9e8e18b4182602f18c24eb6106b97609116557a6460b85712166e6c5
+  data.tar.gz: '09c2a44105dfd1437315f079c15df0b8d38466c48c3230ac7ec7ff44da43db14362ecdfe57f5c644217c61b5ee1beaf03e46f24859e088b3d0f91da1cfafcbbf'

data/lib/multiwoven/integrations/core/base_connector.rb

@@ -15,10 +15,12 @@ module Multiwoven
       end
 
       def meta_data
-        client_meta_data = read_json(META_DATA_PATH).to_h
-        icon_name = client_meta_data.with_indifferent_access["data"]["icon"]
+        client_meta_data = read_json(META_DATA_PATH).deep_symbolize_keys
+        icon_name = client_meta_data[:data][:icon]
         icon_url = "https://raw.githubusercontent.com/Multiwoven/multiwoven-integrations/#{MAIN_BRANCH_SHA}/assets/images/connectors/#{icon_name}"
-        @meta_data ||= client_meta_data.merge!({ "icon": icon_url })
+        client_meta_data[:data][:icon] = icon_url
+
+        @meta_data ||= client_meta_data
       end
 
       def check_connection(_connection_config)

data/lib/multiwoven/integrations/core/source_connector.rb

@@ -5,11 +5,18 @@ module Multiwoven
     class SourceConnector < BaseConnector
       def read(_sync_config)
         raise "Not implemented"
-        # return list of RecordMessage
+        # setup sync configs
+        # call query(connection, query)
       end
 
       private
 
+      # This needs to be implemented as private method
+      # In every source connector. This will be used for model preview
+      def query(connection, query)
+        # return list of RecordMessage
+      end
+
       def batched_query(sql_query, limit, offset)
         offset = offset.to_i
         limit = limit.to_i

data/lib/multiwoven/integrations/destination/facebook_custom_audience/config/meta.json

@@ -1,6 +1,6 @@
 {
   "data": {
-    "name": "Facebook",
+    "name": "FacebookCustomAudience",
     "connector_type": "destination",
     "category": "Adtech",
     "documentation_url": "https://docs.mutliwoven.com",

data/lib/multiwoven/integrations/destination/salesforce_crm/config/meta.json

@@ -1,6 +1,6 @@
 {
   "data": {
-    "name": "Salesforce CRM",
+    "name": "SalesforceCrm",
     "connector_type": "destination",
     "category": "CRM",
     "documentation_url": "https://docs.mutliwoven.com",

data/lib/multiwoven/integrations/rollout.rb

@@ -2,7 +2,7 @@
 
 module Multiwoven
   module Integrations
-    VERSION = "0.1.13"
+    VERSION = "0.1.15"
 
     ENABLED_SOURCES = %w[
       Snowflake

data/lib/multiwoven/integrations/service.rb

@@ -38,8 +38,8 @@ module Multiwoven
         def build_connectors(enabled_connectors, type)
           enabled_connectors.map do |connector|
             client = connector_class(type, connector).new
-            connector_spec = { connector_spec: client.connector_spec.to_h }
-            client.meta_data["data"].to_h.merge!(connector_spec)
+            client.meta_data[:data][:connector_spec] = client.connector_spec.to_h
+            client.meta_data
           end
         end
 

data/lib/multiwoven/integrations/source/bigquery/client.rb

@@ -51,13 +51,8 @@ module Multiwoven::Integrations::Source
         query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
 
         bigquery = create_connection(connection_config)
-        records = []
-        results = bigquery.query(query) || []
-        results.each do |row|
-          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
-        end
 
-        records
+        query(bigquery, query)
       rescue StandardError => e
         handle_exception(
           "BIGQUERY:READ:EXCEPTION",
@@ -66,6 +61,17 @@ module Multiwoven::Integrations::Source
         )
       end
 
+      private
+
+      def query(connection, query)
+        records = []
+        results = connection.query(query) || []
+        results.each do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+        records
+      end
+
       def create_connection(connection_config)
         Google::Cloud::Bigquery.new(
           project: connection_config["project_id"],

data/lib/multiwoven/integrations/source/bigquery/config/meta.json

@@ -1,6 +1,6 @@
 {
   "data": {
-    "name": "BigQuery",
+    "name": "Bigquery",
     "connector_type": "source",
     "category": "Data Warehouse",
     "documentation_url": "https://docs.mutliwoven.com",

data/lib/multiwoven/integrations/source/redshift/client.rb

@@ -50,12 +50,8 @@ module Multiwoven::Integrations::Source
         query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
 
         db = create_connection(connection_config)
-        records = db.exec(query) do |result|
-          result.map do |row|
-            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
-          end
-        end
-        records
+
+        query(db, query)
       rescue StandardError => e
         handle_exception(
           "REDSHIFT:READ:EXCEPTION",
@@ -68,6 +64,14 @@ module Multiwoven::Integrations::Source
 
       private
 
+      def query(connection, query)
+        connection.exec(query) do |result|
+          result.map do |row|
+            RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+          end
+        end
+      end
+
       def create_connection(connection_config)
         raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
 

data/lib/multiwoven/integrations/source/snowflake/client.rb

@@ -43,12 +43,7 @@ module Multiwoven::Integrations::Source
 
         db = create_connection(connection_config)
 
-        records = []
-        db.fetch(query) do |row|
-          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
-        end
-
-        records
+        query(db, query)
       rescue StandardError => e
         handle_exception(
           "SNOWFLAKE:READ:EXCEPTION",
@@ -59,6 +54,14 @@ module Multiwoven::Integrations::Source
 
       private
 
+      def query(connection, query)
+        records = []
+        connection.fetch(query) do |row|
+          records << RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
+        end
+        records
+      end
+
       def create_connection(connection_config)
         raise "Unsupported Auth type" if connection_config[:credentials][:auth_type] != "username/password"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: multiwoven-integrations
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.15
 platform: ruby
 authors:
 - Subin T P