persistence-providers 0.0.4 → 0.0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8da78ca1fd3d1311f19a921360669fba77fb2e27f6323e555c1555959ac0377f
-  data.tar.gz: c2318fe83ad3341c8b4ec7cae5f4dde0e54cf7050b3dd32fa2e27ddc67533f31
+  metadata.gz: 9bc555b9367087540ccddc49bc1ce3fa937b0a36be7b3d6ff832373a9beeadd0
+  data.tar.gz: 5ae84500a46b183544d53b1d7d2f90cf6b49bcf52d34d2b95eba8474f7653c93
 SHA512:
-  metadata.gz: 60932aa0e244371a70705ef19123de4bf025baf2953be26114408f06dbfba64de2c6ea8503046eaa41281de4672198e649fc7b0c50b0b8976c8111add10ae07e
-  data.tar.gz: 518e4712e215c002d9850eb88faac8019c961310fed2e7f78878c422ce2eb98d96d31903217b178544b86f272a633ad0e5b77e9fc2c345d631d3fcaef3c15d3c
+  metadata.gz: 52cc878735fdf04cb19b7a202ff2622872b4514f40c1b849a0ff73a821f4e8e333c1aa9a3ac6ba6b31de29b199f4c49fb5fbd3df2c2bdf64fd4d421f6ff67e7d
+  data.tar.gz: a9b7fbf25e02a5b1dc15220760e91ca1bc703e7f1c2f388f6f2b083c48170c1162ffc483464cbde99f31e2465dd687c78bc47006a5ad9ee6157c5c40538083e9

data/lib/crd_client.rb

@@ -1,10 +1,16 @@
 require 'kubeclient'
 require 'celluloid/io'
-require 'singleton'
+# require 'singleton'
 
 module DTK
   class CrdClient
-    include Singleton
+    # include Singleton
+    DEFAULT_API_VERSION = 'v1alpha1'
+
+    # COMPONENT_DEF_CRD_VERSION     = ENV["COMPONENT_DEF_CRD_VERSION"]
+    # ASSEMBLY_CRD_VERSION          = ENV["ASSEMBLY_CRD_VERSION"]
+    # WORKFLOW_CRD_VERSION          = ENV["WORKFLOW_CRD_VERSION"]
+    # WORKFLOW_INSTANCE_CRD_VERSION = ENV["WORKFLOW_INSTANCE_CRD_VERSION"]
 
     attr_accessor :kubeclient
 
@@ -12,17 +18,26 @@ module DTK
       if @kubeclient = opts[:kubeclient]
         @kubeclient
       else
-        ::DTK::CrdClient.instance.kubeclient
+        kubeclient_version(opts)
       end
     end
 
-    # opts can have keys
-    #   kubernetes_client - already instantiated kubernetes client
-    def initialize(opts = {})
-      if @kubeclient = opts[:kubernetes_client]
-        return @kubeclient
+    def self.kubeclient_version(opts = {})
+      version = opts[:apiVersion] || DEFAULT_API_VERSION
+
+      if existing_version = KubeclientVersions[version]
+        return existing_version
+      else
+        new_instance = new(version).kubeclient
+        KubeclientVersions[version] = new_instance
+        new_instance
       end
+    end
+    KubeclientVersions = {}
 
+    # opts can have keys
+    #   kubernetes_client - already instantiated kubernetes client
+    def initialize(apiVersion)
       ssl_options  = {}
       auth_options = { bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token' }
       
@@ -37,7 +52,7 @@ module DTK
 
       @kubeclient = Kubeclient::Client.new(
         'https://kubernetes.default.svc/apis/',
-        'dtk.io/v1alpha1',
+        "dtk.io/#{apiVersion}",
         auth_options: auth_options,
         ssl_options:  ssl_options,
         socket_options: socket_options 

data/lib/persistence_providers.rb

@@ -1,5 +1,6 @@
 module DTK
   module State
+    require_relative 'utils'
     require_relative 'crd_client'
     require_relative 'state/crd_assembly'
     require_relative 'state/component'

data/lib/state/component/providers/influxdb.rb

@@ -13,88 +13,51 @@ module DTK::State
       end
 
       def get(namespace, component_name, assembly_name, attribute_name, opts = {})
-        required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-        if opts[:provider] == "correlation"
-          errors = client.measurement_helper(:errors)
-          required_tags.merge!({ correlator_type: opts[:entrypoint].split("/").last.split(".")[0] })
-          errors.get_last_point(required_tags)
-        elsif
-          last_value = measurement.get_last_point(required_tags)
-          last_value
-        end
+        required_tags = measurement.get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags.merge! measurement.get_correlator_type(opts[:entrypoint]) if opts[:provider] == "correlation"
+        measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting attribute from InfluxDB.\nError: #{e}"
       end
 
       def write(namespace, component_name, assembly_name, attribute_name, value, opts = {}, timestamp = nil)
-        if opts[:provider] == "correlation"
-          errors = client.measurement_helper(:errors)
-          required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-          required_tags.merge!({ correlator_type: opts[:entrypoint].split("/").last.split(".")[0] })
-          errors.write(value.to_s, required_tags, timestamp)
-        elsif
-          required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-          measurement.write(value, required_tags, timestamp)
-        end
+        required_tags = measurement.get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags.merge! measurement.get_correlator_type(opts[:entrypoint]) if opts[:provider] == "correlation"
+        measurement.write(value.to_s, required_tags, timestamp)
+      rescue => e
+        raise "Error happened while writing attribute into InfluxDB.\Error: #{e}"
       end
 
       def write_event(event_id, pod_name, pod_namespace, event_source, event_message, component_name, attribute_name, task_id, timestamp)
-        begin
-          fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
-          value_to_write = { event_source: event_source, event_message: event_message }
-          required_tags = {
-            event_id: event_id,
-            pod_name: pod_name,
-            pod_namespace: pod_namespace,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
-          measurement.write(value_to_write.to_s, required_tags, timestamp)
-        rescue => error
-          fail error
-        end
+        fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
+        value_to_write = { event_source: event_source, event_message: event_message }
+        required_tags = measurement.get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
+        measurement.write(value_to_write.to_s, required_tags, timestamp)
+      rescue => error
+        raise "Error happened while writing event into InfluxDB.\nError: #{e}"
       end
 
       def get_event(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
-        required_tags = {
-            event_id: event_id,
-            pod_name: pod_name,
-            pod_namespace: pod_namespace,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
+        required_tags = measurement.get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
         last_point = measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting event from InfluxDB.\nError: #{e}"
       end
 
       def write_state(type, name, namespace, object_state, spec, status, component_name, attribute_name, task_id, timestamp)
-        begin
-          fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
-          value_to_write = { spec: spec, status: status }
-          required_tags = {
-            type: type,
-            name: name,
-            namespace: namespace,
-            object_state: object_state,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
-          measurement.write(value_to_write.to_s, required_tags, timestamp)
-        rescue => error
-          fail error
-        end
+        raise "Bad timestamp input, write operation to InfluxDB wont be completed" if timestamp > Time.new
+        value_to_write = { spec: spec, status: status }
+        required_tags = measurement.get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        measurement.write(value_to_write.to_s, required_tags, timestamp)
+      rescue => e
+        raise "Error happened while writing state into InfluxDB.\nError: #{e}"
       end
-      
-      private 
 
-      def get_required_tags(namespace, component_name, assembly_name, attribute_name)
-        required_tags = {
-          namespace: namespace,
-          component_name: component_name,
-          assembly_name: assembly_name,
-          attribute_name: attribute_name,
-          task_id: "1"
-        }
+      def get_state(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        required_tags = measurement.get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting state from InfluxDB.\nError: #{e}"
       end
     end
   end

data/lib/state/component/providers/influxdb/client.rb

@@ -10,30 +10,24 @@ module DTK::State
       end
 
       def query(query_expression)
-        begin
-          query_api = self.connection.create_query_api
-          query_api.query(query_expression)
-        rescue => error
-          fail "Failed in query"
-        end
+        query_api = self.connection.create_query_api
+        query_api.query(query_expression)
+      rescue => e
+        raise "Failed while processing flux query!. Error: #{e}"
       end
 
       def write_point(data)
-        begin
-          write_api = self.connection.create_write_api
-          write_api.write(data: data)
-        rescue => error
-          fail error
-        end
+        write_api = self.connection.create_write_api
+        write_api.write(data: data)
+      rescue => e
+        raise e
       end
 
       def measurement_helper(measurement_name)
-        begin
-          klass = Measurement.const_get(measurement_name.to_sym.capitalize)
-          klass.new(measurement_name, self)
-        rescue => error
-          fail error
-        end
+        klass = Measurement.const_get(measurement_name.to_sym.capitalize)
+        klass.new(measurement_name, self)
+      rescue => e
+        raise e
       end
 
       attr_reader :connection_parameters, :connection
@@ -53,18 +47,21 @@ module DTK::State
           org: params[:org],
           bucket: params[:bucket]
         }
+      rescue => e
+        raise "Problem happened while processing InfluxDB connection parameters. Error: #{e}"
       end
 
       def return_connection(connection_parameters)
-        begin
-          InfluxDB2::Client.new(connection_parameters[:url], connection_parameters[:token],
-                                bucket: connection_parameters[:bucket],
-                                org: connection_parameters[:org],
-                                precision: InfluxDB2::WritePrecision::MILLISECOND,
-                                use_ssl: false)
-        rescue => error
-          fail "Error: #{error}"
-        end
+        client = InfluxDB2::Client.new(connection_parameters[:url], connection_parameters[:token],
+                              bucket: connection_parameters[:bucket],
+                              org: connection_parameters[:org],
+                              precision: InfluxDB2::WritePrecision::MILLISECOND,
+                              use_ssl: false)
+        query_api = client.create_query_api
+        query_api.query(query: 'from(bucket:"' + connection_parameters[:bucket] + '") |> range(start: -5)')
+        client
+      rescue => e
+        raise "Connection with InfluxDB could not be established. #{e}"
       end
     end
   end

data/lib/state/component/providers/influxdb/measurement.rb

@@ -20,14 +20,22 @@ module DTK::State
       end
 
       def get_last_point(params_hash = {})
-        begin
-          check_params_hash(params_hash)
-          flux_query = 'from(bucket:"' + client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn: (r) => r._measurement == "' + name.to_s + '")' + flux_filter(params_hash) + ' |> last()'  +   '|> drop(columns: ["_start", "_stop", "_field", "_measurement", "attribute_name", "assembly_name", "task_id", "component_name", "namespace"])'
-          result  = self.client.query(query: flux_query)
-          result.values.map(&:records).flatten.map(&:values)
-        rescue => error
-          fail error
-        end
+        check_params_hash(params_hash)
+        flux_query = 'from(bucket:"' + client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn: (r) => r._measurement == "' + name.to_s + '")' + flux_filter(params_hash) + ' |> last()'  +   '|> drop(columns: ["_start", "_stop", "_field", "_measurement", "attribute_name", "assembly_name", "task_id", "component_name", "namespace"])'
+        result  = self.client.query(query: flux_query)
+        result.values.map(&:records).flatten.map(&:values)
+      rescue => e
+        raise "Failed while getting last attribute point. Error: #{e}"
+      end
+
+      def get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags = {
+          namespace: namespace,
+          component_name: component_name,
+          assembly_name: assembly_name,
+          attribute_name: attribute_name,
+          task_id: "1"
+        }
       end
       
       protected
@@ -90,7 +98,7 @@ module DTK::State
           fail "Parameter '#{name}' has an illegal type, legal types are #{LEGAL_TAG_CLASSES.join(', ')}"
         end
       end
-      
+
     end
   end
 end

data/lib/state/component/providers/influxdb/measurement/errors.rb

@@ -8,6 +8,12 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_correlator_type(entrypoint)
+          { 
+            correlator_type: entrypoint.split("/").last.split(".")[0] 
+          }
+        end
+
         protected
 
         def required_params

data/lib/state/component/providers/influxdb/measurement/events.rb

@@ -8,12 +8,23 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
+          {
+            event_id: event_id,
+            pod_name: pod_name,
+            pod_namespace: pod_namespace,
+            component_name: component_name,
+            attribute_name: attribute_name,
+            task_id: task_id
+          }
+        end
+
         protected
 
         def required_params
           [:event_id, :pod_name, :pod_namespace, :component_name, :attribute_name, :task_id]
         end
-
+        
       end                       
     end
   end

data/lib/state/component/providers/influxdb/measurement/states.rb

@@ -8,12 +8,24 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+          {
+            type: type,
+            name: name,
+            namespace: namespace,
+            object_state: object_state,
+            component_name: component_name,
+            attribute_name: attribute_name,
+            task_id: task_id
+          }
+        end
+
         protected
 
         def required_params
           [:type, :name, :namespace, :object_state, :component_name, :attribute_name, :task_id]
         end
-
+        
       end                       
     end
   end

data/lib/state/component_def.rb

@@ -2,6 +2,8 @@ module DTK::State
   class ComponentDef
     require_relative 'component_def/attribute_type_info'
 
+    COMPONENT_DEF_CRD_VERSION     = ENV["COMPONENT_DEF_CRD_VERSION"]
+
     attr_reader :name, :namespace, :executable_actions, :attribute_type_info
 
     def initialize(namespace, name, content)
@@ -12,6 +14,7 @@ module DTK::State
     end
 
     def self.get(namespace, name, opts = {})
+      opts[:apiVersion] = COMPONENT_DEF_CRD_VERSION
       crd_component_def = ::DTK::CrdClient.get_kubeclient(opts).get_componentdef(name, namespace)
       ComponentDef.new(namespace, name, crd_component_def)
     end

data/lib/state/crd_assembly.rb

@@ -1,5 +1,7 @@
 module DTK::State
   class CrdAssembly
+    ASSEMBLY_CRD_VERSION = ENV["ASSEMBLY_CRD_VERSION"]
+
     attr_reader :name, :namespace, :crd_content, :components, :references
 
     def initialize(namespace, name, crd_content)
@@ -11,7 +13,7 @@ module DTK::State
     end
 
     def self.get(namespace, name, opts = {})
-      # crd_component = ::DTK::CrdClient.instance.kubeclient.get_component(name, namespace)
+      opts[:apiVersion] = ASSEMBLY_CRD_VERSION
       crd_assembly = ::DTK::CrdClient.get_kubeclient(opts).get_assembly(name, namespace)
       CrdAssembly.new(namespace, name, crd_assembly)
     end

data/lib/state/workflow_instance.rb

@@ -2,6 +2,8 @@ module DTK::State
   class WorkflowInstance
     require_relative 'workflow_instance/attribute_type_info'
 
+    WORKFLOW_INSTANCE_CRD_VERSION = ENV["WORKFLOW_INSTANCE_CRD_VERSION"]
+
     attr_reader :name, :namespace, :assembly, :workflow_template, :attributes, :workflow,  :attribute_type_info
 
     def initialize(namespace, name, crd_content)
@@ -15,6 +17,7 @@ module DTK::State
     end
 
     def self.get(namespace, name, opts = {})
+      opts[:apiVersion] = WORKFLOW_INSTANCE_CRD_VERSION
       workflow_instance = ::DTK::CrdClient.get_kubeclient(opts).get_workflow_instance(name, namespace)
       WorkflowInstance.new(namespace, name, workflow_instance)
     end
@@ -35,6 +38,7 @@ module DTK::State
     def self.update_action_level_result_attributes(namespace, name, attributes, action_id, opts = {})
       return "Dynamic attributes do not exist for action with id #{@action_id}, nothing to update" if attributes.nil? || attributes.empty?
       attributes.delete_if { |key, value| value.nil? || value.to_s.strip == '' }
+      opts[:apiVersion] = WORKFLOW_INSTANCE_CRD_VERSION
       workflow_instance = ::DTK::CrdClient.get_kubeclient(opts).get_workflow_instance(name, namespace)
       workflow = workflow_instance[:spec][:workflow]
 
@@ -63,6 +67,7 @@ module DTK::State
     end
 
     def self.update_action_status(namespace, name, parent_id, action_id, status, error_message = "", opts = {})
+      opts[:apiVersion] = WORKFLOW_INSTANCE_CRD_VERSION
       workflow_instance = ::DTK::CrdClient.get_kubeclient(opts).get_workflow_instance(name, namespace)
       steps = workflow_instance[:spec][:status][:steps]
       action_index_steps = steps.find_index { |action| action[:id].eql? action_id }

data/lib/utils.rb

@@ -0,0 +1,3 @@
+module Utils
+  require_relative 'utils/log'
+end

data/lib/utils/log.rb

@@ -0,0 +1,22 @@
+module Utils
+  module Log
+    require 'logger'
+
+    def self.instance
+      @instance ||= Logger.new('/proc/1/fd/1', formatter: proc { |severity, datetime, progname, msg| 
+        orange_color = "\x1b[33m"
+        white_color = "\x1b[37m"
+        red_color = "\x1b[31m"
+
+        date_format = datetime.strftime("%Y-%m-%d %H:%M:%S:%L")
+        if severity == "INFO"
+          "#{orange_color}[#{date_format}] - #{white_color}#{msg}\n"
+        elsif severity == "WARN"
+          "#{orange_color}[#{date_format}] [WARNING] - #{msg}\n"
+        elsif severity == "ERROR"       
+            "#{red_color}[#{date_format}] [ERROR] - #{msg}\n"
+        end
+      })
+    end 
+  end
+end

data/persistence-providers.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name        = 'persistence-providers'
-  spec.version     = '0.0.4'
+  spec.version     = '0.0.4.1'
   spec.author      = 'Reactor8'
   spec.email       = 'support@reactor8.com'
   spec.description = %q{Persistence providers plugin}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: persistence-providers
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.4.1
 platform: ruby
 authors:
 - Reactor8
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-15 00:00:00.000000000 Z
+date: 2020-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kubeclient
@@ -79,6 +79,8 @@ files:
 - lib/state/executable_action/attribute_type_info.rb
 - lib/state/workflow_instance.rb
 - lib/state/workflow_instance/attribute_type_info.rb
+- lib/utils.rb
+- lib/utils/log.rb
 - persistence-providers.gemspec
 - test-destroy-influxdb.rb
 - test-influxdb.rb