persistence-providers 0.0.3.6 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a18d9fbe5a1507dfde899dc0bf467a6148ae7bda7ed58995854ecc372ef4efa5
-  data.tar.gz: 32cf0fa8b1433d55c7f79da8767ed7de1b8d1daa66f8c1e409d6b8978914c938
+  metadata.gz: c0df9c5f3778b1aba3a4b2474b9e47dad64d60566697c3ee15708e26a1a8bfe6
+  data.tar.gz: a7d39896b337a3579c491b4f13553b28556324b6ecbfdaf7439eafe1f2f6d6b7
 SHA512:
-  metadata.gz: bebb1b072b268c17fff5e2430e37b4fa8dd771cbbdb10930d3e7cf852c008d1d4d49c5f30a5fcce8afa9930da6e06707c9c7bfe3d58f12c13c2101ecf55ec15a
-  data.tar.gz: 05126ad2c6f92ab855877e71929cf00f6955e21873b6782a9805c05884ec6e0e08ad53bdb7314589039acef82b96abf8bf376557252314aacefda8359c3c4070
+  metadata.gz: 55e280a54d5a75f1bc6ae12b5fcf31659bae2efcf7c83897c0a6fed78d576464173fecec7bc2807945caf56cc3c930248947fd3d747c934ba75e06d73762715a
+  data.tar.gz: 3e8f981dd1ca1cbb000f1e926bebee66bb73eeaf710c4f0824154dae1526dd74d7dc2c1a87f40e9a58a94ba48fa9ca39ad054f5c9cbc4d3338c0340ba7283931

data/lib/crd_client.rb

@@ -1,10 +1,16 @@
 require 'kubeclient'
 require 'celluloid/io'
-require 'singleton'
+# require 'singleton'
 
 module DTK
   class CrdClient
-    include Singleton
+    # include Singleton
+    DEFAULT_API_VERSION = 'v1alpha1'
+
+    # COMPONENT_DEF_CRD_VERSION     = ENV["COMPONENT_DEF_CRD_VERSION"]
+    # ASSEMBLY_CRD_VERSION          = ENV["ASSEMBLY_CRD_VERSION"]
+    # WORKFLOW_CRD_VERSION          = ENV["WORKFLOW_CRD_VERSION"]
+    # WORKFLOW_INSTANCE_CRD_VERSION = ENV["WORKFLOW_INSTANCE_CRD_VERSION"]
 
     attr_accessor :kubeclient
 
@@ -12,17 +18,26 @@ module DTK
       if @kubeclient = opts[:kubeclient]
         @kubeclient
       else
-        ::DTK::CrdClient.instance.kubeclient
+        kubeclient_version(opts)
       end
     end
 
-    # opts can have keys
-    #   kubernetes_client - already instantiated kubernetes client
-    def initialize(opts = {})
-      if @kubeclient = opts[:kubernetes_client]
-        return @kubeclient
+    def self.kubeclient_version(opts = {})
+      version = opts[:apiVersion] || DEFAULT_API_VERSION
+
+      if existing_version = KubeclientVersions[version]
+        return existing_version
+      else
+        new_instance = new(version).kubeclient
+        KubeclientVersions[version] = new_instance
+        new_instance
       end
+    end
+    KubeclientVersions = {}
 
+    # opts can have keys
+    #   kubernetes_client - already instantiated kubernetes client
+    def initialize(apiVersion)
       ssl_options  = {}
       auth_options = { bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token' }
       
@@ -37,7 +52,7 @@ module DTK
 
       @kubeclient = Kubeclient::Client.new(
         'https://kubernetes.default.svc/apis/',
-        'dtk.io/v1alpha1',
+        "dtk.io/#{apiVersion}",
         auth_options: auth_options,
         ssl_options:  ssl_options,
         socket_options: socket_options 

data/lib/persistence_providers.rb

@@ -1,10 +1,12 @@
 module DTK
   module State
+    require_relative 'utils'
     require_relative 'crd_client'
     require_relative 'state/crd_assembly'
     require_relative 'state/component'
     require_relative 'state/component_def'
     require_relative 'state/executable_action'
     require_relative 'state/workflow_instance'
+    require_relative 'state/workflow'
   end
 end

data/lib/state/component.rb

@@ -3,12 +3,14 @@ module DTK::State
     require_relative 'component/attribute'
 
     attr_reader :name, :parent, :component_def
+    attr_accessor :raw_attributes
 
     def initialize(name, component_content, parent, opts = {})
       @name           = name
       # @parent     = parent
       @component_defs = parent.references.component_def
       @component_def  = get_component_def(opts)
+      @raw_attributes = component_content[:attributes] || {}
 
       @attribute_objs = Attribute.create_from_kube_hash(component_content[:attributes] || {})# convert_to_attribute_objects(component_content[:attributes])
     end
@@ -17,11 +19,50 @@ module DTK::State
     #   task_id
     def self.get(crd_assembly_namespace, crd_assembly_name, component_name, opts = {})
       crd_assembly = CrdAssembly.get(crd_assembly_namespace, crd_assembly_name, opts)
-      if matching_component = crd_assembly.components.find{ |cmp| cmp.to_hash.keys.first.to_s == component_name }
-        Component.new(component_name, matching_component[component_name], crd_assembly, opts)
+      if matching_component = Component.find_matching_component(crd_assembly, component_name)
+        Component.new(component_name, Component.get_component_content(matching_component, component_name), crd_assembly, opts)
       end
     end
 
+    def self.get_with_influx_data(namespace, assembly_name, component_name, opts = {})
+      component = get(namespace, assembly_name, component_name, opts)
+      return unless component
+      attributes = component.raw_attributes.to_hash
+
+      attr_type_info = component.component_def.attribute_type_info
+      attr_type_info.each do |attr_info|
+        if attr_info.temporal
+          attribute_name = attr_info.name.to_sym
+          influxdb = ::DTK::State::Component::Attribute::Influxdb.new(:attributes)
+          influxdb_attribute = influxdb.get(namespace, component_name, assembly_name, attribute_name, opts)
+          if valid_attribute = influxdb_attribute.first
+            value = valid_attribute['_value']
+            if attributes[attribute_name]
+              if attributes[attribute_name].is_a?(String)
+                attributes[attribute_name] = value
+              else
+                attributes[attribute_name][:value] = value
+              end
+            end
+          end
+        end
+      end
+
+      component.raw_attributes = attributes
+      component
+    end
+
+    def to_hash
+      {
+        name: @name,
+        component_def: {
+          name: @component_def.name,
+          namespace: @component_def.namespace
+        },
+        attributes: @raw_attributes.to_hash
+      }
+    end
+
     # opts can have keys
     # task_id
     # format:
@@ -44,16 +85,64 @@ module DTK::State
       end
     end
 
+    def attribute_metadata
+      attributes         = @raw_attributes.to_hash
+      attr_type_info     = @component_def.attribute_type_info
+      attribute_metadata = {}
+
+      attr_type_info.each do |attr_info|
+        attr_info_hash = attr_info.to_hash
+        attribute_name = attr_info_hash[:name].to_sym
+
+        if attribute = attributes[attribute_name]
+          if attribute.is_a?(String)
+            attribute = { value: attribute }
+          end
+
+          attribute_metadata[attribute_name] = attr_info_hash.merge(attribute)
+        end
+      end
+
+      attribute_metadata
+    end
+
     def attribute_values
-      attribute_with_values = []
-      @content[:attributes].each do |name, content|
-        attribute_with_values << { name => content[:value] }
+      attribute_with_values = {}
+      @raw_attributes.each do |name, content|
+        attribute_with_values.merge!(name => content[:value])
       end
       attribute_with_values
     end
 
+    def self.create_from_kube_array(kube_components, crd_assembly)
+      kube_components.map do |component_hash|
+        if component_hash.is_a?(String)
+          component_name    = component_hash
+          component_content = {}
+        else
+          component_name = component_hash.to_hash.keys.first
+          component_content = component_hash[component_name]
+        end
+        Component.new(component_name, component_content, crd_assembly)
+      end
+    end
+
     private
 
+    def self.get_component_content(matching_component, component_name)
+      return matching_component.is_a?(String) ? {} : matching_component[component_name]
+    end
+
+    def self.find_matching_component(assembly, component_name)
+      assembly.components.find do |cmp|
+        if cmp.is_a? String
+          cmp == component_name
+        else
+          cmp.to_hash.keys.first.to_s == component_name
+        end
+      end
+    end
+
     def get_component_def(opts = {})
       destructured_component = destructure_component_full_name
       component_def = @component_defs.find { |component_def| component_def[:name] == destructured_component[:component_def_name] }

data/lib/state/component/providers/influxdb.rb

@@ -4,6 +4,7 @@ module DTK::State
       require_relative('influxdb/client')
       require_relative('influxdb/measurement')
       require_relative('influxdb/semantictype')
+      require_relative('../../../utils/log')
 
       attr_reader :client, :measurement
 
@@ -13,88 +14,51 @@ module DTK::State
       end
 
       def get(namespace, component_name, assembly_name, attribute_name, opts = {})
-        required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-        if opts[:provider] == "correlation"
-          errors = client.measurement_helper(:errors)
-          required_tags.merge!({ correlator_type: opts[:entrypoint].split("/").last.split(".")[0] })
-          errors.get_last_point(required_tags)
-        elsif
-          last_value = measurement.get_last_point(required_tags)
-          last_value
-        end
+        required_tags = measurement.get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags.merge! measurement.get_correlator_type(opts[:entrypoint]) if opts[:provider] == "correlation"
+        measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting attribute from InfluxDB.\nError: #{e}"
       end
 
       def write(namespace, component_name, assembly_name, attribute_name, value, opts = {}, timestamp = nil)
-        if opts[:provider] == "correlation"
-          errors = client.measurement_helper(:errors)
-          required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-          required_tags.merge!({ correlator_type: opts[:entrypoint].split("/").last.split(".")[0] })
-          errors.write(value.to_s, required_tags, timestamp)
-        elsif
-          required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-          measurement.write(value, required_tags, timestamp)
-        end
+        required_tags = measurement.get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags.merge! measurement.get_correlator_type(opts[:entrypoint]) if opts[:provider] == "correlation"
+        measurement.write(value.to_s, required_tags, timestamp)
+      rescue => e
+        raise "Error happened while writing attribute into InfluxDB.\Error: #{e}"
       end
 
       def write_event(event_id, pod_name, pod_namespace, event_source, event_message, component_name, attribute_name, task_id, timestamp)
-        begin
-          fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
-          value_to_write = { event_source: event_source, event_message: event_message }
-          required_tags = {
-            event_id: event_id,
-            pod_name: pod_name,
-            pod_namespace: pod_namespace,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
-          measurement.write(value_to_write.to_s, required_tags, timestamp)
-        rescue => error
-          fail error
-        end
+        fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
+        value_to_write = { event_source: event_source, event_message: event_message }
+        required_tags = measurement.get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
+        measurement.write(value_to_write.to_s, required_tags, timestamp)
+      rescue => error
+        raise "Error happened while writing event into InfluxDB.\nError: #{e}"
       end
 
       def get_event(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
-        required_tags = {
-            event_id: event_id,
-            pod_name: pod_name,
-            pod_namespace: pod_namespace,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
+        required_tags = measurement.get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
         last_point = measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting event from InfluxDB.\nError: #{e}"
       end
 
       def write_state(type, name, namespace, object_state, spec, status, component_name, attribute_name, task_id, timestamp)
-        begin
-          fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
-          value_to_write = { spec: spec, status: status }
-          required_tags = {
-            type: type,
-            name: name,
-            namespace: namespace,
-            object_state: object_state,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
-          measurement.write(value_to_write.to_s, required_tags, timestamp)
-        rescue => error
-          fail error
-        end
+        raise "Bad timestamp input, write operation to InfluxDB wont be completed" if timestamp > Time.new
+        value_to_write = { spec: spec, status: status }
+        required_tags = measurement.get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        measurement.write(value_to_write.to_s, required_tags, timestamp)
+      rescue => e
+        raise "Error happened while writing state into InfluxDB.\nError: #{e}"
       end
-      
-      private 
 
-      def get_required_tags(namespace, component_name, assembly_name, attribute_name)
-        required_tags = {
-          namespace: namespace,
-          component_name: component_name,
-          assembly_name: assembly_name,
-          attribute_name: attribute_name,
-          task_id: "1"
-        }
+      def get_state(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        required_tags = measurement.get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting state from InfluxDB.\nError: #{e}"
       end
     end
   end

data/lib/state/component/providers/influxdb/client.rb

@@ -10,30 +10,24 @@ module DTK::State
       end
 
       def query(query_expression)
-        begin
-          query_api = self.connection.create_query_api
-          query_api.query(query_expression)
-        rescue => error
-          fail "Failed in query"
-        end
+        query_api = self.connection.create_query_api
+        query_api.query(query_expression)
+      rescue => e
+        raise "Failed while processing flux query!. Error: #{e}"
       end
 
       def write_point(data)
-        begin
-          write_api = self.connection.create_write_api
-          write_api.write(data: data)
-        rescue => error
-          fail error
-        end
+        write_api = self.connection.create_write_api
+        write_api.write(data: data)
+      rescue => e
+        raise e
       end
 
       def measurement_helper(measurement_name)
-        begin
-          klass = Measurement.const_get(measurement_name.to_sym.capitalize)
-          klass.new(measurement_name, self)
-        rescue => error
-          fail error
-        end
+        klass = Measurement.const_get(measurement_name.to_sym.capitalize)
+        klass.new(measurement_name, self)
+      rescue => e
+        raise e
       end
 
       attr_reader :connection_parameters, :connection
@@ -53,18 +47,21 @@ module DTK::State
           org: params[:org],
           bucket: params[:bucket]
         }
+      rescue => e
+        raise "Problem happened while processing InfluxDB connection parameters. Error: #{e}"
       end
 
       def return_connection(connection_parameters)
-        begin
-          InfluxDB2::Client.new(connection_parameters[:url], connection_parameters[:token],
-                                bucket: connection_parameters[:bucket],
-                                org: connection_parameters[:org],
-                                precision: InfluxDB2::WritePrecision::MILLISECOND,
-                                use_ssl: false)
-        rescue => error
-          fail "Error: #{error}"
-        end
+        client = InfluxDB2::Client.new(connection_parameters[:url], connection_parameters[:token],
+                              bucket: connection_parameters[:bucket],
+                              org: connection_parameters[:org],
+                              precision: InfluxDB2::WritePrecision::MILLISECOND,
+                              use_ssl: false)
+        query_api = client.create_query_api
+        query_api.query(query: 'from(bucket:"' + connection_parameters[:bucket] + '") |> range(start: -5)')
+        client
+      rescue => e
+        raise "Connection with InfluxDB could not be established. #{e}"
       end
     end
   end

data/lib/state/component/providers/influxdb/measurement.rb

@@ -20,14 +20,22 @@ module DTK::State
       end
 
       def get_last_point(params_hash = {})
-        begin
-          check_params_hash(params_hash)
-          flux_query = 'from(bucket:"' + client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn: (r) => r._measurement == "' + name.to_s + '")' + flux_filter(params_hash) + ' |> last()'  +   '|> drop(columns: ["_start", "_stop", "_field", "_measurement", "attribute_name", "assembly_name", "task_id", "component_name", "namespace"])'
-          result  = self.client.query(query: flux_query)
-          result.values.map(&:records).flatten.map(&:values)
-        rescue => error
-          fail error
-        end
+        check_params_hash(params_hash)
+        flux_query = 'from(bucket:"' + client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn: (r) => r._measurement == "' + name.to_s + '")' + flux_filter(params_hash) + ' |> last()'  +   '|> drop(columns: ["_start", "_stop", "_field", "_measurement", "attribute_name", "assembly_name", "task_id", "component_name", "namespace"])'
+        result  = self.client.query(query: flux_query)
+        result.values.map(&:records).flatten.map(&:values)
+      rescue => e
+        raise "Failed while getting last attribute point. Error: #{e}"
+      end
+
+      def get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags = {
+          namespace: namespace,
+          component_name: component_name,
+          assembly_name: assembly_name,
+          attribute_name: attribute_name,
+          task_id: "1"
+        }
       end
       
       protected
@@ -90,7 +98,7 @@ module DTK::State
           fail "Parameter '#{name}' has an illegal type, legal types are #{LEGAL_TAG_CLASSES.join(', ')}"
         end
       end
-      
+
     end
   end
 end