persistence-providers 0.0.3.7 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb4b1e1715f9e1ed7b1174ff99a9adc00a13c5c0f56be3ed0d974b394fcf166d
-  data.tar.gz: ec5385e7dea1fac44d2c9d21ef902829a4fdee986d76a9f565ec3d6043d26f3d
+  metadata.gz: 0a15ad06be0a52e1e087f50377fa1669846e8eb2d01ae40776a07543b8440b4b
+  data.tar.gz: 7f68ed470f1ca6ab2bee9b7bbf10d7003059acafbae22bf04351a52653416b0d
 SHA512:
-  metadata.gz: 26dcc9ac221ec5be80eb8114c50ba82d67dfaec44c809dc2435d1bb062d23de6d6faf80e2f4ff1130ad7db8ed4a7d318edd1e220340fe0fda17fd89186089868
-  data.tar.gz: 96207adcb7b8914850b27e6bc3e991221b1180354eb512df0db8595695cb998824f20be0c3ea86fef8ed00a1f8dfcd7fdc38bc506037a5194525a6ae4fcfd03b
+  metadata.gz: ede15c848020024c37f2a79608c96b8591691032c97d0fd10e39b964dd5fddc94f7df5e9d31383a798dcdaaeae8dbaffd8d966214ed332f872fe43bfb2ee875c
+  data.tar.gz: 9e7a4383fbcb11b4402b3f8cd449b8aab48f724a6cbe7ba020aa878ee44e3ee90f6dbfdbd18fb2f8fda04e3cbb0154a8fcd02a7a9d7f6f415c816eefb1ee0d2b

data/lib/crd_client.rb

@@ -1,10 +1,16 @@
 require 'kubeclient'
 require 'celluloid/io'
-require 'singleton'
+# require 'singleton'
 
 module DTK
   class CrdClient
-    include Singleton
+    # include Singleton
+    DEFAULT_API_VERSION = 'v1alpha1'
+
+    # COMPONENT_DEF_CRD_VERSION     = ENV["COMPONENT_DEF_CRD_VERSION"]
+    # ASSEMBLY_CRD_VERSION          = ENV["ASSEMBLY_CRD_VERSION"]
+    # WORKFLOW_CRD_VERSION          = ENV["WORKFLOW_CRD_VERSION"]
+    # WORKFLOW_INSTANCE_CRD_VERSION = ENV["WORKFLOW_INSTANCE_CRD_VERSION"]
 
     attr_accessor :kubeclient
 
@@ -12,17 +18,26 @@ module DTK
       if @kubeclient = opts[:kubeclient]
         @kubeclient
       else
-        ::DTK::CrdClient.instance.kubeclient
+        kubeclient_version(opts)
       end
     end
 
-    # opts can have keys
-    #   kubernetes_client - already instantiated kubernetes client
-    def initialize(opts = {})
-      if @kubeclient = opts[:kubernetes_client]
-        return @kubeclient
+    def self.kubeclient_version(opts = {})
+      version = opts[:apiVersion] || DEFAULT_API_VERSION
+
+      if existing_version = KubeclientVersions[version]
+        return existing_version
+      else
+        new_instance = new(version).kubeclient
+        KubeclientVersions[version] = new_instance
+        new_instance
       end
+    end
+    KubeclientVersions = {}
 
+    # opts can have keys
+    #   kubernetes_client - already instantiated kubernetes client
+    def initialize(apiVersion)
       ssl_options  = {}
       auth_options = { bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token' }
       
@@ -37,7 +52,7 @@ module DTK
 
       @kubeclient = Kubeclient::Client.new(
         'https://kubernetes.default.svc/apis/',
-        'dtk.io/v1alpha1',
+        "dtk.io/#{apiVersion}",
         auth_options: auth_options,
         ssl_options:  ssl_options,
         socket_options: socket_options 

data/lib/persistence_providers.rb

@@ -1,10 +1,12 @@
 module DTK
   module State
+    require_relative 'utils'
     require_relative 'crd_client'
     require_relative 'state/crd_assembly'
     require_relative 'state/component'
     require_relative 'state/component_def'
     require_relative 'state/executable_action'
     require_relative 'state/workflow_instance'
+    require_relative 'state/workflow'
   end
 end

data/lib/state/component.rb

@@ -3,12 +3,14 @@ module DTK::State
     require_relative 'component/attribute'
 
     attr_reader :name, :parent, :component_def
+    attr_accessor :raw_attributes
 
     def initialize(name, component_content, parent, opts = {})
       @name           = name
       # @parent     = parent
       @component_defs = parent.references.component_def
       @component_def  = get_component_def(opts)
+      @raw_attributes = component_content[:attributes] || {}
 
       @attribute_objs = Attribute.create_from_kube_hash(component_content[:attributes] || {})# convert_to_attribute_objects(component_content[:attributes])
     end
@@ -17,11 +19,50 @@ module DTK::State
     #   task_id
     def self.get(crd_assembly_namespace, crd_assembly_name, component_name, opts = {})
       crd_assembly = CrdAssembly.get(crd_assembly_namespace, crd_assembly_name, opts)
-      if matching_component = crd_assembly.components.find{ |cmp| cmp.to_hash.keys.first.to_s == component_name }
-        Component.new(component_name, matching_component[component_name], crd_assembly, opts)
+      if matching_component = Component.find_matching_component(crd_assembly, component_name)
+        Component.new(component_name, Component.get_component_content(matching_component, component_name), crd_assembly, opts)
       end
     end
 
+    def self.get_with_influx_data(namespace, assembly_name, component_name, opts = {})
+      component = get(namespace, assembly_name, component_name, opts)
+      return unless component
+      attributes = component.raw_attributes.to_hash
+
+      attr_type_info = component.component_def.attribute_type_info
+      attr_type_info.each do |attr_info|
+        if attr_info.temporal
+          attribute_name = attr_info.name.to_sym
+          influxdb = ::DTK::State::Component::Attribute::Influxdb.new(:attributes)
+          influxdb_attribute = influxdb.get(namespace, component_name, assembly_name, attribute_name, opts)
+          if valid_attribute = influxdb_attribute.first
+            value = valid_attribute['_value']
+            if attributes[attribute_name]
+              if attributes[attribute_name].is_a?(String)
+                attributes[attribute_name] = value
+              else
+                attributes[attribute_name][:value] = value
+              end
+            end
+          end
+        end
+      end
+
+      component.raw_attributes = attributes
+      component
+    end
+
+    def to_hash
+      {
+        name: @name,
+        component_def: {
+          name: @component_def.name,
+          namespace: @component_def.namespace
+        },
+        attributes: @raw_attributes.to_hash
+      }
+    end
+
     # opts can have keys
     # task_id
     # format:
@@ -44,16 +85,64 @@ module DTK::State
       end
     end
 
+    def attribute_metadata
+      attributes         = @raw_attributes.to_hash
+      attr_type_info     = @component_def.attribute_type_info
+      attribute_metadata = {}
+
+      attr_type_info.each do |attr_info|
+        attr_info_hash = attr_info.to_hash
+        attribute_name = attr_info_hash[:name].to_sym
+
+        if attribute = attributes[attribute_name]
+          if attribute.is_a?(String)
+            attribute = { value: attribute }
+          end
+
+          attribute_metadata[attribute_name] = attr_info_hash.merge(attribute)
+        end
+      end
+
+      attribute_metadata
+    end
+
     def attribute_values
-      attribute_with_values = []
-      @content[:attributes].each do |name, content|
-        attribute_with_values << { name => content[:value] }
+      attribute_with_values = {}
+      @raw_attributes.each do |name, content|
+        attribute_with_values.merge!(name => content[:value])
       end
       attribute_with_values
     end
 
+    def self.create_from_kube_array(kube_components, crd_assembly)
+      kube_components.map do |component_hash|
+        if component_hash.is_a?(String)
+          component_name    = component_hash
+          component_content = {}
+        else
+          component_name = component_hash.to_hash.keys.first
+          component_content = component_hash[component_name]
+        end
+        Component.new(component_name, component_content, crd_assembly)
+      end
+    end
+
     private
 
+    def self.get_component_content(matching_component, component_name)
+      return matching_component.is_a?(String) ? {} : matching_component[component_name]
+    end
+
+    def self.find_matching_component(assembly, component_name)
+      assembly.components.find do |cmp|
+        if cmp.is_a? String
+          cmp == component_name
+        else
+          cmp.to_hash.keys.first.to_s == component_name
+        end
+      end
+    end
+
     def get_component_def(opts = {})
       destructured_component = destructure_component_full_name
       component_def = @component_defs.find { |component_def| component_def[:name] == destructured_component[:component_def_name] }

data/lib/state/component/providers/influxdb.rb

@@ -4,6 +4,7 @@ module DTK::State
       require_relative('influxdb/client')
       require_relative('influxdb/measurement')
       require_relative('influxdb/semantictype')
+      require_relative('../../../utils/log')
 
       attr_reader :client, :measurement
 
@@ -13,88 +14,51 @@ module DTK::State
       end
 
       def get(namespace, component_name, assembly_name, attribute_name, opts = {})
-        required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-        if opts[:provider] == "correlation"
-          errors = client.measurement_helper(:errors)
-          required_tags.merge!({ correlator_type: opts[:entrypoint].split("/").last.split(".")[0] })
-          errors.get_last_point(required_tags)
-        elsif
-          last_value = measurement.get_last_point(required_tags)
-          last_value
-        end
+        required_tags = measurement.get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags.merge! measurement.get_correlator_type(opts[:entrypoint]) if opts[:provider] == "correlation"
+        measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting attribute from InfluxDB.\nError: #{e}"
       end
 
       def write(namespace, component_name, assembly_name, attribute_name, value, opts = {}, timestamp = nil)
-        if opts[:provider] == "correlation"
-          errors = client.measurement_helper(:errors)
-          required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-          required_tags.merge!({ correlator_type: opts[:entrypoint].split("/").last.split(".")[0] })
-          errors.write(value.to_s, required_tags, timestamp)
-        elsif
-          required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-          measurement.write(value, required_tags, timestamp)
-        end
+        required_tags = measurement.get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags.merge! measurement.get_correlator_type(opts[:entrypoint]) if opts[:provider] == "correlation"
+        measurement.write(value.to_s, required_tags, timestamp)
+      rescue => e
+        raise "Error happened while writing attribute into InfluxDB.\Error: #{e}"
       end
 
       def write_event(event_id, pod_name, pod_namespace, event_source, event_message, component_name, attribute_name, task_id, timestamp)
-        begin
-          fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
-          value_to_write = { event_source: event_source, event_message: event_message }
-          required_tags = {
-            event_id: event_id,
-            pod_name: pod_name,
-            pod_namespace: pod_namespace,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
-          measurement.write(value_to_write.to_s, required_tags, timestamp)
-        rescue => error
-          fail error
-        end
+        fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
+        value_to_write = { event_source: event_source, event_message: event_message }
+        required_tags = measurement.get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
+        measurement.write(value_to_write.to_s, required_tags, timestamp)
+      rescue => error
+        raise "Error happened while writing event into InfluxDB.\nError: #{e}"
       end
 
       def get_event(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
-        required_tags = {
-            event_id: event_id,
-            pod_name: pod_name,
-            pod_namespace: pod_namespace,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
+        required_tags = measurement.get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
         last_point = measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting event from InfluxDB.\nError: #{e}"
       end
 
       def write_state(type, name, namespace, object_state, spec, status, component_name, attribute_name, task_id, timestamp)
-        begin
-          fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
-          value_to_write = { spec: spec, status: status }
-          required_tags = {
-            type: type,
-            name: name,
-            namespace: namespace,
-            object_state: object_state,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
-          measurement.write(value_to_write.to_s, required_tags, timestamp)
-        rescue => error
-          fail error
-        end
+        raise "Bad timestamp input, write operation to InfluxDB wont be completed" if timestamp > Time.new
+        value_to_write = { spec: spec, status: status }
+        required_tags = measurement.get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        measurement.write(value_to_write.to_s, required_tags, timestamp)
+      rescue => e
+        raise "Error happened while writing state into InfluxDB.\nError: #{e}"
       end
-      
-      private 
 
-      def get_required_tags(namespace, component_name, assembly_name, attribute_name)
-        required_tags = {
-          namespace: namespace,
-          component_name: component_name,
-          assembly_name: assembly_name,
-          attribute_name: attribute_name,
-          task_id: "1"
-        }
+      def get_state(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        required_tags = measurement.get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting state from InfluxDB.\nError: #{e}"
       end
     end
   end

data/lib/state/component/providers/influxdb/client.rb

@@ -10,30 +10,24 @@ module DTK::State
       end
 
       def query(query_expression)
-        begin
-          query_api = self.connection.create_query_api
-          query_api.query(query_expression)
-        rescue => error
-          fail "Failed in query"
-        end
+        query_api = self.connection.create_query_api
+        query_api.query(query_expression)
+      rescue => e
+        raise "Failed while processing flux query!. Error: #{e}"
       end
 
       def write_point(data)
-        begin
-          write_api = self.connection.create_write_api
-          write_api.write(data: data)
-        rescue => error
-          fail error
-        end
+        write_api = self.connection.create_write_api
+        write_api.write(data: data)
+      rescue => e
+        raise e
       end
 
       def measurement_helper(measurement_name)
-        begin
-          klass = Measurement.const_get(measurement_name.to_sym.capitalize)
-          klass.new(measurement_name, self)
-        rescue => error
-          fail error
-        end
+        klass = Measurement.const_get(measurement_name.to_sym.capitalize)
+        klass.new(measurement_name, self)
+      rescue => e
+        raise e
       end
 
       attr_reader :connection_parameters, :connection
@@ -53,18 +47,21 @@ module DTK::State
           org: params[:org],
           bucket: params[:bucket]
         }
+      rescue => e
+        raise "Problem happened while processing InfluxDB connection parameters. Error: #{e}"
       end
 
       def return_connection(connection_parameters)
-        begin
-          InfluxDB2::Client.new(connection_parameters[:url], connection_parameters[:token],
-                                bucket: connection_parameters[:bucket],
-                                org: connection_parameters[:org],
-                                precision: InfluxDB2::WritePrecision::MILLISECOND,
-                                use_ssl: false)
-        rescue => error
-          fail "Error: #{error}"
-        end
+        client = InfluxDB2::Client.new(connection_parameters[:url], connection_parameters[:token],
+                              bucket: connection_parameters[:bucket],
+                              org: connection_parameters[:org],
+                              precision: InfluxDB2::WritePrecision::MILLISECOND,
+                              use_ssl: false)
+        query_api = client.create_query_api
+        query_api.query(query: 'from(bucket:"' + connection_parameters[:bucket] + '") |> range(start: -5)')
+        client
+      rescue => e
+        raise "Connection with InfluxDB could not be established. #{e}"
       end
     end
   end

data/lib/state/component/providers/influxdb/measurement.rb

@@ -20,14 +20,22 @@ module DTK::State
       end
 
       def get_last_point(params_hash = {})
-        begin
-          check_params_hash(params_hash)
-          flux_query = 'from(bucket:"' + client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn: (r) => r._measurement == "' + name.to_s + '")' + flux_filter(params_hash) + ' |> last()'  +   '|> drop(columns: ["_start", "_stop", "_field", "_measurement", "attribute_name", "assembly_name", "task_id", "component_name", "namespace"])'
-          result  = self.client.query(query: flux_query)
-          result.values.map(&:records).flatten.map(&:values)
-        rescue => error
-          fail error
-        end
+        check_params_hash(params_hash)
+        flux_query = 'from(bucket:"' + client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn: (r) => r._measurement == "' + name.to_s + '")' + flux_filter(params_hash) + ' |> last()'  +   '|> drop(columns: ["_start", "_stop", "_field", "_measurement", "attribute_name", "assembly_name", "task_id", "component_name", "namespace"])'
+        result  = self.client.query(query: flux_query)
+        result.values.map(&:records).flatten.map(&:values)
+      rescue => e
+        raise "Failed while getting last attribute point. Error: #{e}"
+      end
+
+      def get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags = {
+          namespace: namespace,
+          component_name: component_name,
+          assembly_name: assembly_name,
+          attribute_name: attribute_name,
+          task_id: "1"
+        }
       end
       
       protected
@@ -90,7 +98,7 @@ module DTK::State
           fail "Parameter '#{name}' has an illegal type, legal types are #{LEGAL_TAG_CLASSES.join(', ')}"
         end
       end
-      
+
     end
   end
 end

data/lib/state/component/providers/influxdb/measurement/attribute_measurement.rb

@@ -2,7 +2,6 @@ module DTK::State
   class Component::Attribute::Influxdb
     class Measurement
       class Attributes < self
-
         def write(value, params_hash = {}, timestamp)
           checked_params_hash = check_params_hash(params_hash)
           write_point(value, checked_params_hash, timestamp)
@@ -11,10 +10,9 @@ module DTK::State
         protected
 
         def required_params
-          [:namespace, :component_name, :assembly_name, :attribute_name, :task_id]
+          %i[namespace component_name assembly_name attribute_name task_id]
         end
-
-      end                       
+      end
     end
   end
 end

data/lib/state/component/providers/influxdb/measurement/errors.rb

@@ -8,13 +8,18 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_correlator_type(entrypoint)
+          {
+            correlator_type: entrypoint.split('/').last.split('.')[0]
+          }
+        end
+
         protected
 
         def required_params
-          [:namespace, :component_name, :assembly_name, :task_id, :attribute_name, :task_id, :correlator_type]
+          %i[namespace component_name assembly_name task_id attribute_name correlator_type]
         end
-
-      end                       
+      end
     end
   end
-end
+end

data/lib/state/component/providers/influxdb/measurement/events.rb

@@ -8,13 +8,23 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
+          {
+            event_id: event_id,
+            pod_name: pod_name,
+            pod_namespace: pod_namespace,
+            component_name: component_name,
+            attribute_name: attribute_name,
+            task_id: task_id
+          }
+        end
+
         protected
 
         def required_params
-          [:event_id, :pod_name, :pod_namespace, :component_name, :attribute_name, :task_id]
+          %i[event_id pod_name pod_namespace component_name attribute_name task_id]
         end
-
-      end                       
+      end
     end
   end
-end
+end

data/lib/state/component/providers/influxdb/measurement/states.rb

@@ -8,13 +8,24 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+          {
+            type: type,
+            name: name,
+            namespace: namespace,
+            object_state: object_state,
+            component_name: component_name,
+            attribute_name: attribute_name,
+            task_id: task_id
+          }
+        end
+
         protected
 
         def required_params
-          [:type, :name, :namespace, :object_state, :component_name, :attribute_name, :task_id]
+          %i[type name namespace object_state component_name attribute_name task_id]
         end
-
-      end                       
+      end
     end
   end
-end
+end

data/lib/state/component/providers/influxdb/semantictype.rb

@@ -2,7 +2,8 @@ module DTK::State
   class Component::Attribute::Influxdb
     class SemanticType
       require_relative './client'
-      attr_reader :name, :crd_content, :namespace, :client
+      require_relative('../../../../utils/log')
+      attr_reader :name, :crd_content, :namespace, :client, :expanded_semantictype_spec
       attr_accessor :content_to_write
 
       def initialize(name, namespace)
@@ -11,75 +12,103 @@ module DTK::State
         @client = Client.new
         @crd_content = get(name, namespace)
         @content_to_write = []
-      end
-
-      # no namespace because semantictype instances are going to be unique in cluster
-      def get(name, namespace, opts = {})
-        begin
-          semantictype = ::DTK::CrdClient.get_kubeclient(opts).get_semantictype(name, namespace)
-          semantictype.spec[:openAPIV3Schema]
-        rescue => error
-          fail "SemanticType attribute with name '#{name}' not found on the cluster!. Error: #{error}"
-        end
+        @expanded_spec = ::DTK::CrdClient.get_kubeclient({}).get_semantictype(name, namespace).expandedSpec
+        @expanded_semantictype_spec = expand(crd_content.to_h[:properties], @expanded_spec)
       end
 
       def write_semantictype_inventory(inventory, component_id)
-        begin
-          get_influxdb_properties(inventory)
-          content_to_write.each do |point|
-            point[:tags].merge!({ component_id: component_id, attribute_name: @name })
-            @client.write_point({
-              name: point[:name],
-              tags: point[:tags],
-              fields: point[:fields],
-              time: (Time.new.to_f * 1000).to_i
-            })
-          end
-          "Inventory for attribute #{name} written to InfluxDB"
-        rescue => error
-          fail "#{name} inventory write failed. Error: #{error}"
+        get_influxdb_properties(inventory)
+        content_to_write.each do |point|
+          point[:tags].merge!({ component_id: component_id, attribute_name: @name })
+          @client.write_point({
+                                name: point[:name],
+                                tags: point[:tags],
+                                fields: point[:fields],
+                                time: (Time.new.to_f * 1000).to_i
+                              })
         end
+      rescue => e
+        raise "#{name} inventory write failed. Error: #{e}"
       end
 
       def partial_write_update(component_and_attribute, path, field_name, field_value)
         parent, child = validate_parameter(path)
         component_id, attribute_name = component_and_attribute.split('/')
         # getting previous value for given parameters
-        previous_value = { }
+        previous_value = {}
         begin
           flux_query = 'from(bucket:"' + @client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn:(r) => r._measurement == "' + attribute_name + "_" + child[:type] + '") |> filter(fn: (r) => r.parent == "' + parent[:name] + '") |> filter(fn: (r) => r.name == "' + child[:name] + '")|> last()'
           result = @client.query(query: flux_query)
           previous_value = result.values.map(&:records).flatten.map(&:values)
-        rescue => error
-          fail "Partial write could not be completed. Previous point for given parameters not found!"
+        rescue => e
+          raise "Partial write could not be completed. Previous point for given parameters not found!. Error: #{e}"
         end
         update_current(previous_value[0], get_path_to_object(path), field_name, field_value)
       end
 
       private
 
+      def expand(crd_content, expanded_spec)
+        if expanded_spec.nil?
+          expanded_spec = {}
+          populate_semantictypes(crd_content, expanded_spec)
+          ::DTK::CrdClient.get_kubeclient({}).merge_patch_semantictype(name, { expandedSpec: expanded_spec }, namespace)
+        end
+        expanded_spec
+      end
+
+      def get(name, namespace, opts = {})
+        semantictype = ::DTK::CrdClient.get_kubeclient(opts).get_semantictype(name, namespace)
+        semantictype.spec[:openAPIV3Schema]
+      rescue => e
+        raise "SemanticType attribute with name '#{name}' not found  on the cluster!. Error: #{e.inspect}"
+      end
+
+      def populate_semantictypes(crd, expanded_spec, parent = nil)
+        crd.each_pair do |key, value|
+          basic, semantictype = value[:type].split(':')
+          if semantictype
+            expanded_spec[key] = get(basic.to_s, namespace).to_h[:properties]
+            crd[key] = get(basic.to_s, namespace).to_h[:properties]
+            temporary_expanded_spec = populate_semantictypes(crd[key], expanded_spec[key], basic)
+          end
+          if basic == 'array'
+            basic, semantictype = value[:items][:type].split(':')
+            if semantictype
+              expanded_spec[key] = get(basic.to_s, namespace).to_h[:properties]
+              crd[key] = get(basic.to_s, namespace).to_h[:properties]
+              temporary_expanded_spec = populate_semantictypes(crd[key], expanded_spec[key], basic)
+            end
+          end
+          temporary_expanded_spec ||= {}
+          unless temporary_expanded_spec.empty? || parent.nil?
+            ::DTK::CrdClient.get_kubeclient({}).merge_patch_semantictype(parent, { expandedSpec: { key => temporary_expanded_spec } }, namespace)
+          end
+        end
+        expanded_spec
+      end
+
       def update_current(previous_value, path, field_name, field_value)
-        tags = { }
+        tags = {}
         previous_value.each_pair do |key, value|
-            tags[key] = value if key[0..0] != "_" && key != "result" && key != "table"
+          tags[key] = value if key[0..0] != '_' && key != 'result' && key != 'table'
         end
         fields = Hash.new
         fields[field_name.to_sym] = field_value
         validate_fields(get_partial_definition(path), fields)
         @client.write_point({
-          name: previous_value["_measurement"],
-          tags: tags,
-          fields: fields,
-          time: (Time.new.to_f * 1000).to_i
-        })
-        "Partial write update successful"
+                              name: previous_value['_measurement'],
+                              tags: tags,
+                              fields: fields,
+                              time: (Time.new.to_f * 1000).to_i
+                            })
       end
 
       def get_influxdb_properties(inventory, parent_type = [:top], parent_name = nil)
         content_to_write = []
         properties = { }
         inventory.each_pair do |key, value|
-          if value.class.to_s == "Array"
+          if value.class.to_s == 'Array'
             inventory[key].each do |element|
               get_influxdb_properties(element, parent_type.push(key), inventory[:name])
             end
@@ -102,12 +131,10 @@ module DTK::State
       end
 
       def validate_request(partial_definition, request)
-        begin
-          validate_tags(partial_definition, request[:tags])
-          validate_fields(partial_definition, request[:fields])
-        rescue => error
-          fail error  
-        end
+        validate_tags(partial_definition, request[:tags])
+        validate_fields(partial_definition, request[:fields])
+      rescue => e
+        raise e
       end
 
       def get_tags_and_fields(partial_definition, properties)
@@ -115,7 +142,7 @@ module DTK::State
         fields = { }
         properties.each_pair do |key, value|
           if partial_definition[key].nil?
-            fail "Property '#{key}' not found in the definition of attribute"
+            raise "Property '#{key}' not found in the definition of attribute"
           else
             if partial_definition[key][:metric].nil? || partial_definition[key][:metric] == false
               tags[key] = value
@@ -131,14 +158,12 @@ module DTK::State
       end
 
       def validate_fields(partial_definition, fields)
-
         partial_definition.each_pair do |key, value|
           next if key == :required || value[:metric] == (false || nil)
-      
           if fields[key].nil?
-            fail "Field #{key} is missing. Validation of request failed!"
+            raise "Field #{key} is missing. Validation of request failed!"
           elsif value[:type].capitalize != fields[key].class.to_s
-            fail "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{fields[key].class} provided"
+            raise "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{fields[key].class} provided"
           end
         end
       end
@@ -146,20 +171,19 @@ module DTK::State
       def validate_tags(partial_definition, tags)
         partial_definition.each_pair do |key, value|
           next if key == :required || value[:metric] == true
-          
           if tags[key].nil?
             if value[:default].nil?
-              fail "Property #{key} is missing. Validation of request failed!"
-            else 
+              raise "Property #{key} is missing. Validation of request failed!"
+            else
               tags[key] = value[:default]
             end
-          else 
+          else
             type = tags[key].class
-            type = "Boolean" if type == TrueClass || type == FalseClass
+            type = 'Boolean' if type == TrueClass || type == FalseClass
             if value[:type].capitalize == type.to_s
               next
             else
-              fail "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{type} provided"
+              raise "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{type} provided"
             end
           end
         end
@@ -167,24 +191,25 @@ module DTK::State
 
       def get_partial_definition(path)
         i = 0
-        definition = { }
-        semantictype_crd = crd_content[:properties]
+        definition = {}
+        semantictype_crd = crd_content.to_h[:properties]
+        expanded_spec = expanded_semantictype_spec.to_h
         while i < path.length
           if path[i].to_sym == :top
             semantictype_crd.each_pair do |key, value|
               if key == :required
                 definition[key] = value
               else
-                definition[key] = value if value[:type] != "array"
+                basic, semantictype = value[:type].split(':')
+                semantictype.nil? ? definition[key] = value : definition[key] = semantictype_crd[key] if basic != 'array'
               end
             end
           else
             definition = {}
-            definition[:required] = semantictype_crd[path[i].to_sym][:items][:required]
-            semantictype_crd[path[i].to_sym][:items][:properties].each_pair do |key, value|
-                definition[key] = value if value[:type] != "array"
+            expanded_spec[path[i].to_sym].each_pair do |key, value|
+              definition[key] = value unless value[:type].nil?
             end
-            semantictype_crd = semantictype_crd[path[i].to_sym][:items][:properties]
+            expanded_spec = expanded_spec[path[i].to_sym]
           end
           i+=1
         end
@@ -192,7 +217,7 @@ module DTK::State
       end
 
       def get_path_to_object(parameter)
-        path = ["top"]
+        path = ['top']
         array = parameter.split('/')
         array.each do |element|
           path.push(element.split(':')[1])
@@ -202,19 +227,18 @@ module DTK::State
 
       def validate_parameter(parameter)
         array_of_parameters = []
-        begin
-          parameter.split('/').each_with_index do |param, index|
-            name, type = param.split(':')
-            fail unless name && type
-            array_of_parameters.push({
-              name: name,
-              type: type
-            })
-          end
-          array_of_parameters
-        rescue => error
-          fail "Could not resolve parameter '#{parameter}'. It should be in format: 'parent:type/child:type'"
+        parameter.split('/').each_with_index do |param, index|
+          name, type = param.split(':')
+          raise unless name && type
+
+          array_of_parameters.push({
+                                      name: name,
+                                      type: type
+                                   })
         end
+        array_of_parameters
+      rescue => e
+        raise "Could not resolve parameter '#{parameter}'. It should be in format: 'parent:type/child:type'"
       end
     end
   end

data/lib/state/component_def.rb

@@ -2,16 +2,19 @@ module DTK::State
   class ComponentDef
     require_relative 'component_def/attribute_type_info'
 
+    COMPONENT_DEF_CRD_VERSION     = ENV["COMPONENT_DEF_CRD_VERSION"]
+
     attr_reader :name, :namespace, :executable_actions, :attribute_type_info
 
     def initialize(namespace, name, content)
       @name                = name
       @namespace           = namespace
-      @executable_actions  = content[:spec][:actions]
+      @executable_actions  = content[:spec][:actions] || {}
       @attribute_type_info  = AttributeTypeInfo.create_from_kube_hash(content[:spec][:attributes] || {})
     end
 
     def self.get(namespace, name, opts = {})
+      opts[:apiVersion] = COMPONENT_DEF_CRD_VERSION
       crd_component_def = ::DTK::CrdClient.get_kubeclient(opts).get_componentdef(name, namespace)
       ComponentDef.new(namespace, name, crd_component_def)
     end

data/lib/state/component_def/attribute_type_info.rb

@@ -19,6 +19,19 @@ module DTK::State
         end
       end
 
+      def to_hash
+        type_info = {}
+
+        type_info.merge!(name: @name) if @name
+        type_info.merge!(type: @type) if @type
+        type_info.merge!(required: @required) if @required
+        type_info.merge!(dynamic: @dynamic) if @dynamic
+        type_info.merge!(temporal: @temporal) if @temporal
+        type_info.merge!(encrypted: @encrypted) if @encrypted
+
+        type_info
+      end
+
     end
   end
 end

data/lib/state/crd_assembly.rb

@@ -1,17 +1,26 @@
 module DTK::State
   class CrdAssembly
-    attr_reader :name, :namespace, :crd_content, :components, :references
+    ASSEMBLY_CRD_VERSION = ENV["ASSEMBLY_CRD_VERSION"]
+
+    include DTK::Utils::CrdHelper
+
+    attr_reader :name, :namespace, :crd_content, :references, :component_objs
+    attr_accessor :components
 
     def initialize(namespace, name, crd_content)
-      @name        = name
-      @namespace   = namespace
-      @crd_content = crd_content
-      @references  = crd_content.references
-      @components  = crd_content[:spec][:components]
+      @name           = name
+      @namespace      = namespace
+      @api_version    = crd_content.apiVersion
+      @kind           = crd_content.kind
+      @metadata       = crd_content.metadata
+      # @crd_content    = crd_content
+      @references     = crd_content.references
+      @components     = crd_content[:spec][:components] || []
+      # @component_objs = Component.create_from_kube_array(@components, self)
     end
 
     def self.get(namespace, name, opts = {})
-      # crd_component = ::DTK::CrdClient.instance.kubeclient.get_component(name, namespace)
+      opts[:apiVersion] = ASSEMBLY_CRD_VERSION
       crd_assembly = ::DTK::CrdClient.get_kubeclient(opts).get_assembly(name, namespace)
       CrdAssembly.new(namespace, name, crd_assembly)
     end
@@ -50,6 +59,51 @@ module DTK::State
       output
     end
 
+    # TODO: this is a temporal solution to avoid breaking backward compatibility; will change this soon
+    def self.get_with_influx_data(namespace, assembly_name, opts = {})
+      assembly = get(namespace, assembly_name, opts)
+
+      components_hash = {}
+      assembly.components.each do |assembly_component|
+        cmp_name = assembly_component.to_hash.keys.first
+        components_hash[cmp_name] = assembly_component[cmp_name].to_hash
+      end
+
+      component_objs = Component.create_from_kube_array(assembly.components, assembly)
+
+      component_objs.each do |component_obj|
+        component_name = component_obj.name
+        attr_type_info = component_obj.component_def.attribute_type_info
+        attr_type_info.each do |attr_info|
+          if attr_info.temporal
+            attribute_name = attr_info.name
+            influxdb = ::DTK::State::Component::Attribute::Influxdb.new(:attributes)
+            influxdb_attribute = influxdb.get(namespace, component_name, assembly_name, attribute_name, opts)
+            if valid_attribute = influxdb_attribute.first
+              value = valid_attribute['_value']
+              if components_hash[component_name][:attributes][attribute_name].is_a?(String)
+                components_hash[component_name][:attributes][attribute_name] = value
+              else
+                components_hash[component_name][:attributes][attribute_name][:value] = value
+              end
+            end
+          end
+        end
+      end
+      assembly.components = components_hash
+      assembly
+    end
+
+    def to_hash
+      {
+        apiVersion: @api_version,
+        kind: @kind,
+        metadata: filter_metadata(@metadata),
+        references: @references.to_hash,
+        spec: { components: @components.to_hash }
+      }
+    end
+
     private
 
     def self.check_for_missing_components(crd_components, requested_components)

data/lib/state/workflow.rb

@@ -0,0 +1,26 @@
+module DTK::State
+  class Workflow
+    require_relative 'workflow_instance/attribute_type_info'
+
+    WORKFLOW_CRD_VERSION = ENV["WORKFLOW_CRD_VERSION"]
+
+    attr_reader :name, :namespace, :references, :assembly, :workflow, :attribute_type_info
+
+    def initialize(namespace, name, crd_content)
+      @name      = name
+      @namespace = namespace
+
+      @references = crd_content.references
+      @assembly   = @references.assembly
+      @workflow   = crd_content.spec.workflow || {}
+
+      @attribute_type_info = ::DTK::State::WorkflowInstance::AttributeTypeInfo.create_from_kube_hash(crd_content.spec.attributes.to_h || {})
+    end
+
+    def self.get(namespace, name, opts = {})
+      opts[:apiVersion] = WORKFLOW_CRD_VERSION
+      workflow = ::DTK::CrdClient.get_kubeclient(opts).get_workflow(name, namespace)
+      Workflow.new(namespace, name, workflow)
+    end
+  end
+end

data/lib/state/workflow_instance.rb

@@ -2,23 +2,77 @@ module DTK::State
   class WorkflowInstance
     require_relative 'workflow_instance/attribute_type_info'
 
-    attr_reader :name, :namespace, :assembly, :workflow_template, :attributes, :workflow,  :attribute_type_info
+    WORKFLOW_INSTANCE_CRD_VERSION = ENV["WORKFLOW_INSTANCE_CRD_VERSION"]
+
+    include DTK::Utils::CrdHelper
+
+    attr_reader :name, :namespace, :assembly, :workflow_template, :attributes, :attribute_type_info, :status, :workflow
+    attr_accessor :workflow
 
     def initialize(namespace, name, crd_content)
       @name        = name
       @namespace   = namespace
-      @assembly          = crd_content.references.assembly
-      @workflow_template = crd_content.references.workflow
-      @attributes        = crd_content.spec.attributes
-      @workflow          = crd_content.spec.workflow
-      @attribute_type_info = AttributeTypeInfo.create_from_kube_hash(@attributes.to_h || {})
+
+      @api_version    = crd_content.apiVersion
+      @kind           = crd_content.kind
+      @metadata       = crd_content.metadata
+
+      @references        = crd_content.references
+      @assembly          = @references.assembly
+      @workflow_template = @references.workflow
+
+      @attributes        = crd_content.spec.attributes || {}
+      @status            = crd_content.spec.status || {}
+      @workflow          = crd_content.spec.workflow || {}
     end
 
     def self.get(namespace, name, opts = {})
+      opts[:apiVersion] = WORKFLOW_INSTANCE_CRD_VERSION
       workflow_instance = ::DTK::CrdClient.get_kubeclient(opts).get_workflow_instance(name, namespace)
       WorkflowInstance.new(namespace, name, workflow_instance)
     end
 
+    def self.get_with_influx_data(namespace, workflow_instance_name, opts = {})
+      workflow_instance = get(namespace, workflow_instance_name, opts)
+      return unless workflow_instance
+
+      workflow_instance.workflow[:subtasks].each do |subtask|
+        component_name, action_name = subtask[:component].split('.')
+        assembly_name = workflow_instance.assembly[:name]
+
+        executable_action = ::DTK::State::ExecutableAction.get(namespace, assembly_name, component_name, action_name, opts)
+        attr_type_info = executable_action.attribute_type_info
+
+        attr_type_info.each do |attr_info|
+          if attr_info.temporal
+            attribute_name = attr_info.name
+            influxdb = ::DTK::State::Component::Attribute::Influxdb.new(:attributes)
+            influxdb_attribute = influxdb.get(namespace, component_name, assembly_name, attribute_name, opts)
+            if valid_attribute = influxdb_attribute.first
+              value = valid_attribute['_value']
+              subtask[:attributes][attribute_name] = value
+            end
+          end
+        end
+      end
+
+      workflow_instance
+    end
+
+    def to_hash
+      {
+        apiVersion: @api_version,
+        kind: @kind,
+        metadata: filter_metadata(@metadata),
+        references: @references.to_hash,
+        spec: {
+          attributes: @attributes.to_hash,
+          status: @status.to_hash,
+          workflow: @workflow.to_hash
+        }
+      }
+    end
+
     def self.get_attributes(namespace, name, opts = {})
       workflow_instance = get(namespace, name, opts)
       workflow_instance.attributes.to_h
@@ -35,6 +89,7 @@ module DTK::State
     def self.update_action_level_result_attributes(namespace, name, attributes, action_id, opts = {})
       return "Dynamic attributes do not exist for action with id #{@action_id}, nothing to update" if attributes.nil? || attributes.empty?
       attributes.delete_if { |key, value| value.nil? || value.to_s.strip == '' }
+      opts[:apiVersion] = WORKFLOW_INSTANCE_CRD_VERSION
       workflow_instance = ::DTK::CrdClient.get_kubeclient(opts).get_workflow_instance(name, namespace)
       workflow = workflow_instance[:spec][:workflow]
 
@@ -63,6 +118,7 @@ module DTK::State
     end
 
     def self.update_action_status(namespace, name, parent_id, action_id, status, error_message = "", opts = {})
+      opts[:apiVersion] = WORKFLOW_INSTANCE_CRD_VERSION
       workflow_instance = ::DTK::CrdClient.get_kubeclient(opts).get_workflow_instance(name, namespace)
       steps = workflow_instance[:spec][:status][:steps]
       action_index_steps = steps.find_index { |action| action[:id].eql? action_id }
@@ -90,5 +146,38 @@ module DTK::State
       action
     end
 
+    def get_workflow_template(opts = {})
+      Workflow.get(@workflow_template.namespace, @workflow_template.name, opts)
+    end
+
+    def attribute_metadata
+      attributes         = @attributes.to_hash
+      attr_type_info     = get_workflow_template.attribute_type_info
+      attribute_metadata = {}
+
+      attr_type_info.each do |attr_info|
+        attr_info_hash = attr_info.to_hash
+        attribute_name = attr_info_hash[:name].to_sym
+
+        if attribute = attributes[attribute_name]
+          if attribute.is_a?(String)
+            attribute = { value: attribute }
+          end
+
+          attribute_metadata[attribute_name] = attr_info_hash.merge(attribute)
+        end
+      end
+
+      attribute_metadata
+    end
+
+    def attribute_values
+      attribute_with_values = {}
+      @attributes.each_pair do |name, content|
+        attribute_with_values.merge!(name => content[:value])
+      end
+      attribute_with_values
+    end
+
   end
 end

data/lib/state/workflow_instance/attribute_type_info.rb

@@ -21,6 +21,19 @@ module DTK::State
         end
       end
 
+      def to_hash
+        type_info = {}
+
+        type_info.merge!(name: @name) if @name
+        type_info.merge!(type: @type) if @type
+        type_info.merge!(required: @required) if @required
+        type_info.merge!(dynamic: @dynamic) if @dynamic
+        type_info.merge!(temporal: @temporal) if @temporal
+        type_info.merge!(encrypted: @encrypted) if @encrypted
+
+        type_info
+      end
+
     end
   end
 end

data/lib/utils.rb

@@ -0,0 +1,6 @@
+module DTK
+  module Utils
+    require_relative 'utils/log'
+    require_relative 'utils/crd_helper'
+  end
+end

data/lib/utils/crd_helper.rb

@@ -0,0 +1,9 @@
+module DTK::Utils
+  module CrdHelper
+    def filter_metadata(metadata)
+      metadata_hash = metadata.to_hash.dup
+      metadata_hash[:annotations].delete(:"kubectl.kubernetes.io/last-applied-configuration")
+      metadata_hash
+    end
+  end
+end

data/lib/utils/log.rb

@@ -0,0 +1,22 @@
+module DTK::Utils
+  module Log
+    require 'logger'
+
+    def self.instance
+      @instance ||= Logger.new('/proc/1/fd/1', formatter: proc { |severity, datetime, progname, msg| 
+        orange_color = "\x1b[33m"
+        white_color = "\x1b[37m"
+        red_color = "\x1b[31m"
+
+        date_format = datetime.strftime("%Y-%m-%d %H:%M:%S:%L")
+        if severity == "INFO"
+          "#{orange_color}[#{date_format}] - #{white_color}#{msg}\n"
+        elsif severity == "WARN"
+          "#{orange_color}[#{date_format}] [WARNING] - #{msg}\n"
+        elsif severity == "ERROR"       
+            "#{red_color}[#{date_format}] [ERROR] - #{msg}\n"
+        end
+      })
+    end 
+  end
+end

data/persistence-providers.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name        = 'persistence-providers'
-  spec.version     = '0.0.3.7'
+  spec.version     = '0.0.6'
   spec.author      = 'Reactor8'
   spec.email       = 'support@reactor8.com'
   spec.description = %q{Persistence providers plugin}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: persistence-providers
 version: !ruby/object:Gem::Version
-  version: 0.0.3.7
+  version: 0.0.6
 platform: ruby
 authors:
 - Reactor8
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-14 00:00:00.000000000 Z
+date: 2020-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kubeclient
@@ -77,8 +77,12 @@ files:
 - lib/state/crd_assembly.rb
 - lib/state/executable_action.rb
 - lib/state/executable_action/attribute_type_info.rb
+- lib/state/workflow.rb
 - lib/state/workflow_instance.rb
 - lib/state/workflow_instance/attribute_type_info.rb
+- lib/utils.rb
+- lib/utils/crd_helper.rb
+- lib/utils/log.rb
 - persistence-providers.gemspec
 - test-destroy-influxdb.rb
 - test-influxdb.rb
@@ -102,7 +106,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Persistence providers plugin