persistence-providers 0.0.3.5 → 0.0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 62b3d49ea7a138d6f1f9e99e0b8f776dcc0212ea24e8fba1bc35a2876f466951
-  data.tar.gz: 4fecccfff86f73b5b9ab5f647294bb5b866b465bfcc2f59c1991722da5cc0b68
+  metadata.gz: 9bc555b9367087540ccddc49bc1ce3fa937b0a36be7b3d6ff832373a9beeadd0
+  data.tar.gz: 5ae84500a46b183544d53b1d7d2f90cf6b49bcf52d34d2b95eba8474f7653c93
 SHA512:
-  metadata.gz: 7be9867552f0c0cc2cf66cdea3295b6c02c41909d92c645178244b256b4a67079f595ead4a48c91c33a116ba4323498001665ea9e089d68b37941c1618111a7e
-  data.tar.gz: 7bc22d498ec484b7466853e5ca550554f6567769596417b41f086e515fb58ac20b62b2074638cebee00a7f0c5af396dce8f68684b098374407d673338d6749e7
+  metadata.gz: 52cc878735fdf04cb19b7a202ff2622872b4514f40c1b849a0ff73a821f4e8e333c1aa9a3ac6ba6b31de29b199f4c49fb5fbd3df2c2bdf64fd4d421f6ff67e7d
+  data.tar.gz: a9b7fbf25e02a5b1dc15220760e91ca1bc703e7f1c2f388f6f2b083c48170c1162ffc483464cbde99f31e2465dd687c78bc47006a5ad9ee6157c5c40538083e9

data/lib/crd_client.rb

@@ -1,10 +1,16 @@
 require 'kubeclient'
 require 'celluloid/io'
-require 'singleton'
+# require 'singleton'
 
 module DTK
   class CrdClient
-    include Singleton
+    # include Singleton
+    DEFAULT_API_VERSION = 'v1alpha1'
+
+    # COMPONENT_DEF_CRD_VERSION     = ENV["COMPONENT_DEF_CRD_VERSION"]
+    # ASSEMBLY_CRD_VERSION          = ENV["ASSEMBLY_CRD_VERSION"]
+    # WORKFLOW_CRD_VERSION          = ENV["WORKFLOW_CRD_VERSION"]
+    # WORKFLOW_INSTANCE_CRD_VERSION = ENV["WORKFLOW_INSTANCE_CRD_VERSION"]
 
     attr_accessor :kubeclient
 
@@ -12,17 +18,26 @@ module DTK
       if @kubeclient = opts[:kubeclient]
         @kubeclient
       else
-        ::DTK::CrdClient.instance.kubeclient
+        kubeclient_version(opts)
       end
     end
 
-    # opts can have keys
-    #   kubernetes_client - already instantiated kubernetes client
-    def initialize(opts = {})
-      if @kubeclient = opts[:kubernetes_client]
-        return @kubeclient
+    def self.kubeclient_version(opts = {})
+      version = opts[:apiVersion] || DEFAULT_API_VERSION
+
+      if existing_version = KubeclientVersions[version]
+        return existing_version
+      else
+        new_instance = new(version).kubeclient
+        KubeclientVersions[version] = new_instance
+        new_instance
       end
+    end
+    KubeclientVersions = {}
 
+    # opts can have keys
+    #   kubernetes_client - already instantiated kubernetes client
+    def initialize(apiVersion)
       ssl_options  = {}
       auth_options = { bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token' }
       
@@ -37,7 +52,7 @@ module DTK
 
       @kubeclient = Kubeclient::Client.new(
         'https://kubernetes.default.svc/apis/',
-        'dtk.io/v1alpha1',
+        "dtk.io/#{apiVersion}",
         auth_options: auth_options,
         ssl_options:  ssl_options,
         socket_options: socket_options 

data/lib/persistence_providers.rb

@@ -1,5 +1,6 @@
 module DTK
   module State
+    require_relative 'utils'
     require_relative 'crd_client'
     require_relative 'state/crd_assembly'
     require_relative 'state/component'

data/lib/state/component.rb

@@ -17,8 +17,8 @@ module DTK::State
     #   task_id
     def self.get(crd_assembly_namespace, crd_assembly_name, component_name, opts = {})
       crd_assembly = CrdAssembly.get(crd_assembly_namespace, crd_assembly_name, opts)
-      if matching_component = crd_assembly.components.find{ |cmp| cmp.to_hash.keys.first.to_s == component_name }
-        Component.new(component_name, matching_component[component_name], crd_assembly, opts)
+      if matching_component = Component.find_matching_component(crd_assembly, component_name)
+        Component.new(component_name, Component.get_component_content(matching_component, component_name), crd_assembly, opts)
       end
     end
 
@@ -54,6 +54,20 @@ module DTK::State
 
     private
 
+    def self.get_component_content(matching_component, component_name)
+      return matching_component.is_a?(String) ? {} : matching_component[component_name]
+    end
+
+    def self.find_matching_component(assembly, component_name)
+      assembly.components.find do |cmp|
+        if cmp.is_a? String
+          cmp == component_name
+        else
+          cmp.to_hash.keys.first.to_s == component_name
+        end
+      end
+    end
+
     def get_component_def(opts = {})
       destructured_component = destructure_component_full_name
       component_def = @component_defs.find { |component_def| component_def[:name] == destructured_component[:component_def_name] }

data/lib/state/component/providers/influxdb.rb

@@ -3,6 +3,7 @@ module DTK::State
     class Influxdb < self
       require_relative('influxdb/client')
       require_relative('influxdb/measurement')
+      require_relative('influxdb/semantictype')
 
       attr_reader :client, :measurement
 
@@ -12,88 +13,51 @@ module DTK::State
       end
 
       def get(namespace, component_name, assembly_name, attribute_name, opts = {})
-        required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-        if opts[:provider] == "correlation"
-          errors = client.measurement_helper(:errors)
-          required_tags.merge!({ correlator_type: opts[:entrypoint].split("/").last.split(".")[0] })
-          errors.get_last_point(required_tags)
-        elsif
-          last_value = measurement.get_last_point(required_tags)
-          last_value
-        end
+        required_tags = measurement.get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags.merge! measurement.get_correlator_type(opts[:entrypoint]) if opts[:provider] == "correlation"
+        measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting attribute from InfluxDB.\nError: #{e}"
       end
 
       def write(namespace, component_name, assembly_name, attribute_name, value, opts = {}, timestamp = nil)
-        if opts[:provider] == "correlation"
-          errors = client.measurement_helper(:errors)
-          required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-          required_tags.merge!({ correlator_type: opts[:entrypoint].split("/").last.split(".")[0] })
-          errors.write(value.to_s, required_tags, timestamp)
-        elsif
-          required_tags = get_required_tags(namespace, component_name, assembly_name, attribute_name)
-          measurement.write(value, required_tags, timestamp)
-        end
+        required_tags = measurement.get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags.merge! measurement.get_correlator_type(opts[:entrypoint]) if opts[:provider] == "correlation"
+        measurement.write(value.to_s, required_tags, timestamp)
+      rescue => e
+        raise "Error happened while writing attribute into InfluxDB.\Error: #{e}"
       end
 
       def write_event(event_id, pod_name, pod_namespace, event_source, event_message, component_name, attribute_name, task_id, timestamp)
-        begin
-          fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
-          value_to_write = { event_source: event_source, event_message: event_message }
-          required_tags = {
-            event_id: event_id,
-            pod_name: pod_name,
-            pod_namespace: pod_namespace,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
-          measurement.write(value_to_write.to_s, required_tags, timestamp)
-        rescue => error
-          puts error
-        end
+        fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
+        value_to_write = { event_source: event_source, event_message: event_message }
+        required_tags = measurement.get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
+        measurement.write(value_to_write.to_s, required_tags, timestamp)
+      rescue => error
+        raise "Error happened while writing event into InfluxDB.\nError: #{e}"
       end
 
       def get_event(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
-        required_tags = {
-            event_id: event_id,
-            pod_name: pod_name,
-            pod_namespace: pod_namespace,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
+        required_tags = measurement.get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
         last_point = measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting event from InfluxDB.\nError: #{e}"
       end
 
       def write_state(type, name, namespace, object_state, spec, status, component_name, attribute_name, task_id, timestamp)
-        begin
-          fail "Bad timestamp input, write operation wont be completed" if timestamp > Time.new
-          value_to_write = { spec: spec, status: status }
-          required_tags = {
-            type: type,
-            name: name,
-            namespace: namespace,
-            object_state: object_state,
-            component_name: component_name,
-            attribute_name: attribute_name,
-            task_id: task_id
-          }
-          measurement.write(value_to_write.to_s, required_tags, timestamp)
-        rescue => error
-          puts error
-        end
+        raise "Bad timestamp input, write operation to InfluxDB wont be completed" if timestamp > Time.new
+        value_to_write = { spec: spec, status: status }
+        required_tags = measurement.get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        measurement.write(value_to_write.to_s, required_tags, timestamp)
+      rescue => e
+        raise "Error happened while writing state into InfluxDB.\nError: #{e}"
       end
-      
-      private 
 
-      def get_required_tags(namespace, component_name, assembly_name, attribute_name)
-        required_tags = {
-          namespace: namespace,
-          component_name: component_name,
-          assembly_name: assembly_name,
-          attribute_name: attribute_name,
-          task_id: "1"
-        }
+      def get_state(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        required_tags = measurement.get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+        measurement.get_last_point(required_tags)
+      rescue => e
+        raise "Error happened while getting state from InfluxDB.\nError: #{e}"
       end
     end
   end

data/lib/state/component/providers/influxdb/client.rb

@@ -12,24 +12,22 @@ module DTK::State
       def query(query_expression)
         query_api = self.connection.create_query_api
         query_api.query(query_expression)
+      rescue => e
+        raise "Failed while processing flux query!. Error: #{e}"
       end
 
       def write_point(data)
-        begin
-          write_api = self.connection.create_write_api
-          write_api.write(data: data)
-        rescue => error
-          fail error
-        end
+        write_api = self.connection.create_write_api
+        write_api.write(data: data)
+      rescue => e
+        raise e
       end
 
       def measurement_helper(measurement_name)
-        begin
-          klass = Measurement.const_get(measurement_name.to_sym.capitalize)
-          klass.new(measurement_name, self)
-        rescue => error
-          puts error
-        end
+        klass = Measurement.const_get(measurement_name.to_sym.capitalize)
+        klass.new(measurement_name, self)
+      rescue => e
+        raise e
       end
 
       attr_reader :connection_parameters, :connection
@@ -49,18 +47,21 @@ module DTK::State
           org: params[:org],
           bucket: params[:bucket]
         }
+      rescue => e
+        raise "Problem happened while processing InfluxDB connection parameters. Error: #{e}"
       end
 
       def return_connection(connection_parameters)
-        begin
-          InfluxDB2::Client.new(connection_parameters[:url], connection_parameters[:token],
-                                bucket: connection_parameters[:bucket],
-                                org: connection_parameters[:org],
-                                precision: InfluxDB2::WritePrecision::MILLISECOND,
-                                use_ssl: false)
-        rescue => error
-          fail "Error: #{error}"
-        end
+        client = InfluxDB2::Client.new(connection_parameters[:url], connection_parameters[:token],
+                              bucket: connection_parameters[:bucket],
+                              org: connection_parameters[:org],
+                              precision: InfluxDB2::WritePrecision::MILLISECOND,
+                              use_ssl: false)
+        query_api = client.create_query_api
+        query_api.query(query: 'from(bucket:"' + connection_parameters[:bucket] + '") |> range(start: -5)')
+        client
+      rescue => e
+        raise "Connection with InfluxDB could not be established. #{e}"
       end
     end
   end

data/lib/state/component/providers/influxdb/measurement.rb

@@ -20,14 +20,22 @@ module DTK::State
       end
 
       def get_last_point(params_hash = {})
-        begin
-          check_params_hash(params_hash)
-          flux_query = 'from(bucket:"' + client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn: (r) => r._measurement == "' + name.to_s + '")' + flux_filter(params_hash) + ' |> last()'  +   '|> drop(columns: ["_start", "_stop", "_field", "_measurement", "attribute_name", "assembly_name", "task_id", "component_name", "namespace"])'
-          result  = self.client.query(flux_query)
-          result.values.map(&:records).flatten.map(&:values)
-        rescue => error
-          fail error
-        end
+        check_params_hash(params_hash)
+        flux_query = 'from(bucket:"' + client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn: (r) => r._measurement == "' + name.to_s + '")' + flux_filter(params_hash) + ' |> last()'  +   '|> drop(columns: ["_start", "_stop", "_field", "_measurement", "attribute_name", "assembly_name", "task_id", "component_name", "namespace"])'
+        result  = self.client.query(query: flux_query)
+        result.values.map(&:records).flatten.map(&:values)
+      rescue => e
+        raise "Failed while getting last attribute point. Error: #{e}"
+      end
+
+      def get_required_tags(namespace, component_name, assembly_name, attribute_name)
+        required_tags = {
+          namespace: namespace,
+          component_name: component_name,
+          assembly_name: assembly_name,
+          attribute_name: attribute_name,
+          task_id: "1"
+        }
       end
       
       protected
@@ -90,7 +98,7 @@ module DTK::State
           fail "Parameter '#{name}' has an illegal type, legal types are #{LEGAL_TAG_CLASSES.join(', ')}"
         end
       end
-      
+
     end
   end
 end

data/lib/state/component/providers/influxdb/measurement/errors.rb

@@ -8,6 +8,12 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_correlator_type(entrypoint)
+          { 
+            correlator_type: entrypoint.split("/").last.split(".")[0] 
+          }
+        end
+
         protected
 
         def required_params

data/lib/state/component/providers/influxdb/measurement/events.rb

@@ -8,12 +8,23 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
+          {
+            event_id: event_id,
+            pod_name: pod_name,
+            pod_namespace: pod_namespace,
+            component_name: component_name,
+            attribute_name: attribute_name,
+            task_id: task_id
+          }
+        end
+
         protected
 
         def required_params
           [:event_id, :pod_name, :pod_namespace, :component_name, :attribute_name, :task_id]
         end
-
+        
       end                       
     end
   end

data/lib/state/component/providers/influxdb/measurement/states.rb

@@ -8,12 +8,24 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+          {
+            type: type,
+            name: name,
+            namespace: namespace,
+            object_state: object_state,
+            component_name: component_name,
+            attribute_name: attribute_name,
+            task_id: task_id
+          }
+        end
+
         protected
 
         def required_params
           [:type, :name, :namespace, :object_state, :component_name, :attribute_name, :task_id]
         end
-
+        
       end                       
     end
   end

data/lib/state/component/providers/influxdb/semantictype.rb

@@ -0,0 +1,221 @@
+module DTK::State
+  class Component::Attribute::Influxdb
+    class SemanticType
+      require_relative './client'
+      attr_reader :name, :crd_content, :namespace, :client
+      attr_accessor :content_to_write
+
+      def initialize(name, namespace)
+        @name        = name
+        @namespace   = namespace
+        @client = Client.new
+        @crd_content = get(name, namespace)
+        @content_to_write = []
+      end
+
+      # no namespace because semantictype instances are going to be unique in cluster
+      def get(name, namespace, opts = {})
+        begin
+          semantictype = ::DTK::CrdClient.get_kubeclient(opts).get_semantictype(name, namespace)
+          semantictype.spec[:openAPIV3Schema]
+        rescue => error
+          fail "SemanticType attribute with name '#{name}' not found on the cluster!. Error: #{error}"
+        end
+      end
+
+      def write_semantictype_inventory(inventory, component_id)
+        begin
+          get_influxdb_properties(inventory)
+          content_to_write.each do |point|
+            point[:tags].merge!({ component_id: component_id, attribute_name: @name })
+            @client.write_point({
+              name: point[:name],
+              tags: point[:tags],
+              fields: point[:fields],
+              time: (Time.new.to_f * 1000).to_i
+            })
+          end
+          "Inventory for attribute #{name} written to InfluxDB"
+        rescue => error
+          fail "#{name} inventory write failed. Error: #{error}"
+        end
+      end
+
+      def partial_write_update(component_and_attribute, path, field_name, field_value)
+        parent, child = validate_parameter(path)
+        component_id, attribute_name = component_and_attribute.split('/')
+        # getting previous value for given parameters
+        previous_value = { }
+        begin
+          flux_query = 'from(bucket:"' + @client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn:(r) => r._measurement == "' + attribute_name + "_" + child[:type] + '") |> filter(fn: (r) => r.parent == "' + parent[:name] + '") |> filter(fn: (r) => r.name == "' + child[:name] + '")|> last()'
+          result = @client.query(query: flux_query)
+          previous_value = result.values.map(&:records).flatten.map(&:values)
+        rescue => error
+          fail "Partial write could not be completed. Previous point for given parameters not found!"
+        end
+        update_current(previous_value[0], get_path_to_object(path), field_name, field_value)
+      end
+
+      private
+
+      def update_current(previous_value, path, field_name, field_value)
+        tags = { }
+        previous_value.each_pair do |key, value|
+            tags[key] = value if key[0..0] != "_" && key != "result" && key != "table"
+        end
+        fields = Hash.new
+        fields[field_name.to_sym] = field_value
+        validate_fields(get_partial_definition(path), fields)
+        @client.write_point({
+          name: previous_value["_measurement"],
+          tags: tags,
+          fields: fields,
+          time: (Time.new.to_f * 1000).to_i
+        })
+        "Partial write update successful"
+      end
+
+      def get_influxdb_properties(inventory, parent_type = [:top], parent_name = nil)
+        content_to_write = []
+        properties = { }
+        inventory.each_pair do |key, value|
+          if value.class.to_s == "Array"
+            inventory[key].each do |element|
+              get_influxdb_properties(element, parent_type.push(key), inventory[:name])
+            end
+          else
+            properties[key] = value
+          end
+        end
+        resolve_property(parent_type, parent_name, properties)
+        parent_type.pop
+        "Attribute successfully validated!"
+      end
+
+      def resolve_property(parent_type, parent_name, properties)
+        definition = get_partial_definition(parent_type)
+        request = get_tags_and_fields(definition, properties)
+        validate_request(definition, request)
+        request[:name] = name + "_" + parent_type.last.to_s
+        request[:tags][:parent] = parent_name unless parent_name.nil?
+        content_to_write.push(request)
+      end
+
+      def validate_request(partial_definition, request)
+        begin
+          validate_tags(partial_definition, request[:tags])
+          validate_fields(partial_definition, request[:fields])
+        rescue => error
+          fail error  
+        end
+      end
+
+      def get_tags_and_fields(partial_definition, properties)
+        tags = { }
+        fields = { }
+        properties.each_pair do |key, value|
+          if partial_definition[key].nil?
+            fail "Property '#{key}' not found in the definition of attribute"
+          else
+            if partial_definition[key][:metric].nil? || partial_definition[key][:metric] == false
+              tags[key] = value
+            else
+              fields[key] = value
+            end
+          end
+        end
+        {
+          tags: tags,
+          fields: fields
+        }
+      end
+
+      def validate_fields(partial_definition, fields)
+
+        partial_definition.each_pair do |key, value|
+          next if key == :required || value[:metric] == (false || nil)
+      
+          if fields[key].nil?
+            fail "Field #{key} is missing. Validation of request failed!"
+          elsif value[:type].capitalize != fields[key].class.to_s
+            fail "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{fields[key].class} provided"
+          end
+        end
+      end
+
+      def validate_tags(partial_definition, tags)
+        partial_definition.each_pair do |key, value|
+          next if key == :required || value[:metric] == true
+          
+          if tags[key].nil?
+            if value[:default].nil?
+              fail "Property #{key} is missing. Validation of request failed!"
+            else 
+              tags[key] = value[:default]
+            end
+          else 
+            type = tags[key].class
+            type = "Boolean" if type == TrueClass || type == FalseClass
+            if value[:type].capitalize == type.to_s
+              next
+            else
+              fail "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{type} provided"
+            end
+          end
+        end
+      end
+
+      def get_partial_definition(path)
+        i = 0
+        definition = { }
+        semantictype_crd = crd_content[:properties]
+        while i < path.length
+          if path[i].to_sym == :top
+            semantictype_crd.each_pair do |key, value|
+              if key == :required
+                definition[key] = value
+              else
+                definition[key] = value if value[:type] != "array"
+              end
+            end
+          else
+            definition = {}
+            definition[:required] = semantictype_crd[path[i].to_sym][:items][:required]
+            semantictype_crd[path[i].to_sym][:items][:properties].each_pair do |key, value|
+                definition[key] = value if value[:type] != "array"
+            end
+            semantictype_crd = semantictype_crd[path[i].to_sym][:items][:properties]
+          end
+          i+=1
+        end
+        definition
+      end
+
+      def get_path_to_object(parameter)
+        path = ["top"]
+        array = parameter.split('/')
+        array.each do |element|
+          path.push(element.split(':')[1])
+        end
+        path
+      end
+
+      def validate_parameter(parameter)
+        array_of_parameters = []
+        begin
+          parameter.split('/').each_with_index do |param, index|
+            name, type = param.split(':')
+            fail unless name && type
+            array_of_parameters.push({
+              name: name,
+              type: type
+            })
+          end
+          array_of_parameters
+        rescue => error
+          fail "Could not resolve parameter '#{parameter}'. It should be in format: 'parent:type/child:type'"
+        end
+      end
+    end
+  end
+end

data/lib/state/component_def.rb

@@ -2,6 +2,8 @@ module DTK::State
   class ComponentDef
     require_relative 'component_def/attribute_type_info'
 
+    COMPONENT_DEF_CRD_VERSION     = ENV["COMPONENT_DEF_CRD_VERSION"]
+
     attr_reader :name, :namespace, :executable_actions, :attribute_type_info
 
     def initialize(namespace, name, content)
@@ -12,6 +14,7 @@ module DTK::State
     end
 
     def self.get(namespace, name, opts = {})
+      opts[:apiVersion] = COMPONENT_DEF_CRD_VERSION
       crd_component_def = ::DTK::CrdClient.get_kubeclient(opts).get_componentdef(name, namespace)
       ComponentDef.new(namespace, name, crd_component_def)
     end

data/lib/state/crd_assembly.rb

@@ -1,5 +1,7 @@
 module DTK::State
   class CrdAssembly
+    ASSEMBLY_CRD_VERSION = ENV["ASSEMBLY_CRD_VERSION"]
+
     attr_reader :name, :namespace, :crd_content, :components, :references
 
     def initialize(namespace, name, crd_content)
@@ -11,7 +13,7 @@ module DTK::State
     end
 
     def self.get(namespace, name, opts = {})
-      # crd_component = ::DTK::CrdClient.instance.kubeclient.get_component(name, namespace)
+      opts[:apiVersion] = ASSEMBLY_CRD_VERSION
       crd_assembly = ::DTK::CrdClient.get_kubeclient(opts).get_assembly(name, namespace)
       CrdAssembly.new(namespace, name, crd_assembly)
     end

data/lib/state/workflow_instance.rb

@@ -2,6 +2,8 @@ module DTK::State
   class WorkflowInstance
     require_relative 'workflow_instance/attribute_type_info'
 
+    WORKFLOW_INSTANCE_CRD_VERSION = ENV["WORKFLOW_INSTANCE_CRD_VERSION"]
+
     attr_reader :name, :namespace, :assembly, :workflow_template, :attributes, :workflow,  :attribute_type_info
 
     def initialize(namespace, name, crd_content)
@@ -15,6 +17,7 @@ module DTK::State
     end
 
     def self.get(namespace, name, opts = {})
+      opts[:apiVersion] = WORKFLOW_INSTANCE_CRD_VERSION
       workflow_instance = ::DTK::CrdClient.get_kubeclient(opts).get_workflow_instance(name, namespace)
       WorkflowInstance.new(namespace, name, workflow_instance)
     end
@@ -26,13 +29,58 @@ module DTK::State
 
     def self.get_action_attributes(namespace, name, action_id, opts = {})
       workflow_instance = get(namespace, name, opts)
-      action = workflow_instance.find_action(action_id)
+      action = WorkflowInstance.find_action(action_id, workflow_instance.workflow)
       return nil unless action
       attributes = action[:attributes] || {}
       attributes.to_h
     end
 
-    def find_action(id, workflow = @workflow)
+    def self.update_action_level_result_attributes(namespace, name, attributes, action_id, opts = {})
+      return "Dynamic attributes do not exist for action with id #{@action_id}, nothing to update" if attributes.nil? || attributes.empty?
+      attributes.delete_if { |key, value| value.nil? || value.to_s.strip == '' }
+      opts[:apiVersion] = WORKFLOW_INSTANCE_CRD_VERSION
+      workflow_instance = ::DTK::CrdClient.get_kubeclient(opts).get_workflow_instance(name, namespace)
+      workflow = workflow_instance[:spec][:workflow]
+
+      action = WorkflowInstance.find_action(action_id, workflow)
+      action[:attributes] = {} if !action[:attributes]
+      attributes.each do |attr_name, attr_val|
+        action[:attributes][attr_name.to_sym] = {} unless action[:attributes][attr_name.to_sym]
+        unless action[:attributes][attr_name.to_sym][:hidden]
+          if attr_val.is_a? Hash
+            action[:attributes][attr_name.to_sym][:value] = attr_val[:value] || attr_val
+          else
+            action[:attributes][attr_name.to_sym][:value] = attr_val
+          end
+        end
+      end
+      ::DTK::CrdClient.get_kubeclient(opts).update_workflow_instance(workflow_instance)
+    end
+
+    def self.patchError!(patches, message, action_index_steps)
+      errorPatch = {
+        "op" => "add",
+        "path" => "/spec/status/steps/#{action_index_steps}/errorMsg",
+        "value" => message
+      }
+      patches << errorPatch
+    end
+
+    def self.update_action_status(namespace, name, parent_id, action_id, status, error_message = "", opts = {})
+      opts[:apiVersion] = WORKFLOW_INSTANCE_CRD_VERSION
+      workflow_instance = ::DTK::CrdClient.get_kubeclient(opts).get_workflow_instance(name, namespace)
+      steps = workflow_instance[:spec][:status][:steps]
+      action_index_steps = steps.find_index { |action| action[:id].eql? action_id }
+      patch = [{
+        "op" => "replace",
+        "path" => "/spec/status/steps/#{action_index_steps}/state",
+        "value" => status
+      }]
+      patchError!(patch, error_message, action_index_steps) unless error_message.empty? || error_message.nil?
+      ::DTK::CrdClient.get_kubeclient(opts).json_patch_workflow_instance(name, patch, namespace)
+    end
+
+    def self.find_action(id, workflow = @workflow)
       action = nil
       subtasks = workflow[:subtasks]
       subtasks.each do |subtask|

data/lib/utils.rb

@@ -0,0 +1,3 @@
+module Utils
+  require_relative 'utils/log'
+end

data/lib/utils/log.rb

@@ -0,0 +1,22 @@
+module Utils
+  module Log
+    require 'logger'
+
+    def self.instance
+      @instance ||= Logger.new('/proc/1/fd/1', formatter: proc { |severity, datetime, progname, msg| 
+        orange_color = "\x1b[33m"
+        white_color = "\x1b[37m"
+        red_color = "\x1b[31m"
+
+        date_format = datetime.strftime("%Y-%m-%d %H:%M:%S:%L")
+        if severity == "INFO"
+          "#{orange_color}[#{date_format}] - #{white_color}#{msg}\n"
+        elsif severity == "WARN"
+          "#{orange_color}[#{date_format}] [WARNING] - #{msg}\n"
+        elsif severity == "ERROR"       
+            "#{red_color}[#{date_format}] [ERROR] - #{msg}\n"
+        end
+      })
+    end 
+  end
+end

data/persistence-providers.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name        = 'persistence-providers'
-  spec.version     = '0.0.3.5'
+  spec.version     = '0.0.4.1'
   spec.author      = 'Reactor8'
   spec.email       = 'support@reactor8.com'
   spec.description = %q{Persistence providers plugin}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: persistence-providers
 version: !ruby/object:Gem::Version
-  version: 0.0.3.5
+  version: 0.0.4.1
 platform: ruby
 authors:
 - Reactor8
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-12 00:00:00.000000000 Z
+date: 2020-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kubeclient
@@ -70,6 +70,7 @@ files:
 - lib/state/component/providers/influxdb/measurement/errors.rb
 - lib/state/component/providers/influxdb/measurement/events.rb
 - lib/state/component/providers/influxdb/measurement/states.rb
+- lib/state/component/providers/influxdb/semantictype.rb
 - lib/state/component/providers/kube_crd.rb
 - lib/state/component_def.rb
 - lib/state/component_def/attribute_type_info.rb
@@ -78,6 +79,8 @@ files:
 - lib/state/executable_action/attribute_type_info.rb
 - lib/state/workflow_instance.rb
 - lib/state/workflow_instance/attribute_type_info.rb
+- lib/utils.rb
+- lib/utils/log.rb
 - persistence-providers.gemspec
 - test-destroy-influxdb.rb
 - test-influxdb.rb