persistence-providers 0.0.3.6 → 0.0.5

data/lib/state/component/providers/influxdb/measurement/attribute_measurement.rb

@@ -2,7 +2,6 @@ module DTK::State
   class Component::Attribute::Influxdb
     class Measurement
       class Attributes < self
-
         def write(value, params_hash = {}, timestamp)
           checked_params_hash = check_params_hash(params_hash)
           write_point(value, checked_params_hash, timestamp)
@@ -11,10 +10,9 @@ module DTK::State
         protected
 
         def required_params
-          [:namespace, :component_name, :assembly_name, :attribute_name, :task_id]
+          %i[namespace component_name assembly_name attribute_name task_id]
         end
-
-      end                       
+      end
     end
   end
 end

data/lib/state/component/providers/influxdb/measurement/errors.rb

@@ -8,13 +8,18 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_correlator_type(entrypoint)
+          {
+            correlator_type: entrypoint.split('/').last.split('.')[0]
+          }
+        end
+
         protected
 
         def required_params
-          [:namespace, :component_name, :assembly_name, :task_id, :attribute_name, :task_id, :correlator_type]
+          %i[namespace component_name assembly_name task_id attribute_name correlator_type]
         end
-
-      end                       
+      end
     end
   end
-end
+end

data/lib/state/component/providers/influxdb/measurement/events.rb

@@ -8,13 +8,23 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_required_tags(event_id, pod_name, pod_namespace, component_name, attribute_name, task_id)
+          {
+            event_id: event_id,
+            pod_name: pod_name,
+            pod_namespace: pod_namespace,
+            component_name: component_name,
+            attribute_name: attribute_name,
+            task_id: task_id
+          }
+        end
+
         protected
 
         def required_params
-          [:event_id, :pod_name, :pod_namespace, :component_name, :attribute_name, :task_id]
+          %i[event_id pod_name pod_namespace component_name attribute_name task_id]
         end
-
-      end                       
+      end
     end
   end
-end
+end

data/lib/state/component/providers/influxdb/measurement/states.rb

@@ -8,13 +8,24 @@ module DTK::State
           write_point(value, checked_params_hash, timestamp)
         end
 
+        def get_required_tags(type, name, namespace, object_state, component_name, attribute_name, task_id)
+          {
+            type: type,
+            name: name,
+            namespace: namespace,
+            object_state: object_state,
+            component_name: component_name,
+            attribute_name: attribute_name,
+            task_id: task_id
+          }
+        end
+
         protected
 
         def required_params
-          [:type, :name, :namespace, :object_state, :component_name, :attribute_name, :task_id]
+          %i[type name namespace object_state component_name attribute_name task_id]
         end
-
-      end                       
+      end
     end
   end
-end
+end

data/lib/state/component/providers/influxdb/semantictype.rb

@@ -2,7 +2,8 @@ module DTK::State
   class Component::Attribute::Influxdb
     class SemanticType
       require_relative './client'
-      attr_reader :name, :crd_content, :namespace, :client
+      require_relative('../../../../utils/log')
+      attr_reader :name, :crd_content, :namespace, :client, :expanded_semantictype_spec
       attr_accessor :content_to_write
 
       def initialize(name, namespace)
@@ -11,75 +12,103 @@ module DTK::State
         @client = Client.new
         @crd_content = get(name, namespace)
         @content_to_write = []
-      end
-
-      # no namespace because semantictype instances are going to be unique in cluster
-      def get(name, namespace, opts = {})
-        begin
-          semantictype = ::DTK::CrdClient.get_kubeclient(opts).get_semantictype(name, namespace)
-          semantictype.spec[:openAPIV3Schema]
-        rescue => error
-          fail "SemanticType attribute with name '#{name}' not found on the cluster!. Error: #{error}"
-        end
+        @expanded_spec = ::DTK::CrdClient.get_kubeclient({}).get_semantictype(name, namespace).expandedSpec
+        @expanded_semantictype_spec = expand(crd_content.to_h[:properties], @expanded_spec)
       end
 
       def write_semantictype_inventory(inventory, component_id)
-        begin
-          get_influxdb_properties(inventory)
-          content_to_write.each do |point|
-            point[:tags].merge!({ component_id: component_id, attribute_name: @name })
-            @client.write_point({
-              name: point[:name],
-              tags: point[:tags],
-              fields: point[:fields],
-              time: (Time.new.to_f * 1000).to_i
-            })
-          end
-          "Inventory for attribute #{name} written to InfluxDB"
-        rescue => error
-          fail "#{name} inventory write failed. Error: #{error}"
+        get_influxdb_properties(inventory)
+        content_to_write.each do |point|
+          point[:tags].merge!({ component_id: component_id, attribute_name: @name })
+          @client.write_point({
+                                name: point[:name],
+                                tags: point[:tags],
+                                fields: point[:fields],
+                                time: (Time.new.to_f * 1000).to_i
+                              })
         end
+      rescue => e
+        raise "#{name} inventory write failed. Error: #{e}"
       end
 
       def partial_write_update(component_and_attribute, path, field_name, field_value)
         parent, child = validate_parameter(path)
         component_id, attribute_name = component_and_attribute.split('/')
         # getting previous value for given parameters
-        previous_value = { }
+        previous_value = {}
         begin
           flux_query = 'from(bucket:"' + @client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn:(r) => r._measurement == "' + attribute_name + "_" + child[:type] + '") |> filter(fn: (r) => r.parent == "' + parent[:name] + '") |> filter(fn: (r) => r.name == "' + child[:name] + '")|> last()'
           result = @client.query(query: flux_query)
           previous_value = result.values.map(&:records).flatten.map(&:values)
-        rescue => error
-          fail "Partial write could not be completed. Previous point for given parameters not found!"
+        rescue => e
+          raise "Partial write could not be completed. Previous point for given parameters not found!. Error: #{e}"
         end
         update_current(previous_value[0], get_path_to_object(path), field_name, field_value)
       end
 
       private
 
+      def expand(crd_content, expanded_spec)
+        if expanded_spec.nil?
+          expanded_spec = {}
+          populate_semantictypes(crd_content, expanded_spec)
+          ::DTK::CrdClient.get_kubeclient({}).merge_patch_semantictype(name, { expandedSpec: expanded_spec }, namespace)
+        end
+        expanded_spec
+      end
+
+      def get(name, namespace, opts = {})
+        semantictype = ::DTK::CrdClient.get_kubeclient(opts).get_semantictype(name, namespace)
+        semantictype.spec[:openAPIV3Schema]
+      rescue => e
+        raise "SemanticType attribute with name '#{name}' not found  on the cluster!. Error: #{e.inspect}"
+      end
+
+      def populate_semantictypes(crd, expanded_spec, parent = nil)
+        crd.each_pair do |key, value|
+          basic, semantictype = value[:type].split(':')
+          if semantictype
+            expanded_spec[key] = get(basic.to_s, namespace).to_h[:properties]
+            crd[key] = get(basic.to_s, namespace).to_h[:properties]
+            temporary_expanded_spec = populate_semantictypes(crd[key], expanded_spec[key], basic)
+          end
+          if basic == 'array'
+            basic, semantictype = value[:items][:type].split(':')
+            if semantictype
+              expanded_spec[key] = get(basic.to_s, namespace).to_h[:properties]
+              crd[key] = get(basic.to_s, namespace).to_h[:properties]
+              temporary_expanded_spec = populate_semantictypes(crd[key], expanded_spec[key], basic)
+            end
+          end
+          temporary_expanded_spec ||= {}
+          unless temporary_expanded_spec.empty? || parent.nil?
+            ::DTK::CrdClient.get_kubeclient({}).merge_patch_semantictype(parent, { expandedSpec: { key => temporary_expanded_spec } }, namespace)
+          end
+        end
+        expanded_spec
+      end
+
       def update_current(previous_value, path, field_name, field_value)
-        tags = { }
+        tags = {}
         previous_value.each_pair do |key, value|
-            tags[key] = value if key[0..0] != "_" && key != "result" && key != "table"
+          tags[key] = value if key[0..0] != '_' && key != 'result' && key != 'table'
         end
         fields = Hash.new
         fields[field_name.to_sym] = field_value
         validate_fields(get_partial_definition(path), fields)
         @client.write_point({
-          name: previous_value["_measurement"],
-          tags: tags,
-          fields: fields,
-          time: (Time.new.to_f * 1000).to_i
-        })
-        "Partial write update successful"
+                              name: previous_value['_measurement'],
+                              tags: tags,
+                              fields: fields,
+                              time: (Time.new.to_f * 1000).to_i
+                            })
       end
 
       def get_influxdb_properties(inventory, parent_type = [:top], parent_name = nil)
         content_to_write = []
         properties = { }
         inventory.each_pair do |key, value|
-          if value.class.to_s == "Array"
+          if value.class.to_s == 'Array'
             inventory[key].each do |element|
               get_influxdb_properties(element, parent_type.push(key), inventory[:name])
             end
@@ -102,12 +131,10 @@ module DTK::State
       end
 
       def validate_request(partial_definition, request)
-        begin
-          validate_tags(partial_definition, request[:tags])
-          validate_fields(partial_definition, request[:fields])
-        rescue => error
-          fail error  
-        end
+        validate_tags(partial_definition, request[:tags])
+        validate_fields(partial_definition, request[:fields])
+      rescue => e
+        raise e
       end
 
       def get_tags_and_fields(partial_definition, properties)
@@ -115,7 +142,7 @@ module DTK::State
         fields = { }
         properties.each_pair do |key, value|
           if partial_definition[key].nil?
-            fail "Property '#{key}' not found in the definition of attribute"
+            raise "Property '#{key}' not found in the definition of attribute"
           else
             if partial_definition[key][:metric].nil? || partial_definition[key][:metric] == false
               tags[key] = value
@@ -131,14 +158,12 @@ module DTK::State
       end
 
       def validate_fields(partial_definition, fields)
-
         partial_definition.each_pair do |key, value|
           next if key == :required || value[:metric] == (false || nil)
-      
           if fields[key].nil?
-            fail "Field #{key} is missing. Validation of request failed!"
+            raise "Field #{key} is missing. Validation of request failed!"
           elsif value[:type].capitalize != fields[key].class.to_s
-            fail "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{fields[key].class} provided"
+            raise "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{fields[key].class} provided"
           end
         end
       end
@@ -146,20 +171,19 @@ module DTK::State
       def validate_tags(partial_definition, tags)
         partial_definition.each_pair do |key, value|
           next if key == :required || value[:metric] == true
-          
           if tags[key].nil?
             if value[:default].nil?
-              fail "Property #{key} is missing. Validation of request failed!"
-            else 
+              raise "Property #{key} is missing. Validation of request failed!"
+            else
               tags[key] = value[:default]
             end
-          else 
+          else
             type = tags[key].class
-            type = "Boolean" if type == TrueClass || type == FalseClass
+            type = 'Boolean' if type == TrueClass || type == FalseClass
             if value[:type].capitalize == type.to_s
               next
             else
-              fail "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{type} provided"
+              raise "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{type} provided"
             end
           end
         end
@@ -167,24 +191,25 @@ module DTK::State
 
       def get_partial_definition(path)
         i = 0
-        definition = { }
-        semantictype_crd = crd_content[:properties]
+        definition = {}
+        semantictype_crd = crd_content.to_h[:properties]
+        expanded_spec = expanded_semantictype_spec.to_h
         while i < path.length
           if path[i].to_sym == :top
             semantictype_crd.each_pair do |key, value|
               if key == :required
                 definition[key] = value
               else
-                definition[key] = value if value[:type] != "array"
+                basic, semantictype = value[:type].split(':')
+                semantictype.nil? ? definition[key] = value : definition[key] = semantictype_crd[key] if basic != 'array'
               end
             end
           else
             definition = {}
-            definition[:required] = semantictype_crd[path[i].to_sym][:items][:required]
-            semantictype_crd[path[i].to_sym][:items][:properties].each_pair do |key, value|
-                definition[key] = value if value[:type] != "array"
+            expanded_spec[path[i].to_sym].each_pair do |key, value|
+              definition[key] = value unless value[:type].nil?
             end
-            semantictype_crd = semantictype_crd[path[i].to_sym][:items][:properties]
+            expanded_spec = expanded_spec[path[i].to_sym]
           end
           i+=1
         end
@@ -192,7 +217,7 @@ module DTK::State
       end
 
       def get_path_to_object(parameter)
-        path = ["top"]
+        path = ['top']
         array = parameter.split('/')
         array.each do |element|
           path.push(element.split(':')[1])
@@ -202,19 +227,18 @@ module DTK::State
 
       def validate_parameter(parameter)
         array_of_parameters = []
-        begin
-          parameter.split('/').each_with_index do |param, index|
-            name, type = param.split(':')
-            fail unless name && type
-            array_of_parameters.push({
-              name: name,
-              type: type
-            })
-          end
-          array_of_parameters
-        rescue => error
-          fail "Could not resolve parameter '#{parameter}'. It should be in format: 'parent:type/child:type'"
+        parameter.split('/').each_with_index do |param, index|
+          name, type = param.split(':')
+          raise unless name && type
+
+          array_of_parameters.push({
+                                      name: name,
+                                      type: type
+                                   })
         end
+        array_of_parameters
+      rescue => e
+        raise "Could not resolve parameter '#{parameter}'. It should be in format: 'parent:type/child:type'"
       end
     end
   end

data/lib/state/component_def.rb

@@ -2,16 +2,19 @@ module DTK::State
   class ComponentDef
     require_relative 'component_def/attribute_type_info'
 
+    COMPONENT_DEF_CRD_VERSION     = ENV["COMPONENT_DEF_CRD_VERSION"]
+
     attr_reader :name, :namespace, :executable_actions, :attribute_type_info
 
     def initialize(namespace, name, content)
       @name                = name
       @namespace           = namespace
-      @executable_actions  = content[:spec][:actions]
+      @executable_actions  = content[:spec][:actions] || {}
       @attribute_type_info  = AttributeTypeInfo.create_from_kube_hash(content[:spec][:attributes] || {})
     end
 
     def self.get(namespace, name, opts = {})
+      opts[:apiVersion] = COMPONENT_DEF_CRD_VERSION
       crd_component_def = ::DTK::CrdClient.get_kubeclient(opts).get_componentdef(name, namespace)
       ComponentDef.new(namespace, name, crd_component_def)
     end

data/lib/state/component_def/attribute_type_info.rb

@@ -19,6 +19,19 @@ module DTK::State
         end
       end
 
+      def to_hash
+        type_info = {}
+
+        type_info.merge!(name: @name) if @name
+        type_info.merge!(type: @type) if @type
+        type_info.merge!(required: @required) if @required
+        type_info.merge!(dynamic: @dynamic) if @dynamic
+        type_info.merge!(temporal: @temporal) if @temporal
+        type_info.merge!(encrypted: @encrypted) if @encrypted
+
+        type_info
+      end
+
     end
   end
 end

data/lib/state/crd_assembly.rb

@@ -1,17 +1,24 @@
 module DTK::State
   class CrdAssembly
-    attr_reader :name, :namespace, :crd_content, :components, :references
+    ASSEMBLY_CRD_VERSION = ENV["ASSEMBLY_CRD_VERSION"]
+
+    attr_reader :name, :namespace, :crd_content, :references, :component_objs
+    attr_accessor :components
 
     def initialize(namespace, name, crd_content)
-      @name        = name
-      @namespace   = namespace
-      @crd_content = crd_content
-      @references  = crd_content.references
-      @components  = crd_content[:spec][:components]
+      @name           = name
+      @namespace      = namespace
+      @api_version    = crd_content.apiVersion
+      @kind           = crd_content.kind
+      @metadata       = crd_content.metadata
+      # @crd_content    = crd_content
+      @references     = crd_content.references
+      @components     = crd_content[:spec][:components] || []
+      # @component_objs = Component.create_from_kube_array(@components, self)
     end
 
     def self.get(namespace, name, opts = {})
-      # crd_component = ::DTK::CrdClient.instance.kubeclient.get_component(name, namespace)
+      opts[:apiVersion] = ASSEMBLY_CRD_VERSION
       crd_assembly = ::DTK::CrdClient.get_kubeclient(opts).get_assembly(name, namespace)
       CrdAssembly.new(namespace, name, crd_assembly)
     end
@@ -50,6 +57,51 @@ module DTK::State
       output
     end
 
+    # TODO: this is a temporal solution to avoid breaking backward compatibility; will change this soon
+    def self.get_with_influx_data(namespace, assembly_name, opts = {})
+      assembly = get(namespace, assembly_name, opts)
+
+      components_hash = {}
+      assembly.components.each do |assembly_component|
+        cmp_name = assembly_component.to_hash.keys.first
+        components_hash[cmp_name] = assembly_component[cmp_name].to_hash
+      end
+
+      component_objs = Component.create_from_kube_array(assembly.components, assembly)
+
+      component_objs.each do |component_obj|
+        component_name = component_obj.name
+        attr_type_info = component_obj.component_def.attribute_type_info
+        attr_type_info.each do |attr_info|
+          if attr_info.temporal
+            attribute_name = attr_info.name
+            influxdb = ::DTK::State::Component::Attribute::Influxdb.new(:attributes)
+            influxdb_attribute = influxdb.get(namespace, component_name, assembly_name, attribute_name, opts)
+            if valid_attribute = influxdb_attribute.first
+              value = valid_attribute['_value']
+              if components_hash[component_name][:attributes][attribute_name].is_a?(String)
+                components_hash[component_name][:attributes][attribute_name] = value
+              else
+                components_hash[component_name][:attributes][attribute_name][:value] = value
+              end
+            end
+          end
+        end
+      end
+      assembly.components = components_hash
+      assembly
+    end
+
+    def to_hash
+      {
+        apiVersion: @api_version,
+        kind: @kind,
+        metadata: @metadata.to_hash,
+        references: @references.to_hash,
+        spec: { components: @components.to_hash }
+      }
+    end
+
     private
 
     def self.check_for_missing_components(crd_components, requested_components)