persistence-providers 0.0.3.5 → 0.0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 62b3d49ea7a138d6f1f9e99e0b8f776dcc0212ea24e8fba1bc35a2876f466951
-  data.tar.gz: 4fecccfff86f73b5b9ab5f647294bb5b866b465bfcc2f59c1991722da5cc0b68
+  metadata.gz: a18d9fbe5a1507dfde899dc0bf467a6148ae7bda7ed58995854ecc372ef4efa5
+  data.tar.gz: 32cf0fa8b1433d55c7f79da8767ed7de1b8d1daa66f8c1e409d6b8978914c938
 SHA512:
-  metadata.gz: 7be9867552f0c0cc2cf66cdea3295b6c02c41909d92c645178244b256b4a67079f595ead4a48c91c33a116ba4323498001665ea9e089d68b37941c1618111a7e
-  data.tar.gz: 7bc22d498ec484b7466853e5ca550554f6567769596417b41f086e515fb58ac20b62b2074638cebee00a7f0c5af396dce8f68684b098374407d673338d6749e7
+  metadata.gz: bebb1b072b268c17fff5e2430e37b4fa8dd771cbbdb10930d3e7cf852c008d1d4d49c5f30a5fcce8afa9930da6e06707c9c7bfe3d58f12c13c2101ecf55ec15a
+  data.tar.gz: 05126ad2c6f92ab855877e71929cf00f6955e21873b6782a9805c05884ec6e0e08ad53bdb7314589039acef82b96abf8bf376557252314aacefda8359c3c4070

data/lib/state/component/providers/influxdb.rb

@@ -3,6 +3,7 @@ module DTK::State
     class Influxdb < self
       require_relative('influxdb/client')
       require_relative('influxdb/measurement')
+      require_relative('influxdb/semantictype')
 
       attr_reader :client, :measurement
 
@@ -49,7 +50,7 @@ module DTK::State
           }
           measurement.write(value_to_write.to_s, required_tags, timestamp)
         rescue => error
-          puts error
+          fail error
         end
       end
 
@@ -80,7 +81,7 @@ module DTK::State
           }
           measurement.write(value_to_write.to_s, required_tags, timestamp)
         rescue => error
-          puts error
+          fail error
         end
       end
       

data/lib/state/component/providers/influxdb/client.rb

@@ -10,8 +10,12 @@ module DTK::State
       end
 
       def query(query_expression)
-        query_api = self.connection.create_query_api
-        query_api.query(query_expression)
+        begin
+          query_api = self.connection.create_query_api
+          query_api.query(query_expression)
+        rescue => error
+          fail "Failed in query"
+        end
       end
 
       def write_point(data)
@@ -28,7 +32,7 @@ module DTK::State
           klass = Measurement.const_get(measurement_name.to_sym.capitalize)
           klass.new(measurement_name, self)
         rescue => error
-          puts error
+          fail error
         end
       end
 

data/lib/state/component/providers/influxdb/measurement.rb

@@ -23,7 +23,7 @@ module DTK::State
         begin
           check_params_hash(params_hash)
           flux_query = 'from(bucket:"' + client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn: (r) => r._measurement == "' + name.to_s + '")' + flux_filter(params_hash) + ' |> last()'  +   '|> drop(columns: ["_start", "_stop", "_field", "_measurement", "attribute_name", "assembly_name", "task_id", "component_name", "namespace"])'
-          result  = self.client.query(flux_query)
+          result  = self.client.query(query: flux_query)
           result.values.map(&:records).flatten.map(&:values)
         rescue => error
           fail error

data/lib/state/component/providers/influxdb/semantictype.rb

@@ -0,0 +1,221 @@
+module DTK::State
+  class Component::Attribute::Influxdb
+    class SemanticType
+      require_relative './client'
+      attr_reader :name, :crd_content, :namespace, :client
+      attr_accessor :content_to_write
+
+      def initialize(name, namespace)
+        @name        = name
+        @namespace   = namespace
+        @client = Client.new
+        @crd_content = get(name, namespace)
+        @content_to_write = []
+      end
+
+      # no namespace because semantictype instances are going to be unique in cluster
+      def get(name, namespace, opts = {})
+        begin
+          semantictype = ::DTK::CrdClient.get_kubeclient(opts).get_semantictype(name, namespace)
+          semantictype.spec[:openAPIV3Schema]
+        rescue => error
+          fail "SemanticType attribute with name '#{name}' not found on the cluster!. Error: #{error}"
+        end
+      end
+
+      def write_semantictype_inventory(inventory, component_id)
+        begin
+          get_influxdb_properties(inventory)
+          content_to_write.each do |point|
+            point[:tags].merge!({ component_id: component_id, attribute_name: @name })
+            @client.write_point({
+              name: point[:name],
+              tags: point[:tags],
+              fields: point[:fields],
+              time: (Time.new.to_f * 1000).to_i
+            })
+          end
+          "Inventory for attribute #{name} written to InfluxDB"
+        rescue => error
+          fail "#{name} inventory write failed. Error: #{error}"
+        end
+      end
+
+      def partial_write_update(component_and_attribute, path, field_name, field_value)
+        parent, child = validate_parameter(path)
+        component_id, attribute_name = component_and_attribute.split('/')
+        # getting previous value for given parameters
+        previous_value = { }
+        begin
+          flux_query = 'from(bucket:"' + @client.connection_parameters[:bucket] + '") |> range(start:-5) |> filter(fn:(r) => r._measurement == "' + attribute_name + "_" + child[:type] + '") |> filter(fn: (r) => r.parent == "' + parent[:name] + '") |> filter(fn: (r) => r.name == "' + child[:name] + '")|> last()'
+          result = @client.query(query: flux_query)
+          previous_value = result.values.map(&:records).flatten.map(&:values)
+        rescue => error
+          fail "Partial write could not be completed. Previous point for given parameters not found!"
+        end
+        update_current(previous_value[0], get_path_to_object(path), field_name, field_value)
+      end
+
+      private
+
+      def update_current(previous_value, path, field_name, field_value)
+        tags = { }
+        previous_value.each_pair do |key, value|
+            tags[key] = value if key[0..0] != "_" && key != "result" && key != "table"
+        end
+        fields = Hash.new
+        fields[field_name.to_sym] = field_value
+        validate_fields(get_partial_definition(path), fields)
+        @client.write_point({
+          name: previous_value["_measurement"],
+          tags: tags,
+          fields: fields,
+          time: (Time.new.to_f * 1000).to_i
+        })
+        "Partial write update successful"
+      end
+
+      def get_influxdb_properties(inventory, parent_type = [:top], parent_name = nil)
+        content_to_write = []
+        properties = { }
+        inventory.each_pair do |key, value|
+          if value.class.to_s == "Array"
+            inventory[key].each do |element|
+              get_influxdb_properties(element, parent_type.push(key), inventory[:name])
+            end
+          else
+            properties[key] = value
+          end
+        end
+        resolve_property(parent_type, parent_name, properties)
+        parent_type.pop
+        "Attribute successfully validated!"
+      end
+
+      def resolve_property(parent_type, parent_name, properties)
+        definition = get_partial_definition(parent_type)
+        request = get_tags_and_fields(definition, properties)
+        validate_request(definition, request)
+        request[:name] = name + "_" + parent_type.last.to_s
+        request[:tags][:parent] = parent_name unless parent_name.nil?
+        content_to_write.push(request)
+      end
+
+      def validate_request(partial_definition, request)
+        begin
+          validate_tags(partial_definition, request[:tags])
+          validate_fields(partial_definition, request[:fields])
+        rescue => error
+          fail error  
+        end
+      end
+
+      def get_tags_and_fields(partial_definition, properties)
+        tags = { }
+        fields = { }
+        properties.each_pair do |key, value|
+          if partial_definition[key].nil?
+            fail "Property '#{key}' not found in the definition of attribute"
+          else
+            if partial_definition[key][:metric].nil? || partial_definition[key][:metric] == false
+              tags[key] = value
+            else
+              fields[key] = value
+            end
+          end
+        end
+        {
+          tags: tags,
+          fields: fields
+        }
+      end
+
+      def validate_fields(partial_definition, fields)
+
+        partial_definition.each_pair do |key, value|
+          next if key == :required || value[:metric] == (false || nil)
+      
+          if fields[key].nil?
+            fail "Field #{key} is missing. Validation of request failed!"
+          elsif value[:type].capitalize != fields[key].class.to_s
+            fail "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{fields[key].class} provided"
+          end
+        end
+      end
+
+      def validate_tags(partial_definition, tags)
+        partial_definition.each_pair do |key, value|
+          next if key == :required || value[:metric] == true
+          
+          if tags[key].nil?
+            if value[:default].nil?
+              fail "Property #{key} is missing. Validation of request failed!"
+            else 
+              tags[key] = value[:default]
+            end
+          else 
+            type = tags[key].class
+            type = "Boolean" if type == TrueClass || type == FalseClass
+            if value[:type].capitalize == type.to_s
+              next
+            else
+              fail "Defined type for SemanticType attribute property '#{key}' is #{value[:type].capitalize}, #{type} provided"
+            end
+          end
+        end
+      end
+
+      def get_partial_definition(path)
+        i = 0
+        definition = { }
+        semantictype_crd = crd_content[:properties]
+        while i < path.length
+          if path[i].to_sym == :top
+            semantictype_crd.each_pair do |key, value|
+              if key == :required
+                definition[key] = value
+              else
+                definition[key] = value if value[:type] != "array"
+              end
+            end
+          else
+            definition = {}
+            definition[:required] = semantictype_crd[path[i].to_sym][:items][:required]
+            semantictype_crd[path[i].to_sym][:items][:properties].each_pair do |key, value|
+                definition[key] = value if value[:type] != "array"
+            end
+            semantictype_crd = semantictype_crd[path[i].to_sym][:items][:properties]
+          end
+          i+=1
+        end
+        definition
+      end
+
+      def get_path_to_object(parameter)
+        path = ["top"]
+        array = parameter.split('/')
+        array.each do |element|
+          path.push(element.split(':')[1])
+        end
+        path
+      end
+
+      def validate_parameter(parameter)
+        array_of_parameters = []
+        begin
+          parameter.split('/').each_with_index do |param, index|
+            name, type = param.split(':')
+            fail unless name && type
+            array_of_parameters.push({
+              name: name,
+              type: type
+            })
+          end
+          array_of_parameters
+        rescue => error
+          fail "Could not resolve parameter '#{parameter}'. It should be in format: 'parent:type/child:type'"
+        end
+      end
+    end
+  end
+end

data/persistence-providers.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name        = 'persistence-providers'
-  spec.version     = '0.0.3.5'
+  spec.version     = '0.0.3.6'
   spec.author      = 'Reactor8'
   spec.email       = 'support@reactor8.com'
   spec.description = %q{Persistence providers plugin}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: persistence-providers
 version: !ruby/object:Gem::Version
-  version: 0.0.3.5
+  version: 0.0.3.6
 platform: ruby
 authors:
 - Reactor8
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-12 00:00:00.000000000 Z
+date: 2020-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: kubeclient
@@ -70,6 +70,7 @@ files:
 - lib/state/component/providers/influxdb/measurement/errors.rb
 - lib/state/component/providers/influxdb/measurement/events.rb
 - lib/state/component/providers/influxdb/measurement/states.rb
+- lib/state/component/providers/influxdb/semantictype.rb
 - lib/state/component/providers/kube_crd.rb
 - lib/state/component_def.rb
 - lib/state/component_def/attribute_type_info.rb