k8s-client 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 629b1fd35aba3aa4999fb320033c537e239768a3ea2a9163b19a2ba69f00466d
-  data.tar.gz: 35c3043b726d29a0fb038433b4764fdd8de9b2665b1db8c6a7649c6403f9e98d
+  metadata.gz: ac23332f18be9e1b95502a9526eab9ed5e4f31a379540387c452802be09b61dc
+  data.tar.gz: 673e7eb930084bb9dd49353d87de7dc54631e8b4a8a85f3bb707c28573ca8441
 SHA512:
-  metadata.gz: cac2f739ee3dcfe3885fe4070c4497e22cc86d227a449c18c773390299e5e416458971a2d424abbcae52b163f4077fc8d85138865c1f1857ca3a83f27dae9e84
-  data.tar.gz: a2bd611f565e0f6e3e9f54d201f89bc2a5eef307b9d3ce7327c267139fd29744a7f721d655f6b67b9aa4126db0c97830b1e2a8330f5abde8009f3ac954c6e7f4
+  metadata.gz: f19573347f0f74af1e6eb9da2d8aa61d469aa6de69b2deef3938a39f74758191e5edc0f931d33e3953c3f3a96accb0463cfda704dec16d731b700a10e6f8fe7c
+  data.tar.gz: 426356558b0772b51324624ad06ee4ed3b6bef519fd78d845df5f020cf515f81ae1a4bc11febcae3609b425d46ca804645b17e53d891c52e26f9215d4586b4a4

data/k8s-client.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "dry-struct", "~> 0.5.0"
   spec.add_runtime_dependency "deep_merge", "~> 1.2.1"
   spec.add_runtime_dependency "recursive-open-struct", "~> 1.1.0"
+  spec.add_runtime_dependency 'hashdiff', '~> 0.3.7'
 
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/k8s/client.rb

@@ -165,9 +165,15 @@ module K8s
     end
 
     # @param resource [K8s::Resource]
+    # @param options [Hash]
+    # @see ResourceClient#delete for options
     # @return [K8s::Resource]
-    def delete_resource(resource)
-      client_for_resource(resource).delete_resource(resource)
+    def delete_resource(resource, **options)
+      client_for_resource(resource).delete_resource(resource, **options)
+    end
+
+    def patch_resource(resource, attrs)
+      client_for_resource(resource).json_patch(resource.metadata.name, attrs)
     end
   end
 end

data/lib/k8s/client/version.rb

@@ -1,6 +1,6 @@
 module K8s
   class Client
     # Updated on releases using semver.
-    VERSION = "0.3.4"
+    VERSION = "0.4.0"
   end
 end

data/lib/k8s/resource.rb

@@ -1,5 +1,6 @@
 require 'deep_merge'
 require 'recursive-open-struct'
+require 'hashdiff'
 
 module K8s
   # generic untyped resource
@@ -60,9 +61,36 @@ module K8s
       h = to_hash
 
       # merge in-place
-      h.deep_merge!(attrs.to_hash, overwrite_arrays: true)
+      h.deep_merge!(attrs.to_hash, overwrite_arrays: true, merge_nil_values: true)
 
       self.class.new(h)
     end
+
+    def checksum
+      @checksum ||= Digest::MD5.hexdigest(Marshal::dump(to_hash))
+    end
+
+    def merge_patch_ops(attrs, config_annotation)
+      Util.json_patch(current_config(config_annotation), stringify_hash(attrs))
+    end
+
+    # Gets the existing resources (on kube api) configuration, an empty hash if not present
+    #
+    # @return [Hash]
+    def current_config(config_annotation)
+      current_cfg = self.metadata.annotations&.dig(config_annotation)
+
+      return JSON.parse(current_cfg) if current_cfg
+
+      {}
+    end
+
+    def can_patch?(config_annotation)
+      !!self.metadata.annotations&.dig(config_annotation)
+    end
+
+    def stringify_hash(hash)
+      JSON.load(JSON.dump(hash))
+    end
   end
 end

data/lib/k8s/resource_client.rb

@@ -44,8 +44,9 @@ module K8s
     # @param namespace [String, nil]
     # @param labelSelector [nil, String, Hash{String => String}]
     # @param fieldSelector [nil, String, Hash{String => String}]
+    # @param skip_forbidden [Boolean] skip resources that return HTTP 403 errors
     # @return [Array<K8s::Resource>]
-    def self.list(resources, transport, namespace: nil, labelSelector: nil, fieldSelector: nil)
+    def self.list(resources, transport, namespace: nil, labelSelector: nil, fieldSelector: nil, skip_forbidden: false)
       api_paths = resources.map{|resource| resource.path(namespace: namespace) }
       api_lists = transport.gets(*api_paths,
          response_class: K8s::API::MetaV1::List,
@@ -53,9 +54,10 @@ module K8s
            'labelSelector' => selector_query(labelSelector),
            'fieldSelector' => selector_query(fieldSelector),
          ),
+         skip_forbidden: skip_forbidden,
        )
 
-      resources.zip(api_lists).map {|resource, api_list| resource.process_list(api_list) }.flatten
+      resources.zip(api_lists).map {|resource, api_list| api_list ? resource.process_list(api_list) : [] }.flatten
     end
 
     # @param transport [K8s::Transport]
@@ -243,6 +245,20 @@ module K8s
       )
     end
 
+    # @param name [String]
+    # @param ops [Hash] json-patch operations
+    # @param namespace [String]
+    # @return [resource_class]
+    def json_patch(name, ops, namespace: @namespace)
+      @transport.request(
+        method: 'PATCH',
+        path: self.path(name, namespace: namespace),
+        content_type: 'application/json-patch+json',
+        request_object: ops,
+        response_class: @resource_class,
+      )
+    end
+
     # @return [Bool]
     def delete?
       @api_resource.verbs.include? 'delete'
@@ -250,13 +266,14 @@ module K8s
 
     # @param name [String]
     # @param namespace [String]
+    # @param propagationPolicy [String] The propagationPolicy to use for the API call. Possible values include “Orphan”, “Foreground”, or “Background”
     # @return [K8s::API::MetaV1::Status]
     def delete(name, namespace: @namespace, propagationPolicy: nil)
       @transport.request(
         method: 'DELETE',
         path: self.path(name, namespace: namespace),
         query: make_query(
-          'propagationPolicy' => propagationPolicy,
+          'propagationPolicy' => propagationPolicy
         ),
         response_class: @resource_class, # XXX: documented as returning Status
       )
@@ -281,6 +298,8 @@ module K8s
     end
 
     # @param resource [resource_class] with metadata
+    # @param options [Hash]
+    # @see #delete for possible options
     # @return [K8s::API::MetaV1::Status]
     def delete_resource(resource, **options)
       delete(resource.metadata.name, namespace: resource.metadata.namespace, **options)

data/lib/k8s/stack.rb

@@ -11,6 +11,9 @@ module K8s
     # Annotation used to identify resource versions
     CHECKSUM_ANNOTATION = 'k8s.kontena.io/stack-checksum'
 
+    # Annotation used to identify last applied configuration
+    LAST_CONFIG_ANNOTATION = 'kubectl.kubernetes.io/last-applied-configuration'
+
     # List of apiVersion:kind combinations to skip for stack prune
     # These would lead to stack prune misbehaving if not skipped.
     PRUNE_IGNORE = [
@@ -44,37 +47,33 @@ module K8s
 
     attr_reader :name, :resources
 
-    def initialize(name, resources = [], debug: false, label: self.class::LABEL, checksum_annotation: self.class::CHECKSUM_ANNOTATION)
+    def initialize(name, resources = [], debug: false, label: self.class::LABEL, checksum_annotation: self.class::CHECKSUM_ANNOTATION, last_configuration_annotation: self.class::LAST_CONFIG_ANNOTATION)
       @name = name
       @resources = resources
       @keep_resources = {}
       @label = label
       @checksum_annotation = checksum_annotation
+      @last_config_annotation = last_configuration_annotation
 
       logger! progname: name, debug: debug
     end
 
-    # Random "checksum" used to identify different stack resource versions using an annotation.
-    #
-    # NOTE: This is not actually a checksum.
-    #
-    # @return [String]
-    def checksum
-      @checksum ||= SecureRandom.hex(16)
-    end
-
     # @param resource [K8s::Resource] to apply
     # @param base_resource [K8s::Resource] preserve existing attributes from base resource
     # @return [K8s::Resource]
     def prepare_resource(resource, base_resource: nil)
-      if base_resource
-        resource = base_resource.merge(resource)
-      end
+      # XXX: base_resource is not really used anymore, kept for backwards compatibility for a while
+
+      # calculate checksum  only from the "local" source
+      checksum = resource.checksum
 
       # add stack metadata
       resource.merge(metadata: {
         labels: { @label => name },
-        annotations: { @checksum_annotation => checksum },
+        annotations: {
+          @checksum_annotation => checksum,
+          @last_config_annotation => resource.to_json
+        },
       })
     end
 
@@ -83,24 +82,21 @@ module K8s
       server_resources = client.get_resources(resources)
 
       resources.zip(server_resources).map do |resource, server_resource|
-        if server_resource
-          # keep server checksum for comparison
-          # NOTE: this will not compare equal for resources with arrays containing hashes with default values applied by the server
-          #       however, that will just cause extra PUTs, so it doesn't have any functional effects
-          compare_resource = server_resource.merge(resource).merge(metadata: {
-            labels: { @label => name },
-          })
-        end
-
         if !server_resource
-          logger.info "Create resource #{resource.apiVersion}:#{resource.kind}/#{resource.metadata.name} in namespace #{resource.metadata.namespace} with checksum=#{checksum}"
+          logger.info "Create resource #{resource.apiVersion}:#{resource.kind}/#{resource.metadata.name} in namespace #{resource.metadata.namespace} with checksum=#{resource.checksum}"
           keep_resource! client.create_resource(prepare_resource(resource))
-        elsif server_resource != compare_resource
-          logger.info "Update resource #{resource.apiVersion}:#{resource.kind}/#{resource.metadata.name} in namespace #{resource.metadata.namespace} with checksum=#{checksum}"
-          keep_resource! client.update_resource(prepare_resource(resource, base_resource: server_resource))
+        elsif server_resource.metadata.annotations[@checksum_annotation] != resource.checksum
+          logger.info "Update resource #{resource.apiVersion}:#{resource.kind}/#{resource.metadata.name} in namespace #{resource.metadata.namespace} with checksum=#{resource.checksum}"
+          r = prepare_resource(resource)
+          if server_resource.can_patch?(@last_config_annotation)
+            keep_resource! client.patch_resource(server_resource, server_resource.merge_patch_ops(r.to_hash, @last_config_annotation))
+          else
+            # try to update with PUT
+            keep_resource! client.update_resource(server_resource.merge(prepare_resource(resource)))
+          end
         else
-          logger.info "Keep resource #{resource.apiVersion}:#{resource.kind}/#{resource.metadata.name} in namespace #{resource.metadata.namespace} with checksum=#{compare_resource.metadata.annotations[@checksum_annotation]}"
-          keep_resource! compare_resource
+          logger.info "Keep resource #{server_resource.apiVersion}:#{server_resource.kind}/#{server_resource.metadata.name} in namespace #{server_resource.metadata.namespace} with checksum=#{server_resource.metadata.annotations[@checksum_annotation]}"
+          keep_resource! server_resource
         end
       end
 
@@ -116,8 +112,9 @@ module K8s
     end
 
     # Delete all stack resources that were not applied
-    def prune(client, keep_resources: )
-      client.list_resources(labelSelector: {@label => name}).each do |resource|
+    def prune(client, keep_resources: , skip_forbidden: true)
+      # using skip_forbidden: assume we can't create resource types that we are forbidden to list, so we don't need to prune them either
+      client.list_resources(labelSelector: {@label => name}, skip_forbidden: skip_forbidden).each do |resource|
         next if PRUNE_IGNORE.include? "#{resource.apiVersion}:#{resource.kind}"
 
         resource_label = resource.metadata.labels ? resource.metadata.labels[@label] : nil
@@ -132,7 +129,7 @@ module K8s
         else
           logger.info "Delete resource #{resource.apiVersion}:#{resource.kind}/#{resource.metadata.name} in namespace #{resource.metadata.namespace}"
           begin
-            client.delete_resource(resource)
+            client.delete_resource(resource, propagationPolicy: 'Background')
           rescue K8s::Error::NotFound
             # assume aliased objects in multiple API groups, like for Deployments
             # alternatively, a custom resource whose definition was already deleted earlier

data/lib/k8s/transport.rb

@@ -224,10 +224,11 @@ module K8s
 
     # @param options [Array<Hash>] @see #request
     # @param skip_missing [Boolean] return nil for HTTP 404 responses
+    # @param skip_forbidden [Boolean] return nil for HTTP 403 responses
     # @param retry_errors [Boolean] retry with non-pipelined request for HTTP 503 responses
     # @param common_options [Hash] @see #request, merged with the per-request options
     # @return [Array<response_class, Hash, nil>]
-    def requests(*options, skip_missing: false, retry_errors: true, **common_options)
+    def requests(*options, skip_missing: false, skip_forbidden: false, retry_errors: true, **common_options)
       return [] if options.empty? # excon chokes
 
       start = Time.now
@@ -249,6 +250,12 @@ module K8s
           else
             raise
           end
+        rescue K8s::Error::Forbidden
+          if skip_forbidden
+            nil
+          else
+            raise
+          end
         rescue K8s::Error::ServiceUnavailable => exc
           if retry_errors
             logger.warn { "Retry #{format_request(request_options)} => HTTP #{exc.code} #{exc.reason} in #{'%.3f' % t}s" }

data/lib/k8s/util.rb

@@ -19,5 +19,47 @@ module K8s
 
       args.map{|arg| value_map[arg] }
     end
+
+    # Produces a set of json-patch operations so that applying
+    # the operations on a, gives you the results of b
+    # Used in correctly patching the Kube resources on stack updates
+    #
+    # @param a [Hash] Hash to compute patches against
+    # @param a [Hash] New Hash to compute patches "from"
+    def self.json_patch(a, b)
+      diffs = HashDiff.diff(a, b, array_path: true)
+      ops = []
+      # Each diff is like ["+", "spec.selector.aziz", "kebab"]
+      # or ["-", "spec.selector.aziz", "kebab"]
+      diffs.each do |diff|
+        operator = diff[0]
+        # substitute '/' with '~1' and '~' with '~0'
+        # according to RFC 6901
+        path = diff[1].map {|p| p.to_s.gsub('/', '~1')}.map {|p| p.to_s.gsub('~', '~0')}
+        if operator == '-'
+          ops << {
+            op: "remove",
+            path: "/" + path.join('/')
+          }
+        elsif operator == '+'
+          ops << {
+            op: "add",
+            path: "/" + path.join('/'),
+            value: diff[2]
+          }
+        elsif operator == '~'
+          ops << {
+            op: "replace",
+            path: "/" + path.join('/'),
+            value: diff[3]
+          }
+        else
+          raise "Unknown diff operator: #{operator}!"
+        end
+
+      end
+
+      ops
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: k8s-client
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Kontena, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-21 00:00:00.000000000 Z
+date: 2018-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: excon
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.0
+- !ruby/object:Gem::Dependency
+  name: hashdiff
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.7
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.7
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement