skull_island 1.2.0 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 67a8f014f5c284c8fe9a8201fa1296130abb88a08b52e8f512e935eb178bb337
-  data.tar.gz: ed28d0b9011b978e0939ae7ba73b8934d147f77df0ebee8f6a38464f6ed12440
+  metadata.gz: 782471de2e1ce1e974ac22f0420d33ce0903f909b4286094537c7169caa872f5
+  data.tar.gz: 7edb2bcdb5094f27932c2a3e13f6865e969ba6f43e78f34d6a26c7416cb01bc9
 SHA512:
-  metadata.gz: e63e8b8e203f408df32846e4aec20cbbcea5ac7688ef2ec06db386c9e1f7f88a13094706dcb2b12dba98639c25d9488209ff6b6ce6b9ce60af3c2237bd39f1b0
-  data.tar.gz: e150792100e0b64286c7dee6338bf723aa0f2e88487bc9f5846485e410e57207504c974b7e861dc366082e849bea1b845a20d00cf9250951db46da62cc05ce5d
+  metadata.gz: 6a83f0d13f17f858332882836b22be450c9974283c9e1a8fe0db20b007f13b080a73d4c9fe43bc3516a2a2a5768821f1a4b7494bf9262874774a63c9feb78068
+  data.tar.gz: 7dc758ebad1c39cbaa8b64789ac0ff7db061349cd0a086a2d50cfedf2cf822e6d8745d92a1a976d63f7398b810f7ca634bf46b7f2408709c33b8cbf2a18131d0

data/.gitignore

@@ -10,3 +10,4 @@
 # rspec failure tracking
 .rspec_status
 *.gem
+.DS_Store

data/.rubocop.yml

@@ -8,7 +8,7 @@ Metrics/LineLength:
   Max: 100
 
 Metrics/ClassLength:
-  Max: 175
+  Max: 190
 
 Metrics/ModuleLength:
   Max: 175
@@ -23,7 +23,7 @@ Metrics/PerceivedComplexity:
     - 'lib/skull_island/cli.rb'
 
 Metrics/AbcSize:
-  Max: 27
+  Max: 28
 
 Metrics/BlockLength:
   Max: 35

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    skull_island (1.2.0)
+    skull_island (1.2.2)
       deepsort (~> 0.4)
       erubi (~> 1.8)
       json (~> 2.1)
@@ -15,11 +15,11 @@ GEM
   specs:
     addressable (2.4.0)
     ast (2.4.0)
-    backports (3.12.0)
-    deepsort (0.4.1)
+    backports (3.15.0)
+    deepsort (0.4.2)
     diff-lcs (1.3)
-    docile (1.3.1)
-    domain_name (0.5.20180417)
+    docile (1.3.2)
+    domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     erubi (1.8.0)
     ethon (0.12.0)
@@ -28,7 +28,7 @@ GEM
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.13.1)
       faraday (>= 0.7.4, < 1.0)
-    ffi (1.10.0)
+    ffi (1.11.1)
     gh (0.15.1)
       addressable (~> 2.4.0)
       backports
@@ -39,7 +39,7 @@ GEM
     highline (1.7.10)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
-    jaro_winkler (1.5.2)
+    jaro_winkler (1.5.3)
     json (2.2.0)
     launchy (2.4.3)
       addressable (~> 2.3)
@@ -50,15 +50,13 @@ GEM
       mime-types-data (~> 3.2015)
     mime-types-data (3.2019.0331)
     multi_json (1.13.1)
-    multipart-post (2.0.0)
+    multipart-post (2.1.1)
     net-http-persistent (2.9.4)
     net-http-pipeline (1.0.1)
     netrc (0.11.0)
-    parallel (1.14.0)
-    parser (2.6.0.0)
+    parallel (1.17.0)
+    parser (2.6.3.0)
       ast (~> 2.4.0)
-    powerpack (0.1.2)
-    psych (3.1.0)
     pusher-client (0.6.2)
       json
       websocket (~> 1.0)
@@ -72,32 +70,30 @@ GEM
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
       rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
+    rspec-core (3.8.2)
       rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    rspec-expectations (3.8.4)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+    rspec-mocks (3.8.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    rubocop (0.65.0)
+    rspec-support (3.8.2)
+    rubocop (0.74.0)
       jaro_winkler (~> 1.5.1)
       parallel (~> 1.10)
-      parser (>= 2.5, != 2.5.1.1)
-      powerpack (~> 0.1)
-      psych (>= 3.1.0)
+      parser (>= 2.6)
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.4.0)
-    ruby-progressbar (1.10.0)
-    simplecov (0.16.1)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    ruby-progressbar (1.10.1)
+    simplecov (0.17.0)
       docile (~> 1.1)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
     thor (0.20.3)
-    travis (1.8.9)
+    travis (1.8.10)
       backports
       faraday (~> 0.9)
       faraday_middleware (~> 0.9, >= 0.9.1)
@@ -110,11 +106,11 @@ GEM
       ethon (>= 0.8.0)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.5)
-    unicode-display_width (1.4.1)
+    unf_ext (0.0.7.6)
+    unicode-display_width (1.6.0)
     websocket (1.2.8)
     will_paginate (3.1.7)
-    yard (0.9.18)
+    yard (0.9.20)
 
 PLATFORMS
   ruby
@@ -127,7 +123,7 @@ DEPENDENCIES
   simplecov (~> 0.15)
   skull_island!
   travis (~> 1.8)
-  yard (~> 0.9)
+  yard (~> 0.9.20)
 
 BUNDLED WITH
    2.0.1

data/README.md

@@ -81,6 +81,8 @@ KONG_ADMIN_URL='https://api-admin.mydomain.com' \
 skull_island export --verbose /path/to/export.yml
 ```
 
+Exporting, by default, exports the entire configuration of a Kong gateway, but will strip out special meta-data tags added by Skull Island to track projects. If, instead, you'd like to export **only** the configuration for a specific project, you can add `--project foo` (where `foo` is the name of your project) to export only those resources associated with it and maintain the special key in the exported YAML.
+
 ### Importing
 
 Skull Island also supports importing configurations (both partial and full) from a YAML + ERB document:
@@ -112,6 +114,16 @@ KONG_ADMIN_URL='https://api-admin.mydomain.com' \
 skull_island import --verbose --test /path/to/export.yml
 ```
 
+Note that `--test` has a high likelihood of generating errors with a complicated import if required/dependent resources do not exist.
+
+#### Importing with Projects
+
+Skull Island 1.2.1 introduces the ability to use a special top-level key in the configuration called `project` that uses meta-data to track which resources belong to a project. This meta-data can safely be added at another time as this tool will "adopt" otherwise matching resources into a project.
+
+To use this functionality, either add the `project` key to your configuration file (usually directly below the `version` key) with some value that will be unique on your gateway, or use `--project foo` (where `foo` is the name of your project) as a CLI flag.
+
+When using the `project` feature of Skull Island, the CLI tool will automatically clean up old resources no longer found in your config file. This is, in fact, the _only_ circumstance under which this tool actually removes resources. Use this feature with care, as it can delete large swaths of your configuration if used incorrectly. It is **critical** that this value is unique since this project functionality is used to delete resources.
+
 ### Migrating
 
 With Skull Island, it is possible to migrate a configuration from a 0.14.x gateway to one compatible with a 1.2.x gateway. If you have a previous export, you can just run `skull_island migrate /path/to/export.yml` and you'll receive a 1.2 compatible config on standard out. If you'd prefer, you can have that config written to a file as well (just like the export command) like so:
@@ -133,6 +145,7 @@ The import/export/migrate CLI functions produce YAML with support for embedded R
 ```yaml
 ---
 version: '1.2'
+project: FooV2
 certificates: []
 consumers:
 - username: foo
@@ -190,7 +203,7 @@ plugins:
   service: "<%= lookup :service, 'search_api' %>"
 ```
 
-All top-level keys (other than `version`) require an Array as a parameter, either by providing a list of entries or an empty Array (`[]`). The above shows how to use the `lookup()` function to refer to another resource. This "looks up" the resource type (`service` in this case) by `name` (`search_api` in this case) and resolves its `id`. This function can also be used to lookup a `route` or `upstream` by its `name`, or a `consumer` by its `username`. Note that Kong itself doesn't _require_ `route` resources to have unique names, so you'll need to enforce that practice yourself for `lookup` to be useful for Routes.
+All top-level keys (other than `version` and `project`) require an Array as a parameter, either by providing a list of entries or an empty Array (`[]`). The above shows how to use the `lookup()` function to refer to another resource. This "looks up" the resource type (`service` in this case) by `name` (`search_api` in this case) and resolves its `id`. This function can also be used to lookup a `route` or `upstream` by its `name`, or a `consumer` by its `username`. Note that Kong itself doesn't _require_ `route` resources to have unique names, so you'll need to enforce that practice yourself for `lookup` to be useful for Routes.
 
 While technically _any_ Ruby is valid, the following are pretty helpful for templating your YAML files:
 

data/lib/skull_island.rb

@@ -39,6 +39,7 @@ require 'skull_island/api_client_base'
 require 'skull_island/api_client'
 require 'skull_island/simple_api_client'
 require 'skull_island/resource_collection'
+require 'skull_island/helpers/meta'
 require 'skull_island/helpers/resource'
 require 'skull_island/helpers/resource_class'
 require 'skull_island/helpers/migration'

data/lib/skull_island/api_client_base.rb

@@ -67,6 +67,12 @@ module SkullIsland
       end
     end
 
+    def delete(uri)
+      client_action do |client|
+        client[uri].delete(json_headers)
+      end
+    end
+
     private
 
     def client_action

data/lib/skull_island/cli.rb

@@ -15,9 +15,10 @@ module SkullIsland
     class_option :verbose, type: :boolean
 
     desc 'export [OPTIONS] [OUTPUT|-]', 'Export the current configuration to OUTPUT'
+    option :project, desc: 'Project identifier for metadata'
     def export(output_file = '-')
       if output_file == '-'
-        STDERR.puts '[INFO] Outputting to STDOUT' if options['verbose']
+        warn '[INFO] Outputting to STDOUT' if options['verbose']
       else
         full_filename = File.expand_path(output_file)
         dirname = File.dirname(full_filename)
@@ -29,6 +30,7 @@ module SkullIsland
       validate_server_version
 
       output = { 'version' => '1.2' }
+      output['project'] = options['project'] if options['project']
 
       [
         Resources::Certificate,
@@ -46,6 +48,7 @@ module SkullIsland
     end
 
     desc 'import [OPTIONS] [INPUT|-]', 'Import a configuration from INPUT'
+    option :project, desc: 'Project identifier for metadata'
     option :test, type: :boolean, desc: "Don't do anything, just show what would happen"
     def import(input_file = '-')
       raw ||= acquire_input(input_file, options['verbose'])
@@ -56,19 +59,23 @@ module SkullIsland
 
       validate_config_version input['version']
 
+      import_time = Time.now.utc.to_i
+      input['project'] = options['project'] if options['project']
+
       [
         Resources::Certificate,
         Resources::Consumer,
         Resources::Upstream,
         Resources::Service,
         Resources::Plugin
-      ].each { |clname| import_class(clname, input) }
+      ].each { |clname| import_class(clname, input, import_time) }
     end
 
     desc(
       'migrate [OPTIONS] [INPUT|-] [OUTPUT|-]',
       'Migrate an older config from INPUT to OUTPUT'
     )
+    option :project, desc: 'Project identifier for metadata'
     def migrate(input_file = '-', output_file = '-')
       raw ||= acquire_input(input_file, options['verbose'])
 
@@ -79,9 +86,10 @@ module SkullIsland
       validate_migrate_version input['version']
 
       output = migrate_config(input)
+      output['project'] = options['project'] if options['project']
 
       if output_file == '-'
-        STDERR.puts '[INFO] Outputting to STDOUT' if options['verbose']
+        warn '[INFO] Outputting to STDOUT' if options['verbose']
         STDOUT.puts output.to_yaml
       else
         full_filename = File.expand_path(output_file)
@@ -97,23 +105,30 @@ module SkullIsland
     private
 
     def export_class(class_name, output_data)
-      STDERR.puts "[INFO] Processing #{class_name.route_key}" if options['verbose']
-      output_data[class_name.route_key] = class_name.all.collect(&:export)
+      warn "[INFO] Processing #{class_name.route_key}" if options['verbose']
+      output_data[class_name.route_key] = if options['project']
+                                            class_name.where(:project, options['project'])
+                                                      .collect(&:export)
+                                          else
+                                            class_name.all.collect(&:export)
+                                          end
     end
 
-    def import_class(class_name, import_data)
-      STDERR.puts "[INFO] Processing #{class_name.route_key}" if options['verbose']
+    def import_class(class_name, import_data, import_time)
+      warn "[INFO] Processing #{class_name.route_key}" if options['verbose']
       class_name.batch_import(
         import_data[class_name.route_key],
         verbose: options['verbose'],
-        test: options['test']
+        test: options['test'],
+        time: import_time,
+        project: import_data['project']
       )
     end
 
     # Used to pull input from either STDIN or the specified file
     def acquire_input(input_file, verbose = false)
       if input_file == '-'
-        STDERR.puts '[INFO] Reading from STDIN' if verbose
+        warn '[INFO] Reading from STDIN' if verbose
         STDIN.read
       else
         full_filename = File.expand_path(input_file)
@@ -133,10 +148,10 @@ module SkullIsland
       if version && ['1.1', '1.2'].include?(version)
         validate_server_version
       elsif version && ['0.14', '1.0'].include?(version)
-        STDERR.puts '[CRITICAL] Config version is too old. Try `migrate` instead of `import`.'
+        warn '[CRITICAL] Config version is too old. Try `migrate` instead of `import`.'
         exit 2
       else
-        STDERR.puts '[CRITICAL] Config version is unknown or not supported.'
+        warn '[CRITICAL] Config version is unknown or not supported.'
         exit 3
       end
     end
@@ -145,7 +160,7 @@ module SkullIsland
       if version && version == '0.14'
         true
       else
-        STDERR.puts '[CRITICAL] Config version must be 0.14 for migration.'
+        warn '[CRITICAL] Config version must be 0.14 for migration.'
         exit 4
       end
     end
@@ -155,7 +170,7 @@ module SkullIsland
       if server_version.match?(/^1.[12]/)
         true
       else
-        STDERR.puts '[CRITICAL] Server version mismatch!'
+        warn '[CRITICAL] Server version mismatch!'
         exit 1
       end
     end

data/lib/skull_island/helpers/meta.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module SkullIsland
+  module Helpers
+    # Useful for embedding meta-data into special tags
+    module Meta
+      def add_meta(key, value)
+        metatag = "_meta~#{key}~#{value}"
+
+        # filter out any existing duplicate metatags
+        existing_tags = raw_tags.reject { |tag| tag.start_with?("_meta~#{key}~") }
+
+        # Add the new tag directly, bypassing preprocessing
+        raw_set('tags', existing_tags + [metatag])
+      end
+
+      def import_time
+        metatags['import_time']
+      end
+
+      def import_time=(time)
+        add_meta('import_time', time)
+      end
+
+      def remove_meta(key)
+        # filter out an existing metatags
+        filtered_tags = raw_tags.reject { |tag| tag.start_with?("_meta~#{key}~") }
+
+        # Bypassing preprocessing
+        raw_set('tags', filtered_tags)
+      end
+
+      def metatags
+        metadata = {}
+        raw_tags.select { |tag| tag.start_with?('_meta~') }.each do |tag|
+          key, value = tag.gsub('_meta~', '').split('~', 2)
+          metadata[key] = value
+        end
+        metadata
+      end
+
+      def project
+        metatags['project']
+      end
+
+      def project=(project_id)
+        unless project_id.is_a?(String) && project_id.match?(/^[\w_\-\.~]+$/)
+          raise Exceptions::InvalidArguments, 'project'
+        end
+
+        add_meta('project', project_id)
+      end
+
+      def raw_tags
+        reload if @lazy && !@entity.key?('tags')
+        @entity['tags'] || []
+      end
+
+      def preprocess_tags(input)
+        input.uniq + metatags.map { |k, v| "_meta~#{k}~#{v}" }
+      end
+
+      def postprocess_tags(value)
+        (value || []).reject { |tag| tag.start_with?('_meta~') }
+      end
+
+      def supports_meta?
+        true
+      end
+    end
+  end
+end

data/lib/skull_island/helpers/resource.rb

@@ -30,12 +30,13 @@ module SkullIsland
 
       def digest
         Digest::MD5.hexdigest(
-          digest_properties.sort.map { |prop| "#{prop}=#{send(prop.to_sym)}" }.compact.join(':')
+          digest_properties.sort.map { |prp| "#{prp}=#{send(prp.to_sym) || ''}" }.compact.join(':')
         )
       end
 
       def digest_properties
-        properties.keys.reject { |k| %i[created_at updated_at].include? k }
+        props = properties.keys.reject { |k| %i[created_at updated_at].include? k }
+        supports_meta? ? props + [:project] : props
       end
 
       # Tests for an existing version of this resource based on its properties rather than its `id`
@@ -45,8 +46,8 @@ module SkullIsland
           entity_data = @api_client.cache(result.first.relative_uri.to_s) do |client|
             client.get(result.first.relative_uri.to_s)
           end
-          @entity  = entity_data
-          @lazy    = false
+          @entity = entity_data
+          @lazy = false
           @tainted = false
           true
         else
@@ -172,8 +173,8 @@ module SkullIsland
           entity_data = @api_client.cache(relative_uri.to_s) do |client|
             client.get(relative_uri.to_s)
           end
-          @entity  = entity_data
-          @lazy    = false
+          @entity = entity_data
+          @lazy = false
           @tainted = false
           true
         end
@@ -199,6 +200,10 @@ module SkullIsland
         self.class.relative_uri
       end
 
+      def supports_meta?
+        false
+      end
+
       # ActiveRecord ActiveModel compatibility method
       def update(params)
         new_params = {}

data/lib/skull_island/resource.rb

@@ -179,6 +179,13 @@ module SkullIsland
       )
     end
 
+    def self.cleanup_except(project, keep_these)
+      where(:project, project).reject { |res| keep_these.include?(res.id) }.map do |res|
+        puts "[WARN] ! Removing #{name} (#{res.id})"
+        res.destroy
+      end
+    end
+
     def initialize(options = {})
       # TODO: better options validations
       raise Exceptions::InvalidOptions unless options.is_a?(Hash)
@@ -207,5 +214,17 @@ module SkullIsland
     def relative_uri
       "#{self.class.relative_uri}/#{id}"
     end
+
+    private
+
+    # A way to add things _without_ preprocessing them
+    def raw_set(key, value)
+      raise Exceptions::ImmutableModification if immutable?
+
+      @entity[key.to_s] = value
+
+      @modified_properties << key.to_sym
+      @tainted = true
+    end
   end
 end

data/lib/skull_island/resources/basicauth_credential.rb

@@ -18,13 +18,18 @@ module SkullIsland
       def self.batch_import(data, verbose: false, test: false)
         raise(Exceptions::InvalidArguments) unless data.is_a?(Array)
 
+        known_ids = []
+
         data.each_with_index do |resource_data, index|
           resource = new
-          resource.username = resource_data['username']
-          resource.password = resource_data['password'] if resource_data['password']
+          resource.delayed_set(:username, resource_data, 'username')
+          resource.delayed_set(:password, resource_data, 'password') if resource_data['password']
           resource.delayed_set(:consumer, resource_data, 'consumer')
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
+          known_ids << resource.id
         end
+
+        known_ids
       end
 
       def self.relative_uri
@@ -56,6 +61,10 @@ module SkullIsland
         false
       end
 
+      def project
+        consumer ? consumer.project : nil
+      end
+
       private
 
       def postprocess_consumer(value)

data/lib/skull_island/resources/certificate.rb

@@ -7,29 +7,42 @@ module SkullIsland
     #
     # @see https://docs.konghq.com/1.1.x/admin-api/#certificate-object Certificate API definition
     class Certificate < Resource
+      include Helpers::Meta
+
       property :cert, required: true, validate: true
       property :key, required: true, validate: true
       property :snis, validate: true
       property :created_at, read_only: true, postprocess: true
-      property :tags, validate: true
+      property :tags, validate: true, preprocess: true, postprocess: true
 
-      def self.batch_import(data, verbose: false, test: false)
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
+      def self.batch_import(data, verbose: false, test: false, project: nil, time: nil)
         raise(Exceptions::InvalidArguments) unless data.is_a?(Array)
 
+        known_ids = []
+
         data.each_with_index do |resource_data, index|
           resource = new
-          resource.cert = resource_data['cert']
-          resource.key = resource_data['key']
+          resource.delayed_set(:cert, resource_data, 'cert')
+          resource.delayed_set(:key, resource_data, 'key')
           resource.snis = resource_data['snis'] if resource_data['snis']
           resource.tags = resource_data['tags'] if resource_data['tags']
+          resource.project = project if project
+          resource.import_time = (time || Time.now.utc.to_i) if project
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
+          known_ids << resource.id
         end
+
+        cleanup_except(project, known_ids) if project
       end
+      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/PerceivedComplexity
 
       def export(options = {})
         hash = { 'cert' => cert, 'key' => key }
         hash['snis'] = snis if snis && !snis.empty?
-        hash['tags'] = tags if tags
+        hash['tags'] = tags unless tags.empty?
         [*options[:exclude]].each do |exclude|
           hash.delete(exclude.to_s)
         end

data/lib/skull_island/resources/consumer.rb

@@ -7,22 +7,33 @@ module SkullIsland
     #
     # @see https://docs.konghq.com/1.1.x/admin-api/#consumer-object Consumer API definition
     class Consumer < Resource
+      include Helpers::Meta
+
       property :username
       property :custom_id
       property :created_at, read_only: true, postprocess: true
-      property :tags, validate: true
+      property :tags, validate: true, preprocess: true, postprocess: true
 
-      def self.batch_import(data, verbose: false, test: false)
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
+      # rubocop:disable Metrics/AbcSize
+      def self.batch_import(data, verbose: false, test: false, project: nil, time: nil)
         raise(Exceptions::InvalidArguments) unless data.is_a?(Array)
 
+        known_ids = []
+
+        # rubocop:disable Metrics/BlockLength
         data.each_with_index do |resource_data, index|
           resource = new
           resource.username = resource_data['username']
           resource.custom_id = resource_data['custom_id']
           resource.tags = resource_data['tags'] if resource_data['tags']
+          resource.project = project if project
+          resource.import_time = (time || Time.now.utc.to_i) if project
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
+          known_ids << resource.id
 
-          BasicauthCredential.batch_import(
+          known_basic_auths = BasicauthCredential.batch_import(
             (
               resource_data.dig('credentials', 'basic-auth') || []
             ).map { |t| t.merge('consumer' => { 'id' => resource.id }) },
@@ -30,7 +41,7 @@ module SkullIsland
             test: test
           )
 
-          JWTCredential.batch_import(
+          known_jwt_auths = JWTCredential.batch_import(
             (
               resource_data.dig('credentials', 'jwt') || []
             ).map { |t| t.merge('consumer' => { 'id' => resource.id }) },
@@ -38,20 +49,36 @@ module SkullIsland
             test: test
           )
 
-          KeyauthCredential.batch_import(
+          known_key_auths = KeyauthCredential.batch_import(
             (
               resource_data.dig('credentials', 'key-auth') || []
             ).map { |t| t.merge('consumer' => { 'id' => resource.id }) },
             verbose: verbose,
             test: test
           )
+
+          next unless project
+
+          BasicauthCredential.all.reject { |res| known_basic_auths.include?(res.id) }.map do |res|
+            puts "[WARN] ! Removing #{res.class.name} (#{res.id})"
+            res.destroy
+          end
+
+          JWTCredential.all.reject { |res| known_jwt_auths.include?(res.id) }.map do |res|
+            puts "[WARN] ! Removing #{res.class.name} (#{res.id})"
+            res.destroy
+          end
+
+          KeyauthCredential.all.reject { |res| known_key_auths.include?(res.id) }.map do |res|
+            puts "[WARN] ! Removing #{res.class.name} (#{res.id})"
+            res.destroy
+          end
         end
+        # rubocop:enable Metrics/BlockLength
+
+        cleanup_except(project, known_ids) if project
       end
 
-      # Convenience method to add upstream targets
-      # rubocop:disable Metrics/AbcSize
-      # rubocop:disable Metrics/CyclomaticComplexity
-      # rubocop:disable Metrics/PerceivedComplexity
       def add_credential!(details)
         r = if [BasicauthCredential, JWTCredential, KeyauthCredential].include? details.class
               details
@@ -100,7 +127,7 @@ module SkullIsland
         hash = { 'username' => username, 'custom_id' => custom_id }
         creds = credentials_for_export
         hash['credentials'] = creds unless creds.empty?
-        hash['tags'] = tags if tags
+        hash['tags'] = tags unless tags.empty?
         [*options[:exclude]].each do |exclude|
           hash.delete(exclude.to_s)
         end

data/lib/skull_island/resources/jwt_credential.rb

@@ -20,17 +20,22 @@ module SkullIsland
       def self.batch_import(data, verbose: false, test: false)
         raise(Exceptions::InvalidArguments) unless data.is_a?(Array)
 
+        known_ids = []
+
         data.each_with_index do |resource_data, index|
           resource = new
           resource.algorithm = resource_data['algorithm']
-          resource.key = resource_data['key'] if resource_data['key']
-          resource.secret = resource_data['secret'] if resource_data['secret']
+          resource.delayed_set(:key, resource_data, 'key') if resource_data['key']
+          resource.delayed_set(:secret, resource_data, 'secret') if resource_data['secret']
           if resource_data['rsa_public_key']
-            resource.rsa_public_key = resource_data['rsa_public_key']
+            resource.delayed_set(:rsa_public_key, resource_data, 'rsa_public_key')
           end
           resource.delayed_set(:consumer, resource_data, 'consumer')
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
+          known_ids << resource.id
         end
+
+        known_ids
       end
 
       def self.relative_uri
@@ -65,6 +70,10 @@ module SkullIsland
         false
       end
 
+      def project
+        consumer ? consumer.project : nil
+      end
+
       private
 
       def postprocess_consumer(value)

data/lib/skull_island/resources/keyauth_credential.rb

@@ -19,7 +19,7 @@ module SkullIsland
 
         data.each_with_index do |resource_data, index|
           resource = new
-          resource.key = resource_data['key']
+          resource.delayed_set(:key, resource_data, 'key')
           resource.delayed_set(:consumer, resource_data, 'consumer')
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
         end

data/lib/skull_island/resources/plugin.rb

@@ -7,6 +7,8 @@ module SkullIsland
     #
     # @see https://docs.konghq.com/1.1.x/admin-api/#plugin-object Plugin API definition
     class Plugin < Resource
+      include Helpers::Meta
+
       property :name
       property :enabled, type: :boolean
       property :run_on, validate: true
@@ -15,11 +17,15 @@ module SkullIsland
       property :route, validate: true, preprocess: true, postprocess: true
       property :service, validate: true, preprocess: true, postprocess: true
       property :created_at, read_only: true, postprocess: true
-      property :tags, validate: true
+      property :tags, validate: true, preprocess: true, postprocess: true
 
-      def self.batch_import(data, verbose: false, test: false)
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
+      def self.batch_import(data, verbose: false, test: false, project: nil, time: nil)
         raise(Exceptions::InvalidArguments) unless data.is_a?(Array)
 
+        known_ids = []
+
         data.each_with_index do |resource_data, index|
           resource = new
           resource.name = resource_data['name']
@@ -27,12 +33,19 @@ module SkullIsland
           resource.run_on = resource_data['run_on'] if resource_data['run_on']
           resource.config = resource_data['config'].deep_sort if resource_data['config']
           resource.tags = resource_data['tags'] if resource_data['tags']
+          resource.project = project if project
+          resource.import_time = (time || Time.now.utc.to_i) if project
           resource.delayed_set(:consumer, resource_data, 'consumer')
           resource.delayed_set(:route, resource_data, 'route')
           resource.delayed_set(:service, resource_data, 'service')
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
+          known_ids << resource.id
         end
+
+        cleanup_except(project, known_ids) if project
       end
+      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/PerceivedComplexity
 
       def self.enabled_names(api_client: APIClient.instance)
         api_client.get("#{relative_uri}/enabled")['enabled_plugins']
@@ -55,7 +68,7 @@ module SkullIsland
         hash['consumer'] = "<%= lookup :consumer, '#{consumer.username}' %>" if consumer
         hash['route'] = "<%= lookup :route, '#{route.name}' %>" if route
         hash['service'] = "<%= lookup :service, '#{service.name}' %>" if service
-        hash['tags'] = tags if tags
+        hash['tags'] = tags unless tags.empty?
         [*options[:exclude]].each do |exclude|
           hash.delete(exclude.to_s)
         end

data/lib/skull_island/resources/route.rb

@@ -7,6 +7,8 @@ module SkullIsland
     #
     # @see https://docs.konghq.com/1.1.x/admin-api/#route-object Route API definition
     class Route < Resource
+      include Helpers::Meta
+
       property :name
       property :methods
       property :paths
@@ -21,14 +23,16 @@ module SkullIsland
       property :service, validate: true, preprocess: true, postprocess: true
       property :created_at, read_only: true, postprocess: true
       property :updated_at, read_only: true, postprocess: true
-      property :tags, validate: true
+      property :tags, validate: true, preprocess: true, postprocess: true
 
       # rubocop:disable Metrics/CyclomaticComplexity
       # rubocop:disable Metrics/PerceivedComplexity
       # rubocop:disable Metrics/AbcSize
-      def self.batch_import(data, verbose: false, test: false)
+      def self.batch_import(data, verbose: false, test: false, project: nil, time: nil)
         raise(Exceptions::InvalidArguments) unless data.is_a?(Array)
 
+        known_ids = []
+
         data.each_with_index do |rdata, index|
           resource = new
           resource.name = rdata['name']
@@ -41,9 +45,14 @@ module SkullIsland
           resource.preserve_host = rdata['preserve_host'] unless rdata['preserve_host'].nil?
           resource.snis = rdata['snis'] if rdata['snis']
           resource.tags = rdata['tags'] if rdata['tags']
+          resource.project = project if project
+          resource.import_time = (time || Time.now.utc.to_i) if project
           resource.delayed_set(:service, rdata, 'service')
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
+          known_ids << resource.id
         end
+
+        cleanup_except(project, known_ids) if project
       end
       # rubocop:enable Metrics/CyclomaticComplexity
       # rubocop:enable Metrics/PerceivedComplexity
@@ -68,7 +77,7 @@ module SkullIsland
         }
         hash['service'] = "<%= lookup :service, '#{service.name}' %>" if service
         hash['snis'] = snis if snis && !snis.empty?
-        hash['tags'] = tags if tags
+        hash['tags'] = tags unless tags.empty?
         [*options[:exclude]].each do |exclude|
           hash.delete(exclude.to_s)
         end

data/lib/skull_island/resources/service.rb

@@ -7,6 +7,8 @@ module SkullIsland
     #
     # @see https://docs.konghq.com/1.1.x/admin-api/#service-object Service API definition
     class Service < Resource
+      include Helpers::Meta
+
       property :name
       property :retries
       property :protocol,           validate: true, required: true
@@ -18,27 +20,32 @@ module SkullIsland
       property :read_timeout,       validate: true
       property :created_at, read_only: true, postprocess: true
       property :updated_at, read_only: true, postprocess: true
-      property :tags, validate: true
+      property :tags, validate: true, preprocess: true, postprocess: true
 
       # rubocop:disable Metrics/CyclomaticComplexity
       # rubocop:disable Metrics/PerceivedComplexity
       # rubocop:disable Metrics/AbcSize
-      def self.batch_import(data, verbose: false, test: false)
+      def self.batch_import(data, verbose: false, test: false, project: nil, time: nil)
         raise(Exceptions::InvalidArguments) unless data.is_a?(Array)
 
+        known_ids = []
+
         data.each_with_index do |rdata, index|
           resource = new
           resource.name = rdata['name']
           resource.retries = rdata['retries'] if rdata['retries']
           resource.protocol = rdata['protocol']
-          resource.host = rdata['host']
-          resource.port = rdata['port']
+          resource.delayed_set(:host, rdata, 'host')
+          resource.delayed_set(:port, rdata, 'port')
           resource.path = rdata['path'] if rdata['path']
           resource.connect_timeout = rdata['connect_timeout'] if rdata['connect_timeout']
           resource.write_timeout = rdata['write_timeout'] if rdata['write_timeout']
           resource.read_timeout = rdata['read_timeout'] if rdata['read_timeout']
           resource.tags = rdata['tags'] if rdata['tags']
+          resource.project = project if project
+          resource.import_time = (time || Time.now.utc.to_i) if project
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
+          known_ids << resource.id
 
           Route.batch_import(
             (rdata['routes'] || []).map { |r| r.merge('service' => { 'id' => resource.id }) },
@@ -46,6 +53,8 @@ module SkullIsland
             test: test
           )
         end
+
+        cleanup_except(project, known_ids) if project
       end
       # rubocop:enable Metrics/CyclomaticComplexity
       # rubocop:enable Metrics/PerceivedComplexity
@@ -59,6 +68,11 @@ module SkullIsland
         r.save
       end
 
+      def destroy
+        routes.each(&:destroy)
+        super
+      end
+
       # Provides a collection of related {Route} instances
       def routes
         Route.where(:service, self, api_client: api_client)
@@ -82,7 +96,7 @@ module SkullIsland
           'read_timeout' => read_timeout
         }
         hash['routes'] = routes.collect { |route| route.export(exclude: 'service') }
-        hash['tags'] = tags if tags
+        hash['tags'] = tags unless tags.empty?
         [*options[:exclude]].each do |exclude|
           hash.delete(exclude.to_s)
         end

data/lib/skull_island/resources/upstream.rb

@@ -7,6 +7,8 @@ module SkullIsland
     #
     # @see https://docs.konghq.com/1.1.x/admin-api/#upstream-objects Upstream API definition
     class Upstream < Resource
+      include Helpers::Meta
+
       property :name, required: true, validate: true
       property :slots, validate: true
       property :hash_on, validate: true
@@ -17,14 +19,16 @@ module SkullIsland
       property :hash_on_cookie_path, validate: true
       property :healthchecks, validate: true
       property :created_at, read_only: true, postprocess: true
-      property :tags, validate: true
+      property :tags, validate: true, preprocess: true, postprocess: true
 
       # rubocop:disable Metrics/CyclomaticComplexity
       # rubocop:disable Metrics/PerceivedComplexity
       # rubocop:disable Metrics/AbcSize
-      def self.batch_import(data, verbose: false, test: false)
+      def self.batch_import(data, verbose: false, test: false, project: nil, time: nil)
         raise(Exceptions::InvalidArguments) unless data.is_a?(Array)
 
+        known_ids = []
+
         data.each_with_index do |rdata, index|
           resource = new
           resource.name = rdata['name']
@@ -41,15 +45,22 @@ module SkullIsland
           end
           resource.healthchecks = rdata['healthchecks'] if rdata['healthchecks']
           resource.tags = rdata['tags'] if rdata['tags']
+          resource.project = project if project
+          resource.import_time = (time || Time.now.utc.to_i) if project
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
+          known_ids << resource.id
           puts '[INFO] Processing UpstreamTarget entries...' if verbose
 
           UpstreamTarget.batch_import(
             (rdata['targets'] || []).map { |t| t.merge('upstream' => { 'id' => resource.id }) },
             verbose: verbose,
-            test: test
+            test: test,
+            project: project,
+            time: time
           )
         end
+
+        cleanup_except(project, known_ids) if project
       end
       # rubocop:enable Metrics/CyclomaticComplexity
       # rubocop:enable Metrics/PerceivedComplexity
@@ -113,7 +124,7 @@ module SkullIsland
           'healthchecks' => healthchecks
         }
         hash['targets'] = targets.collect { |target| target.export(exclude: 'upstream') }
-        hash['tags'] = tags if tags
+        hash['tags'] = tags unless tags.empty?
         [*options[:exclude]].each do |exclude|
           hash.delete(exclude.to_s)
         end

data/lib/skull_island/resources/upstream_target.rb

@@ -7,6 +7,8 @@ module SkullIsland
     #
     # @see https://docs.konghq.com/1.1.x/admin-api/#target-object Target API definition
     class UpstreamTarget < Resource
+      include Helpers::Meta
+
       property :target, required: true, validate: true, preprocess: true
       property(
         :upstream,
@@ -14,20 +16,31 @@ module SkullIsland
       )
       property :weight, validate: true
       property :created_at, read_only: true, postprocess: true
-      property :tags, validate: true
+      property :tags, validate: true, preprocess: true, postprocess: true
 
-      def self.batch_import(data, verbose: false, test: false)
+      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable Metrics/PerceivedComplexity
+      def self.batch_import(data, verbose: false, test: false, project: nil, time: nil)
         raise(Exceptions::InvalidArguments) unless data.is_a?(Array)
 
+        known_ids = []
+
         data.each_with_index do |resource_data, index|
           resource = new
-          resource.target = resource_data['target']
+          resource.delayed_set(:target, resource_data, 'target')
           resource.delayed_set(:upstream, resource_data, 'upstream')
           resource.weight = resource_data['weight'] if resource_data['weight']
           resource.tags = resource_data['tags'] if resource_data['tags']
+          resource.project = project if project
+          resource.import_time = (time || Time.now.utc.to_i) if project
           resource.import_update_or_skip(index: index, verbose: verbose, test: test)
+          known_ids << resource.id
         end
+
+        cleanup_except(project, known_ids) if project
       end
+      # rubocop:enable Metrics/CyclomaticComplexity
+      # rubocop:enable Metrics/PerceivedComplexity
 
       def self.all(options = {})
         api_client = options[:api_client] || APIClient.instance
@@ -68,7 +81,7 @@ module SkullIsland
       def export(options = {})
         hash = { 'target' => target, 'weight' => weight }
         hash['upstream'] = "<%= lookup :upstream, '#{upstream.name}' %>" if upstream
-        hash['tags'] = tags if tags
+        hash['tags'] = tags unless tags.empty?
         [*options[:exclude]].each do |exclude|
           hash.delete(exclude.to_s)
         end
@@ -79,19 +92,7 @@ module SkullIsland
       end
 
       def modified_existing?
-        return false unless new?
-
-        # Find routes of the same name
-        same_target_and_upstream = self.class.where(:target, target).and(:upstream, upstream)
-
-        existing = same_target_and_upstream.size == 1 ? same_target_and_upstream.first : nil
-
-        if existing
-          @entity['id'] = existing.id
-          save
-        else
-          false
-        end
+        false # Upstream Targets can not be PATCHed, so can not be modified
       end
 
       private

data/lib/skull_island/validations/resource.rb

@@ -24,7 +24,7 @@ module SkullIsland
       def validate_tags(value)
         # allow only valid hostnames
         value.each do |tag|
-          return false unless tag.is_a?(String) && tag.match?(/\w_-\.~/)
+          return false unless tag.is_a?(String) && tag.match?(/^[\w_\-\.~]+$/)
         end
         true
       end

data/lib/skull_island/version.rb

@@ -4,6 +4,6 @@ module SkullIsland
   VERSION = [
     1, # Major
     2, # Minor
-    0  # Patch
+    2  # Patch
   ].join('.')
 end

data/skull_island.gemspec

@@ -40,5 +40,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rubocop', '~> 0.50'
   spec.add_development_dependency 'simplecov', '~> 0.15'
   spec.add_development_dependency 'travis', '~> 1.8'
-  spec.add_development_dependency 'yard', '~> 0.9'
+  spec.add_development_dependency 'yard', '~> 0.9.20'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skull_island
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.2.2
 platform: ruby
 authors:
 - Jonathan Gnagy
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-26 00:00:00.000000000 Z
+date: 2019-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: deepsort
@@ -198,14 +198,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.20
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.20
 description: A Ruby SDK for Kong 0.14.x
 email:
 - jonathan.gnagy@gmail.com
@@ -242,6 +242,7 @@ files:
 - lib/skull_island/exceptions/invalid_where_query.rb
 - lib/skull_island/exceptions/new_instance_with_id.rb
 - lib/skull_island/helpers/api_client.rb
+- lib/skull_island/helpers/meta.rb
 - lib/skull_island/helpers/migration.rb
 - lib/skull_island/helpers/resource.rb
 - lib/skull_island/helpers/resource_class.rb
@@ -284,7 +285,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubyforge_project: 
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Ruby SDK for Kong