MovableInkAWS 2.7.5 → 2.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96e412e659e0f48c3a469c1151da0e2e610aaafe2b0817ad22e7dae4eae39765
-  data.tar.gz: 5ae7c22cfedb9ddbd89c902f9531e01f88293e6843e1d057b9c6ca1f818b49b9
+  metadata.gz: f349ad96411750bb34591fdab29321576d2d253571311ac28b17945befe0da25
+  data.tar.gz: 1dfddfb2d024bf894a6b769115e57116a711458788da44920f15c1a75a9c0d55
 SHA512:
-  metadata.gz: a877074eb719b30bdba76b22176e3cfc7965760a93de3d98cb9197a1e362c190d8a559f7cebdc5ebb0ad46138fdb8992622fd40c753cf593ced446aa5c68ed8f
-  data.tar.gz: 399aae9bbe6a4f14fa2fae2cb44c9bc2b378b898af8961bf2a8f01f3555f7db613c350cfedced06ffc357e863f7415a8e4a57f95c6eaabd76655ded6e080e2c9
+  metadata.gz: 65af34acc71341c2c30f7e05cfc43c8712cc92b659f3505dbf520904524da80ccbd17e0e6e2430ee40fc406d25c200a9f3ca8088733bccbee0692a1cc11b83b5
+  data.tar.gz: f4f630f2920477c57af48c5613207e646c0d870029ee72663250fdc95dbe108d652f9786b2af97940c087195c1611aab11f10ccf5a5a158d5d2931cc6219600c

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    MovableInkAWS (2.7.5)
+    MovableInkAWS (2.8.0)
       aws-sdk-athena (~> 1)
       aws-sdk-autoscaling (~> 1)
       aws-sdk-cloudwatch (~> 1)

data/lib/movable_ink/aws/ec2.rb

@@ -211,12 +211,26 @@ module MovableInk
         private_ip_addresses(ordered_instances)
       end
 
-      def redis_by_role(role, port)
-        instance_ip_addresses_by_role(role: role)
-          .shuffle
-          .inject([]) { |redii, instance|
-            redii.push({"host" => instance, "port" => port})
-          }
+      def redis_by_role(role, port, availability_zones = [])
+        if availability_zones.class != Array
+          raise MovableInk::AWS::Errors::AvailabilityZonesListInvalidError
+        end
+
+        redis_instances = []
+
+        def redis(host, port)
+          {"host" => host, "port" => port}
+        end
+
+        if availability_zones.length > 0
+          availability_zones.each do |az|
+            redis_instances << instance_ip_addresses_by_role(role: role, availability_zone: az).inject([]) { |redii, instance| redii.push(redis(instance, port)) }
+          end
+        else
+          redis_instances << instance_ip_addresses_by_role(role: role).inject([]) { |redii, instance| redii.push(redis(instance, port)) }
+        end
+
+        redis_instances.flatten.shuffle
       end
 
       def elastic_ips

data/lib/movable_ink/aws/errors.rb

@@ -8,6 +8,7 @@ module MovableInk
       class InvalidDiscoveryTypeError < StandardError; end
       class RoleNameRequiredError < StandardError; end
       class RoleNameInvalidError < StandardError; end
+      class AvailabilityZonesListInvalidError < StandardError; end
 
       class ExpectedError
         def initialize(error_class, message_patterns = [])

data/lib/movable_ink/aws/route53.rb

@@ -35,11 +35,20 @@ module MovableInk
       end
 
       def list_all_r53_resource_record_sets(hosted_zone_id, client = nil)
-        run_with_backoff do
-          route53(client).list_resource_record_sets({
-            hosted_zone_id: hosted_zone_id
-          }).flat_map(&:resource_record_sets)
+        resp = run_with_backoff { route53(client).list_resource_record_sets({ hosted_zone_id: hosted_zone_id }) }
+        rrs = resp.resource_record_sets
+
+        # https://docs.aws.amazon.com/Route53/latest/APIReference/API_ListResourceRecordSets.html
+        while resp.is_truncated
+          resp = run_with_backoff { route53(client).list_resource_record_sets({
+            hosted_zone_id: hosted_zone_id,
+            start_record_name: resp.next_record_name,
+            start_record_type: resp.next_record_type,
+            start_record_identifier: resp.next_record_identifier
+          }) }
+          rrs += resp.resource_record_sets
         end
+        rrs
       end
 
       def list_health_checks(client = nil)
@@ -62,6 +71,18 @@ module MovableInk
           get_health_check_tags(health_check.id, client).detect { |tag| tag.key == key && tag.value.include?(value) }
         end
       end
+
+      def list_hosted_zones(client: nil)
+        resp = run_with_backoff { route53(client).list_hosted_zones() }
+        zones = resp.hosted_zones
+
+        # https://docs.aws.amazon.com/Route53/latest/APIReference/API_ListHostedZones.html
+        while resp.is_truncated
+          resp = run_with_backoff { route53(client).list_hosted_zones({marker: resp.next_marker}) }
+          zones += resp.hosted_zones
+        end
+        zones
+      end
     end
   end
 end

data/lib/movable_ink/version.rb

@@ -1,5 +1,5 @@
 module MovableInk
   class AWS
-    VERSION = '2.7.5'
+    VERSION = '2.8.0'
   end
 end

data/spec/ec2_spec.rb

@@ -589,6 +589,34 @@ describe MovableInk::AWS::EC2 do
           }
         ]])
       }
+      let(:redis_data_cross_azs) { ec2.stub_data(:describe_instances, reservations: [
+        instances: [
+          {
+            tags: [
+              {
+                key: 'mi:roles',
+                value: 'visitor_redis'
+              }
+            ],
+            private_ip_address: '10.0.0.1',
+            placement: {
+              availability_zone: availability_zone
+            }
+          },
+          {
+            tags: [
+              {
+                key: 'mi:roles',
+                value: 'visitor_redis'
+              }
+            ],
+            private_ip_address: '10.0.0.2',
+            placement: {
+              availability_zone: 'us-east-1z'
+            }
+          }
+        ]])
+      }
       let(:redii) { [{"host" => '10.0.0.1', "port" => 6379},{"host" => '10.0.0.2', "port" => 6379}] }
 
       it "should return redis IPs and ports" do
@@ -600,6 +628,20 @@ describe MovableInk::AWS::EC2 do
 
         expect(aws.redis_by_role('visitor_redis', port)).to match_array(redii)
       end
+
+      it "should return redis IPs and ports only from specified AZs" do
+        ec2.stub_responses(:describe_instances, redis_data_cross_azs)
+        allow(aws).to receive(:mi_env).and_return('test')
+        allow(aws).to receive(:availability_zone).and_return(availability_zone)
+        allow(aws).to receive(:my_region).and_return('us-east-1')
+        allow(aws).to receive(:ec2).and_return(ec2)
+
+        expect(aws.redis_by_role('visitor_redis', port, ['us-east-1z'])).to match_array([{"host" => '10.0.0.2', "port" => 6379}])
+      end
+
+      it "should throw exception if specified AZs list isnt array" do
+        expect { aws.redis_by_role('visitor_redis', port, 'us-east-1z') }.to raise_error(MovableInk::AWS::Errors::AvailabilityZonesListInvalidError)
+      end
     end
 
     context "elastic IPs" do

data/spec/route53_spec.rb

@@ -3,9 +3,60 @@ require_relative '../lib/movable_ink/aws'
 describe MovableInk::AWS::Route53 do
   let(:aws) { MovableInk::AWS.new }
 
+  context 'hosted zones' do
+    let(:route53) { Aws::Route53::Client.new(stub_responses: true) }
+
+    it "should list all hosted zones" do
+      zones_data = route53.stub_data(:list_hosted_zones, is_truncated: false, hosted_zones: [{
+        id: '123456789X',
+        name: 'domain.tld'
+      }])
+      route53.stub_responses(:list_hosted_zones, zones_data)
+      allow(aws).to receive(:route53).and_return(route53)
+
+      expect(aws.list_hosted_zones.count).to eq(1)
+      expect(aws.list_hosted_zones.first.id).to eq('123456789X')
+      expect(aws.list_hosted_zones.first.name).to eq('domain.tld')
+    end
+
+    it "should list all hosted zones w/ pagination" do
+      zones_response_1 = {
+        is_truncated: true,
+        next_marker: 'this is fake marker',
+        marker: 'this is fake marker',
+        max_items: 1,
+        hosted_zones: [{
+          id: '123456789X',
+          name: 'domain.tld',
+          caller_reference: '123'
+      }]}
+
+      zones_response_2 = {
+        is_truncated: false,
+        next_marker: nil,
+        marker: 'this is fake marker',
+        max_items: 1,
+        hosted_zones: [{
+          id: 'X123456789',
+          name: 'tld.domain',
+          caller_reference: '321'
+      }]}
+
+      route53.stub_responses(:list_hosted_zones, [ zones_response_1, zones_response_2 ])
+      allow(aws).to receive(:route53).and_return(route53)
+
+      zones = aws.list_hosted_zones
+      expect(zones.count).to eq(2)
+      expect(zones.first.id).to eq('123456789X')
+      expect(zones.first.name).to eq('domain.tld')
+      expect(zones[1].id).to eq('X123456789')
+      expect(zones[1].name).to eq('tld.domain')
+    end
+  end
+
   context "resource record sets" do
     let(:route53) { Aws::Route53::Client.new(stub_responses: true) }
-    let(:rrset_data) { route53.stub_data(:list_resource_record_sets, resource_record_sets: [
+    let(:rrset_data) { route53.stub_data(:list_resource_record_sets, is_truncated: false, resource_record_sets: [
         {
           name: 'host1.domain.tld.',
           set_identifier: '10_0_0_1',
@@ -34,6 +85,42 @@ describe MovableInk::AWS::Route53 do
       expect(aws.resource_record_sets('Z123')[2].name).to eq('host2-other.domain.tld.')
     end
 
+    it "should retrieve all rrsets for zone w/ pagination" do
+
+      rrs_response_1 = {
+        is_truncated: true,
+        max_items: 1,
+        next_record_name: 'record2.domain.',
+        next_record_type: 'A',
+        next_record_identifier: nil,
+        resource_record_sets: [{
+          name: 'record1.domain.',
+          type: 'A',
+          set_identifier: nil
+      }]}
+
+      rrs_response_2 = {
+        is_truncated: false,
+        max_items: 1,
+        next_record_name: nil,
+        next_record_type: nil,
+        next_record_identifier: nil,
+        resource_record_sets: [{
+          name: 'record2.domain.',
+          type: 'A',
+          set_identifier: nil
+      }]}
+
+      rrset_data = [rrs_response_1, rrs_response_2]
+      route53.stub_responses(:list_resource_record_sets, rrset_data)
+      allow(aws).to receive(:route53).and_return(route53)
+
+      rrs = aws.resource_record_sets('Z123')
+      expect(rrs.count).to eq(2)
+      expect(rrs[0].name).to eq('record1.domain.')
+      expect(rrs[1].name).to eq('record2.domain.')
+    end
+
     it "returns all sets with an identifier" do
       route53.stub_responses(:list_resource_record_sets, rrset_data)
       allow(aws).to receive(:route53).and_return(route53)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: MovableInkAWS
 version: !ruby/object:Gem::Version
-  version: 2.7.5
+  version: 2.8.0
 platform: ruby
 authors:
 - MI SRE
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-22 00:00:00.000000000 Z
+date: 2022-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-core