activerecord-aurora-serverless-adapter 5.2.1 → 5.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3ad604df999e26ca0a4db13f00ca1c71616b6dbceecc24549cf38ea4c9c28a5
-  data.tar.gz: b6a78002100ba027c0a83809f77de08887d40e360051c6efc583d181192b93f4
+  metadata.gz: 60b59ea21f9e813832a9b157fd65ae5d9d686afb0f04c51700cd5664eeb2edcd
+  data.tar.gz: 6555d62f42ccb9c421382353f5fa14990c456a2d3555437b03965a3ed0d3d97d
 SHA512:
-  metadata.gz: 7dccc50158b4681545a944ddb3d4df0c28dbad0db48ee005edcf195aeaf5c8413ca58e97567cee3d7945afd853b962f224e21487f5759df739656b6e11a4c836
-  data.tar.gz: ada81597e30e524a533391229e684b9d0de77171c92a875e7caa7ab4ed3d260cc303180f7d6f9a54aa85257c8f5cb83727b228ae2329e12c0c20d947c996b92c
+  metadata.gz: 4271e26bc9ad5f0a8de508b30914cd39d7d870e45c5985a6857ec1881fcc2c6ff6b23a38d1c611c9beba607a658cabd4723b8d45140afd365121a0d992f9b762
+  data.tar.gz: 4cc64d2e19eff3da7f7fdc5d939f90a40c952cfdaf72fd6168b3fd7280fec7aa409038707c6c691da3522a29c49dc313f0949e5f01b6fb21b43d3fe491bb9e2a

data/Gemfile.lock

@@ -67,9 +67,10 @@ GIT
 PATH
   remote: .
   specs:
-    activerecord-aurora-serverless-adapter (5.2.1)
+    activerecord-aurora-serverless-adapter (5.2.2)
       activerecord (~> 5.2.0)
       aws-sdk-rdsdataservice
+      retriable
 
 GEM
   remote: https://rubygems.org/
@@ -81,8 +82,8 @@ GEM
       thor (>= 0.14.0)
     arel (9.0.0)
     aws-eventstream (1.0.3)
-    aws-partitions (1.260.0)
-    aws-sdk-core (3.86.0)
+    aws-partitions (1.265.0)
+    aws-sdk-core (3.89.1)
       aws-eventstream (~> 1.0, >= 1.0.2)
       aws-partitions (~> 1, >= 1.239.0)
       aws-sigv4 (~> 1.1)
@@ -141,6 +142,7 @@ GEM
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
     rake (13.0.1)
+    retriable (3.1.2)
     ruby-progressbar (1.10.1)
     sprockets (4.0.0)
       concurrent-ruby (~> 1.0)

data/activerecord-aurora-serverless-adapter.gemspec

@@ -12,13 +12,16 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/customink/activerecord-aurora-serverless-adapter'
   spec.license       = 'MIT'
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+    `git ls-files -z`.split("\x0").reject do |f|
+      f.match(%r{^(test|spec|features|docker)/i})
+    end
   end
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
   spec.add_runtime_dependency     'activerecord', '~> 5.2.0'
   spec.add_runtime_dependency     'aws-sdk-rdsdataservice'
+  spec.add_runtime_dependency     'retriable'
   spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'dotenv'
   spec.add_development_dependency 'minitest'

data/lib/active_record/connection_adapters/aurora_serverless/client.rb

@@ -23,6 +23,14 @@ module ActiveRecord
           "#<#{self.class} database: #{database.inspect}, raw_client: #{raw_client.inspect}>"
         end
 
+        def execute_statement_retry(sql)
+          if @connected
+            execute_statement(sql)
+          else
+            auto_paused_retry { execute_statement(sql) }
+          end
+        end
+
         def execute_statement(sql)
           id = @transactions.first
           debug_transactions "EXECUTE: #{sql}", id
@@ -34,6 +42,7 @@ module ActiveRecord
             include_result_metadata: true,
             transaction_id: id
           }).tap do |r|
+            @connected = true
             @affected_rows = affected_rows_result(r)
             @last_id = last_id_result(r)
           end
@@ -100,6 +109,22 @@ module ActiveRecord
           field.long_value || field.string_value || field.double_value
         end
 
+        def auto_paused_retry
+          error_klass = Aws::RDSDataService::Errors::BadRequestException
+          error_msg = /last packet sent successfully to the server was/
+          retry_msg = 'Aurora auto paused, retrying...'
+          on_retry = Proc.new { sleep(1) ; ::Rails.logger.info(retry_msg)  }
+          Retriable.retriable({
+            on: { error_klass => error_msg },
+            on_retry: on_retry,
+            tries: auto_paused_retry_count
+          }) { yield }
+        end
+
+        def auto_paused_retry_count
+          10
+        end
+
         def debug_transactions(name, id = 'NOID')
           return unless @debug_transactions
           ActiveRecord::Base.logger.debug "  \e[36m#{name} #{id} #{object_id}\e[0m"

data/lib/active_record/connection_adapters/aurora_serverless/version.rb

@@ -1,7 +1,7 @@
 module ActiveRecord
   module ConnectionAdapters
     module AuroraServerless
-      VERSION = "5.2.1"
+      VERSION = "5.2.2"
     end
   end
 end

data/test/aasa_helper.rb

@@ -0,0 +1,163 @@
+ENV['AASA_ENV'] = 'test'
+require 'bundler/setup'
+Bundler.require :default, :development
+Dotenv.load('.env')
+require 'minitest/spec'
+require 'minitest/autorun'
+require 'minitest/retry'
+require 'minitest/reporters'
+require_relative 'support/aasa_mysqlpatch'
+require 'cases/helper' unless ENV['TEST_FILES'] || ENV['ONLY_AASA']
+require_relative 'support/aasa_coerceable'
+require_relative 'support/aasa_env'
+require_relative 'support/aasa_fixtures'
+require_relative 'support/aasa_minitest'
+Rails.backtrace_cleaner.remove_silencers! if ENV['REMOVE_SILENCERS']
+
+module ActiveRecord
+  module ConnectionAdapters
+    module AuroraServerless
+      class TestCase < Minitest::Spec
+
+        before { setup_table }
+        after  { drop_table }
+
+        private
+
+        def client
+          @client ||= AuroraServerless::Client.new(
+            'activerecord_unittest',
+            ENV['AASA_RESOURCE_ARN'],
+            ENV['AASA_SECRET_ARN']
+          )
+        end
+
+        def execute(sql)
+          r = client.execute_statement(sql)
+          # TODO: [PG] Make this a conditional result wrapper.
+          AuroraServerless::Mysql2::Result.new(r)
+        end
+
+        def setup_table
+          # TODO: [PG] Make this a conditional for constant SQL.
+          execute MYSQL_CREATE_TABLE_SQL
+          execute MYSQL_INSERT_SQL
+        end
+
+        def drop_table
+          execute 'DROP TABLE IF EXISTS aurora_test'
+        end
+
+        MYSQL_CREATE_TABLE_SQL = %[
+          CREATE TABLE IF NOT EXISTS aurora_test (
+            id int NOT NULL AUTO_INCREMENT,
+            PRIMARY KEY (id),
+            null_test VARCHAR(10),
+            bit_test BIT,
+            tiny_int_test TINYINT,
+            small_int_test SMALLINT,
+            medium_int_test MEDIUMINT,
+            int_test INT,
+            big_int_test BIGINT,
+            float_test FLOAT(10,3),
+            float_zero_test FLOAT(10,3),
+            double_test DOUBLE(10,3),
+            decimal_test DECIMAL(10,3),
+            decimal_zero_test DECIMAL(10,3),
+            date_test DATE,
+            date_time_test DATETIME,
+            timestamp_test TIMESTAMP,
+            time_test TIME,
+            year_test YEAR(4),
+            char_test CHAR(10),
+            varchar_test VARCHAR(10),
+            binary_test BINARY(10),
+            varbinary_test VARBINARY(10),
+            tiny_blob_test TINYBLOB,
+            tiny_text_test TINYTEXT,
+            blob_test BLOB,
+            text_test TEXT,
+            medium_blob_test MEDIUMBLOB,
+            medium_text_test MEDIUMTEXT,
+            long_blob_test LONGBLOB,
+            long_text_test LONGTEXT,
+            enum_test ENUM('val1', 'val2'),
+            set_test SET('val1', 'val2')
+          ) DEFAULT CHARSET=utf8
+        ]
+
+        MYSQL_INSERT_SQL = %[
+          INSERT INTO aurora_test (
+            null_test,
+            bit_test,
+            tiny_int_test,
+            small_int_test,
+            medium_int_test,
+            int_test,
+            big_int_test,
+            float_test,
+            float_zero_test,
+            double_test,
+            decimal_test,
+            decimal_zero_test,
+            date_test,
+            date_time_test,
+            timestamp_test,
+            time_test,
+            year_test,
+            char_test,
+            varchar_test,
+            binary_test,
+            varbinary_test,
+            tiny_blob_test,
+            tiny_text_test,
+            blob_test,
+            text_test,
+            medium_blob_test,
+            medium_text_test,
+            long_blob_test,
+            long_text_test,
+            enum_test,
+            set_test
+          )
+          VALUES (
+            NULL,
+            1,
+            5,
+            32766,
+            8388606,
+            2147483646,
+            9223372036854775806,
+            156.68449197860963,
+            0.0,
+            606682.8877005348,
+            676254.5454545454,
+            0,
+            '2010-4-4',
+            '2010-4-4 11:44:00',
+            '2010-4-4 11:44:00',
+            '11:44:00',
+            2019,
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'abcdefg',
+            'val1',
+            'val1,val2'
+          )
+        ]
+
+      end
+    end
+  end
+end
+
+require "mocha/setup"

data/test/aurora-serverless/bin/aurora-serverless.ts

@@ -0,0 +1,17 @@
+#!/usr/bin/env node
+import "source-map-support/register";
+import cdk = require("@aws-cdk/core");
+import { AuroraServerlessStack } from "../lib/aurora-serverless-stack";
+
+const APP = new cdk.App();
+
+new AuroraServerlessStack(APP, "AuroraServerlessStack", {
+  stackName: "aasa-mysql-unit",
+  description: "Test customink/activerecord-aurora-serverless-adapter.",
+  tags: {
+    env: "dev",
+    group: "shared",
+    application: "activerecord-aurora-serverless-adapter",
+    owner: "kcollins"
+  }
+});

data/test/aurora-serverless/cdk.context.json

@@ -0,0 +1,3 @@
+{
+  "@aws-cdk/core:enableStackNameDuplicates": "true"
+}

data/test/aurora-serverless/cdk.json

@@ -0,0 +1,3 @@
+{
+  "app": "ts-node bin/aurora-serverless.ts"
+}

data/test/aurora-serverless/lib/aurora-serverless-stack.ts

@@ -0,0 +1,194 @@
+// Thanks to https://almirzulic.com/posts/create-serverless-aurora-cluster-with-cdk/
+
+import { CfnOutput, Construct, Stack, StackProps } from "@aws-cdk/core";
+import { Vpc, SubnetType } from "@aws-cdk/aws-ec2";
+import {
+  CfnDBCluster,
+  CfnDBSubnetGroup,
+  CfnDBClusterProps,
+  CfnDBClusterParameterGroup
+} from "@aws-cdk/aws-rds";
+import { CfnSecret, CfnSecretProps } from "@aws-cdk/aws-secretsmanager";
+import { User, Policy, PolicyStatement, CfnAccessKey } from "@aws-cdk/aws-iam";
+
+const MASTER_USER = process.env.AASA_MASTER_USER;
+const MASTER_PASS = process.env.AASA_MASTER_PASS;
+const DB_NAME = "activerecord_unittest";
+const DB_CLUSTER_ID = "aasa-mysql-unit";
+
+function auroraProps(
+  dbName: string,
+  dbClusterId: string,
+  dbSubnetGroupName: string | undefined,
+  dbParamGroup: CfnDBClusterParameterGroup
+) {
+  return {
+    databaseName: dbName,
+    dbClusterIdentifier: dbClusterId,
+    engine: "aurora",
+    engineMode: "serverless",
+    masterUsername: MASTER_USER,
+    masterUserPassword: MASTER_PASS,
+    enableHttpEndpoint: true,
+    port: 3306,
+    dbSubnetGroupName: dbSubnetGroupName,
+    dbClusterParameterGroupName: dbParamGroup.ref,
+    scalingConfiguration: {
+      autoPause: true,
+      minCapacity: 1,
+      maxCapacity: 4,
+      secondsUntilAutoPause: 3600
+    }
+  } as CfnDBClusterProps;
+}
+
+function secretProps(aurora: CfnDBCluster, dbClusterId: string) {
+  return {
+    secretString: JSON.stringify({
+      username: MASTER_USER,
+      password: MASTER_PASS,
+      engine: "mysql",
+      host: aurora.attrEndpointAddress,
+      port: aurora.attrEndpointPort,
+      dbClusterIdentifier: dbClusterId
+    }),
+    description: "AASA Master Credentials."
+  } as CfnSecretProps;
+}
+
+export class AuroraServerlessStack extends Stack {
+  constructor(scope: Construct, id: string, props?: StackProps) {
+    super(scope, id, props);
+
+    // VPC
+
+    const vpc = new Vpc(this, "Vpc", {
+      cidr: "10.0.0.0/16",
+      natGateways: 0,
+      subnetConfiguration: [
+        { name: "aasa_isolated", subnetType: SubnetType.ISOLATED }
+      ]
+    });
+    const subnetIds: string[] = [];
+    vpc.isolatedSubnets.forEach(subnet => {
+      subnetIds.push(subnet.subnetId);
+    });
+
+    // SUBNET GROUP
+
+    const dbSubnetGroup: CfnDBSubnetGroup = new CfnDBSubnetGroup(
+      this,
+      "AuroraSubnetGroup",
+      {
+        dbSubnetGroupDescription: "Subnet group to AASA Aurora",
+        dbSubnetGroupName: "aasa-subnet-group",
+        subnetIds
+      }
+    );
+
+    // RDS PARAMETER GROUP
+
+    const dbParamGroup = new CfnDBClusterParameterGroup(
+      this,
+      "ParameterGroup",
+      {
+        family: "aurora5.6",
+        description: "Test customink/activerecord-aurora-serverless-adapter.",
+        parameters: {
+          innodb_large_prefix: "1",
+          innodb_file_per_table: "1",
+          innodb_file_format: "Barracuda",
+          character_set_client: "utf8mb4",
+          character_set_connection: "utf8mb4",
+          character_set_database: "utf8mb4",
+          character_set_results: "utf8mb4",
+          character_set_server: "utf8mb4",
+          collation_server: "utf8mb4_unicode_ci",
+          collation_connection: "utf8mb4_unicode_ci"
+        }
+      }
+    );
+    dbParamGroup.addDependsOn(dbSubnetGroup);
+
+    // AURORA SERVERLESS CLUSTERS
+
+    const aurora = new CfnDBCluster(
+      this,
+      "AuroraServerless",
+      auroraProps(
+        DB_NAME,
+        DB_CLUSTER_ID,
+        dbSubnetGroup.dbSubnetGroupName,
+        dbParamGroup
+      )
+    );
+    const aurora2 = new CfnDBCluster(
+      this,
+      "AuroraServerless2",
+      auroraProps(
+        `${DB_NAME}2`,
+        `${DB_CLUSTER_ID}2`,
+        dbSubnetGroup.dbSubnetGroupName,
+        dbParamGroup
+      )
+    );
+    aurora.addDependsOn(dbParamGroup);
+    aurora2.addDependsOn(dbParamGroup);
+    new CfnOutput(this, "AASAResourceArn", {
+      value: `arn:aws:rds:${this.region}:${this.account}:cluster:${DB_CLUSTER_ID}`
+    });
+    new CfnOutput(this, "AASAResourceArn2", {
+      value: `arn:aws:rds:${this.region}:${this.account}:cluster:${DB_CLUSTER_ID}2`
+    });
+
+    // SECRETS
+
+    const secret = new CfnSecret(
+      this,
+      "Secret",
+      secretProps(aurora, DB_CLUSTER_ID)
+    );
+    const secret2 = new CfnSecret(
+      this,
+      "Secret2",
+      secretProps(aurora2, `${DB_CLUSTER_ID}2`)
+    );
+    secret.addDependsOn(aurora);
+    secret2.addDependsOn(aurora2);
+    new CfnOutput(this, "AASASecretArn", {
+      value: secret.ref
+    });
+    new CfnOutput(this, "AASASecretArn2", {
+      value: secret2.ref
+    });
+
+    // TEST USER
+
+    const user = new User(this, "TestUser");
+    const policy = new Policy(this, "TestUserPolicy", {
+      statements: [
+        new PolicyStatement({
+          actions: ["rds-data:*"],
+          resources: [
+            `arn:aws:rds:${this.region}:${this.account}:cluster:${DB_CLUSTER_ID}*`,
+            `arn:aws:rds:${this.region}:${this.account}:cluster:${DB_CLUSTER_ID}2*`
+          ]
+        }),
+        new PolicyStatement({
+          actions: ["secretsmanager:*"],
+          resources: [`${secret.ref}*`, `${secret2.ref}*`]
+        })
+      ]
+    });
+    user.attachInlinePolicy(policy);
+    const key = new CfnAccessKey(this, "TestUserKey", {
+      userName: user.userName
+    });
+    new CfnOutput(this, "AASAUserAccessKeyId", {
+      value: key.ref
+    });
+    new CfnOutput(this, "AASAUserSecretAccessKey", {
+      value: key.attrSecretAccessKey
+    });
+  }
+}