aurora-data-api 0.1.0

data/example/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "aurora-data-api-example",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC",
+  "devDependencies": {
+    "serverless": "^3.17.0",
+    "serverless-offline": "^8.8.0",
+    "serverless-ruby-layer": "^1.6.0",
+    "serverless-vpc-plugin": "^1.0.4"
+  }
+}

data/example/serverless.yml

@@ -0,0 +1,184 @@
+service: aurora-data-api-example
+
+frameworkVersion: '3'
+
+provider:
+  name: aws
+  runtime: ruby2.7
+  stage: ${opt:stage, self:custom.defaultStage}
+  endpointType: REGIONAL
+  region: ap-northeast-1
+  environment:
+    STAGE: ${sls:stage}
+    TZ: Asia/Tokyo
+    DATA_API_ENDPOINT:
+        Fn::Join:
+          - ""
+          - - "https://"
+            - !GetAtt "RDSCluster.Endpoint.Address"
+    RDS_RESOURCE_ARN: ${self:custom.db_resource_arn}
+    RDS_SECRET_ARN: !Ref DBSecret
+    PGDATABASE: ${self:custom.myEnvironment.PGDATABASE}
+  iam:
+    role:
+      statements:
+        - Effect: "Allow"
+          Action: lambda:InvokeFunction
+          Resource:
+            - !Join
+              - ""
+              - - "arn:aws:lambda:${self:provider.region}:"
+                - !Ref AWS::AccountId
+                - ":function:"
+                - "*"
+        - Effect: "Allow"
+          Action: secretsmanager:GetSecretValue
+          Resource:
+            - !Ref DBSecret
+        - Effect: "Allow"
+          Action:
+            - rds-data:BatchExecuteStatement
+            - rds-data:BeginTransaction
+            - rds-data:CommitTransaction
+            - rds-data:ExecuteStatement
+            - rds-data:RollbackTransaction
+          Resource:
+            - ${self:custom.db_resource_arn}
+  logs:
+    restApi:
+      accessLogging: true
+      format: '{"requestId":"$context.requestId","ip":"$context.identity.sourceIp","requestTime":"$context.requestTime","httpMethod":"$context.httpMethod","routeKey":"$context.routeKey","status":"$context.status","protocol":"$context.protocol","responseLength":"$context.responseLength","errorMessage":"$context.integrationErrorMessage"}'
+      executionLogging: true
+      level: INFO
+      fullExecutionData: true
+
+plugins:
+  - serverless-offline
+  - serverless-vpc-plugin
+  - serverless-ruby-layer
+
+custom:
+  defaultStage: offline
+  serverless-offline:
+    httpPort: 4000
+  db_resource_arn: !Join
+    - ""
+    - - "arn:aws:rds:${self:provider.region}:"
+      - !Ref AWS::AccountId
+      - ":cluster:"
+      - !Ref RDSCluster
+  myEnvironment:
+    DBMaxCapacity:
+      prod: 4
+    PGDATABASE: mydatabase
+    ALLOWED_ORIGIN:
+      offline: http://localhost:4000
+      prod: "*"  # TODO: You have to specify it
+  vpcConfig:
+    # see https://www.serverless.com/plugins/serverless-vpc-plugin
+    enabled: true
+    cidrBlock: '10.0.0.0/16'
+    createNatGateway: false
+    createNetworkAcl: false
+    createDbSubnet: true
+    createFlowLogs: false
+    createBastionHost: false
+    createNatInstance: false
+    createParameters: true
+    services:
+      - secretsmanager
+    subnetGroups:
+      - rds
+    exportOutputs: true
+
+resources:
+  Resources:
+    RDSCluster:
+      Type: AWS::RDS::DBCluster
+      Properties:
+        DBClusterIdentifier: !Sub "${self:service}-${self:provider.stage}-aurora-psql"
+        MasterUsername: !Join ['', ['{{resolve:secretsmanager:', !Ref DBSecret, ':SecretString:username}}' ]]
+        MasterUserPassword: !Join ['', ['{{resolve:secretsmanager:', !Ref DBSecret, ':SecretString:password}}' ]]
+        DatabaseName: ${self:custom.myEnvironment.PGDATABASE}
+        Engine: aurora-postgresql
+        EngineMode: serverless
+        EnableHttpEndpoint: true
+        EngineVersion: 10.14
+        ScalingConfiguration:
+          AutoPause: true
+          MaxCapacity: ${self:custom.myEnvironment.DBMaxCapacity.${sls:stage}, 2}
+          MinCapacity: 2
+          SecondsUntilAutoPause: 900 # 15 min for example
+        DBSubnetGroupName:
+          Ref: DBSubnetGroup
+    DBSecret:
+      Type: AWS::SecretsManager::Secret
+      Properties:
+        Name: !Sub "${self:service}-${self:provider.stage}-AuroraUserSecret"
+        Description: RDS database auto-generated user password
+        GenerateSecretString:
+          SecretStringTemplate: !Sub '{"username": "${self:provider.stage}Root"}'
+          GenerateStringKey: "password"
+          PasswordLength: 30
+          ExcludeCharacters: '"@/\'
+    DBSubnetGroup:
+      Type: AWS::RDS::DBSubnetGroup
+      Properties:
+        DBSubnetGroupDescription: CloudFormation managed DB subnet group.
+        SubnetIds:
+          - !Ref "DBSubnet1"
+          - !Ref "DBSubnet2"
+          - !Ref "DBSubnet3"
+
+functions:
+  hello:
+    handler: app/handlers/main.hello
+    events:
+      - http:
+          path: hello
+          method: get
+  users:
+    handler: app/handlers/main.users
+    events:
+      - http:
+          path: users
+          method: get
+  create_user:
+    handler: app/handlers/main.create_user
+    events:
+      - http:
+          path: create_user
+          method: post
+  update_user:
+    handler: app/handlers/main.update_user
+    events:
+      - http:
+          path: update_user
+          method: put
+  entries:
+    handler: app/handlers/main.entries
+    events:
+      - http:
+          path: entries
+          method: get
+  create_entry:
+    handler: app/handlers/main.create_entry
+    events:
+      - http:
+          path: create_entry
+          method: post
+  delete_entry:
+    handler: app/handlers/main.delete_entry
+    events:
+      - http:
+          path: delete_entry
+          method: post
+  count_entry:
+    handler: app/handlers/main.count_entry
+    events:
+      - http:
+          path: count_entry
+          method: get
+
+
+

data/exe/aurora-data-api

@@ -0,0 +1,161 @@
+#!/usr/bin/env ruby
+
+require "aurora-data-api/version"
+require "thor"
+
+class Date
+end
+
+module AuroraDataApi
+  class Error < StandardError; end
+
+  class Schema
+    CREATE_TABLE = []
+    ALTER_TABLE = []
+  end
+
+  class Model
+    SCHEMA = {literal_id: :id}
+
+    def self.literal_id(lit)
+      SCHEMA[:literal_id] = lit
+    end
+
+    def self.table(name)
+      SCHEMA[:table_name] = name
+    end
+
+    def self.schema(&block)
+      converter = Converter.new(SCHEMA, self)
+      converter.head
+      converter.instance_eval(&block)
+      converter.tail
+      Schema::CREATE_TABLE << converter.create_table
+      Schema::ALTER_TABLE << converter.alter_table
+    end
+  end
+
+  class Converter
+    TYPES = {
+      ::Date => "date",
+      ::Time => "timestamp with time zone",
+      ::String => "text",
+      ::Integer => "bigint",
+      ::Float => "double precision"
+    }
+
+    def initialize(schema, klass)
+      @table_name = schema[:table_name] || "#{klass.name.downcase}s"
+      @literal_id = schema[:literal_id]
+      @create_table = []
+      @alter_table = []
+    end
+
+    attr_reader :create_table, :alter_table
+
+    def head
+      @create_table << %/CREATE TABLE "#{@table_name}" (/
+      @create_table << %(  "#{@literal_id}" bigint NOT NULL GENERATED ALWAYS AS IDENTITY,)
+    end
+
+    def tail
+      @create_table << %/  PRIMARY KEY ("#{@literal_id}")/
+      @create_table << %/);\n/
+    end
+
+    def timestamp
+      @create_table << %(  "created_at" #{TYPES[Time]} NOT NULL,)
+      @create_table << %(  "updated_at" #{TYPES[Time]} NOT NULL,)
+    end
+
+    def col(name, type, **params)
+      line = "  "
+      case type
+      when Symbol
+        col_name = "#{name}_#{@literal_id}"
+        line << %("#{col_name}" )
+        line << TYPES[Integer]
+        @alter_table << %/ALTER TABLE ONLY "#{@table_name}" ADD CONSTRAINT "#{@table_name}_#{col_name}_fkey" FOREIGN KEY ("#{col_name}") REFERENCES "#{params[:table]}" ("#{@literal_id}");/
+      else
+        line << %("#{name}" )
+        line << TYPES[type]
+      end
+      params.each do |k, v|
+        case k
+        when :null
+          line << " NOT NULL" unless v
+        when :default
+          line << " DEFAULT "
+          line << case v
+          when String, Symbol
+            "'#{v}'"
+          else
+            v.to_s
+          end
+        when :unique
+          line << " UNIQUE" if v
+        end
+      end
+      line << ","
+      @create_table << line
+    end
+  end
+
+  class Tool < Thor
+    DEFAULT_MODELS_DIR = "app/models"
+    DEFAULT_OUTPUT_PATH = "db/schema.sql"
+
+    desc "version", "Print version"
+    def version
+      puts "aurora-data-api v#{AuroraDataApi::VERSION}"
+    end
+
+    desc "export", "Overwrite #{DEFAULT_OUTPUT_PATH} by aggregating #{DEFAULT_MODELS_DIR}/*.rb"
+    option :models, aliases: :m, default: DEFAULT_MODELS_DIR
+    option :output, aliases: :o, default: DEFAULT_OUTPUT_PATH
+    def export
+      models_dir = options[:models]
+      output_path = options[:output]
+      Dir.glob("#{models_dir}/*.rb").each do |rb|
+        load rb
+      end
+      if Schema::CREATE_TABLE.empty? && Schema::ALTER_TABLE.empty?
+        puts "Nothing to be exported."
+        exit 1
+      end
+      overwrite = false
+      if File.exist? output_path
+        print "#{output_path} exists. Overwrite? [Y/n]: "
+        answer = $stdin.gets.chomp
+        if %w[Y y yes].include?(answer.chomp)
+          overwrite = true
+        else
+          puts "Abort."
+          exit 1
+        end
+      else
+        FileUtils.touch output_path
+        overwrite = true
+      end
+      if overwrite
+        File.open output_path, "w" do |f|
+          f.write <<~COMMENT
+            /*
+             * This file was automatically genarated by the command:
+             *   aurora-data-api export --models #{models_dir} --output #{output_path}
+             *
+             * Genarated at #{Time.now}
+             *
+             * https://github.com/hasumikin/aurora-data-api
+             */\n
+          COMMENT
+          f.write Schema::CREATE_TABLE.flatten.join("\n")
+          f.write "\n"
+          f.write Schema::ALTER_TABLE.flatten.join("\n")
+        end
+      end
+    end
+  end
+end
+
+AuroraDataApi::Tool.start

data/lib/aurora-data-api/data_service.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "aws-sdk-rdsdataservice"
+require_relative "environment"
+
+module AuroraDataApi
+  class DataService
+    def initialize
+      client_param = {region: Environment.region}
+      if Environment.offline?
+        offline_config_update
+        client_param[:endpoint] = Environment.offline_endpoint
+      end
+      @client = Aws::RDSDataService::Client.new(client_param)
+      @execute_params = {
+        database: Environment.database_name,
+        secret_arn: Environment.secret_arn,
+        resource_arn: Environment.resource_arn,
+        include_result_metadata: true
+      }
+    end
+
+    def execute(hash, without_database = false)
+      if without_database
+        @client.execute_statement(
+          hash.merge(@execute_params.reject { |k, _v| k == :database })
+        )
+      else
+        @client.execute_statement(hash.merge(@execute_params))
+      end
+    end
+
+    private def offline_config_update
+      Aws.config.update({
+        credentials: Aws::Credentials.new(
+          "AWS_ACCESS_KEY_dummy",
+          "AWS_SECRET_ACCESS_KEY_dummy"
+        )
+      })
+    end
+  end
+end

data/lib/aurora-data-api/depot.rb

@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+module AuroraDataApi
+  class Depot
+    def self.[](model, &block)
+      # @type var block: Proc
+      depot = new(model, block)
+      AuroraDataApi.const_set("#{model.name}Depot".to_sym, depot)
+      depot
+    end
+
+    def initialize(model, block)
+      @model = model
+      instance_eval { block.call }
+      @data_service = DataService.new
+    end
+
+    def table_name
+      @table_name ||= Model::SCHEMA[@model.name.to_sym][:table_name] || "#{@model.to_s.downcase}s"
+    end
+
+    def literal_id
+      @literal_id ||= Model::SCHEMA[@model.name.to_sym][:literal_id] || :id
+    end
+
+    def insert(obj)
+      obj.set_timestamp(at: :create)
+      params = obj.build_params
+      res = query(<<~SQL, **params)
+        INSERT INTO "#{obj.table_name}"
+          (#{params.keys.map { |k| "\"#{k}\"" }.join(",")})
+          VALUES
+          (#{params.keys.map { |k| ":#{k}" }.join(",")})
+          RETURNING "#{obj.literal_id}";
+      SQL
+      obj._set_id(res.records[0][0].value)
+    end
+
+    def update(obj)
+      obj.set_timestamp(at: :update)
+      params = obj.build_params
+      query(<<~SQL, **params)
+        UPDATE "#{obj.table_name}" SET
+          #{params.keys.reject { |k| k == obj.literal_id }.map { |k| "\"#{k}\" = :#{k}" }.join(", ")}
+          WHERE "#{obj.literal_id}" = :#{obj.literal_id};
+      SQL
+      true
+    end
+
+    def delete(obj)
+      query(<<~SQL, id: obj.id)
+        DELETE FROM "#{obj.table_name}" WHERE "#{obj.literal_id}" = :id;
+      SQL
+      obj._destroy
+      true
+    end
+
+    def count(str = "", **params)
+      query(
+        "SELECT COUNT(\"#{literal_id}\") FROM \"#{table_name}\" #{str};",
+        **params
+      ).records[0][0].long_value
+    end
+
+    def select(str, **params)
+      result = query("select * from \"#{table_name}\" #{str};", **params)
+      related_objects = {}
+      result.records.map do |record|
+        relationships = {}
+        attributes = {}.tap do |attrribute|
+          result.column_metadata.each_with_index do |meta, index|
+            if meta.table_name == table_name.to_s
+              attrribute[meta.name.to_sym] = column_data(meta, record[index])
+            else
+              table_sym = meta.table_name.to_sym
+              name, rel_model = @model.relationship_by(table_sym)
+              if name
+                relationships[name] ||= {}
+                relationships[name][:attr] ||= {}
+                relationships[name][:model] ||= rel_model
+                relationships[name][:attr][meta.name.to_sym] = column_data(meta, record[index])
+              end
+            end
+          end
+        end
+        relationships.each do |name, data|
+          related_objects[name] ||= {}
+          id = data[:attr][literal_id]
+          obj = related_objects.dig(name, id)
+          unless obj
+            obj = Kernel.const_get(data[:model]).new(**data[:attr])
+            related_objects[name][id] = obj
+          end
+          attributes[name] = obj
+        end
+        @model.new(attributes)
+      end
+    end
+
+    def column_data(meta, col)
+      return nil if col.is_null
+      case meta.type_name
+      when "text"
+        col.value.gsub("''", "'")
+      when "timestamptz"
+        Time.parse(col.value) + 9 * 60 * 60 # workaround
+      else
+        col.value
+      end
+    end
+
+    def query(str, **params)
+      @data_service.execute({
+        sql: str,
+        parameters: params.map do |param|
+          hash = {name: param[0].to_s, value: {}}
+          case param[1]
+          when Integer
+            hash[:value][:long_value] = param[1]
+          when Float
+            hash[:value][:double_value] = param[1]
+          when TrueClass, FalseClass
+            hash[:value][:boolean_value] = param[1]
+          else # TODO: confirm format and timezone when Time
+            hash[:value][:string_value] = param[1].to_s.gsub("'", "''")
+          end
+          hash
+        end
+      })
+    end
+  end
+end

data/lib/aurora-data-api/environment.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module AuroraDataApi
+  class Environment
+    # Resources about "OFFLINE":
+    #  https://www.serverless.com/plugins/serverless-offline
+    #  https://github.com/koxudaxi/local-data-api
+
+    def self.offline?
+      ENV["IS_OFFLINE"] == "true"
+    end
+
+    def self.offline_endpoint
+      "http://local-data-api"
+    end
+
+    def self.database_name
+      ENV["PGDATABASE"] || ENV["MYSQL_DATABASE"]
+    end
+
+    def self.region
+      ENV.fetch("AWS_DEFAULT_REGION", "ap-northeast-1")
+    end
+
+    def self.secret_arn
+      if offline?
+        "arn:aws:secretsmanager:us-east-1:123456789012:secret:dummy"
+      else
+        ENV["RDS_SECRET_ARN"]
+      end
+    end
+
+    def self.resource_arn
+      if offline?
+        "arn:aws:rds:us-east-1:123456789012:cluster:dummy"
+      else
+        ENV["RDS_RESOURCE_ARN"]
+      end
+    end
+  end
+end