pgdice 0.4.0 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 12fa7e75440961dbabdca09146b652c0b849f4d0e92f7c6e66c5e95d932f7449
-  data.tar.gz: 6c9df08948a598ae17ddfe76ceb9c594c5566a479dce26bf2dd698f9eb54969d
+  metadata.gz: ad12a02674afc68542e4b8038aaddcfb107eba52bc208f700ef57e880783d100
+  data.tar.gz: 6d384e71518d19b9e8319d6d84632ded6b3de0bcdd4fb6fa2cef80d837fc100d
 SHA512:
-  metadata.gz: a54b4dcdc2e9bed2798f5ccee31c1189036b0b37123e23b29c949d53bfbfe3fe7af1f288d4077ff66f3ce04374c113a91746e583bca3404aa221408afa05ceda
-  data.tar.gz: a418c411de0d82e221a1c7de358b1eb9dfd18a0bafd4207c84441852e87adfa1eb674063a194b50b1472e783f21842635dbc2c381958d1d0fc792a788c0914d3
+  metadata.gz: 8880202263ce06487635f87118d82b723405f465a5d39f087310961e965a553b4510088bc11f23d12fb37b76faf1bc0e8d5a706b26d28ad2afcb10111cbf07c8
+  data.tar.gz: a3c14f23d6b52a991f7997a9d887877e58b93e8c0b8717e65ea42ade699eae5fbfd1642eea788ddcc8eefe9d9dd01a12cff540772e1d2f69281119fcb6a73cad

data/.circleci/config.yml

@@ -6,13 +6,13 @@ version: 2
 defaults: &defaults
   working_directory: ~/repo
   docker:
-  - image: circleci/ruby:2.5.1
+  - image: circleci/ruby:2.6.6
     environment:
       DATABASE_HOST: 127.0.0.1
       DATABASE_USERNAME: pgdice
       PGDICE_LOG_TARGET: STDOUT
 
-  - image: circleci/postgres:10.6-alpine-ram
+  - image: circleci/postgres:12.1-alpine-ram
     environment:
       POSTGRES_USER: pgdice
       POSTGRES_DB: pgdice_test

data/.codeclimate.yml

@@ -2,4 +2,7 @@ version: "2"
 checks:
   argument-count:
     config:
-      threshold: 5
+      threshold: 5
+
+exclude_patterns:
+  - "examples/**/*"

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: needs investigation
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Please include this information**
+ - Postgres Version: [e.g. 10.6]
+ - PgDice Version [e.g. 0.14.3]
+
+**Additional context**
+Add any other context about the problem here.

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: needs investigation
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

data/.github/workflows/gempush.yml

@@ -0,0 +1,41 @@
+name: Ruby Gem
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        version: 2.6.x
+
+#     - name: Publish to GPR
+#       run: |
+#         mkdir -p $HOME/.gem
+#         touch $HOME/.gem/credentials
+#         chmod 0600 $HOME/.gem/credentials
+#         printf -- "---\n:github: Bearer ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+#         gem build *.gemspec
+#         gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+#       env:
+#         GEM_HOST_API_KEY: ${{secrets.GPR_AUTH_TOKEN}}
+#         OWNER: username
+
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: ${{secrets.RUBYGEMS_AUTH_TOKEN}}

data/.rubocop.yml

@@ -8,4 +8,8 @@ Metrics/BlockLength:
 # TODO: Evaluate if this is an acceptable parameter list. It's a constructor after all.
 Metrics/ParameterLists:
   Exclude:
-    - lib/pgdice/table.rb
+    - lib/pgdice/table.rb
+
+AllCops:
+  Exclude:
+    - 'examples/**/*'

data/.ruby-version

@@ -1 +1 @@
-ruby-2.5.1
+ruby-2.6.6

data/CHANGELOG.md

@@ -2,6 +2,27 @@
 All notable changes to this project will be documented in this file.
 This project adheres to [Semantic Versioning](http://semver.org/).
 
+## [v1.0.1] : 2020-01-31
+### Changes
+  - Added better examples/docs.
+  - Bump `pg` to `~> 1.2.2`
+
+## [v0.4.3] : 2019-04-23
+### Changes
+  - Fix #31 where the new connection retry code was eagerly initializing the logger
+
+
+## [v0.4.2] : 2019-04-22
+### Changes
+  - Fix #19 where PgDice wouldn't recover from a broken PG::Connection 
+  by adding new retry behavior
+
+
+## [v0.4.1] : 2019-03-21
+### Changes
+  - Fix bug where partitioning by months would break when the month was < 10
+
+
 ## [v0.4.0] : 2018-12-06
 ### Changes
   - Add `only:` option to `assert_tables` so users can assert on only `past`

data/README.md

@@ -12,18 +12,9 @@ PgDice is a utility for creating and maintaining partitioned database tables tha
 PgDice is intended to be used by scheduled background jobs in frameworks like [Sidekiq](https://github.com/mperham/sidekiq)
 where logging and clear exception messages are crucial.
 
+# Maintenance status
 
-## Disclaimer
-
-There are some features in this gem which allow you to drop database tables. 
-
-If you choose to use this software without a __tested and working__ backup and restore strategy in place then you 
-are a fool and will pay the price for your negligence. THIS SOFTWARE IS PROVIDED "AS IS",
-WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. By using this software you agree that the creator, 
-maintainers and any affiliated parties CANNOT BE HELD LIABLE FOR DATA LOSS OR LOSSES OF ANY KIND.
-
-See the [LICENSE](LICENSE) for more information.
-
+This project is stable and used daily in production.
 
 # Installation
 
@@ -61,6 +52,7 @@ PgDice.configure do |config|
  
   # Set a config file or build the tables manually
   config.config_file = Rails.root.join('config', 'pgdice.yml') # If you are using rails, else provide the absolute path.
+  config.config_file = Rails.root.join('config', 'pgdice.yml') # If you are using rails, else provide the absolute path.
   # and/or
   config.approved_tables = PgDice::ApprovedTables.new(
     PgDice::Table.new(table_name: 'comments', past: 90, future: 7, period: 'day'),
@@ -69,7 +61,6 @@ PgDice.configure do |config|
 end
 ```
 
-
 ### Configuration Parameters
 
 - `database_url` - **Required**: The postgres database url to connect to. 
@@ -179,7 +170,6 @@ PgDice.undo_partitioning!('comments')
 This method will revert the changes made by partitioning a table. Don't rely on this 
 in production if you mess up; you need to test everything thoroughly.
 
-
 ## Maintaining partitioned tables
 
 ### Adding more tables
@@ -196,7 +186,6 @@ PgDice.add_new_partitions('comments')
 that the partitioned table was defined with.
   - The example `comments` table we have been using was configured to always keep `7` future partitions above.
 
-
 ### Listing droppable partitions
 
 Sometimes you just want to know what's out there and if there are tables ready to be dropped.
@@ -213,7 +202,6 @@ PgDice.list_droppable_partitions_by_batch_size('comments')
 
 This method will show partitions that are within the configured `batch_size`.
 
-
 #### Notes on `list_droppable_partitions`
 
 - This method uses the `past` value from the `PgDice::Table` to determine which tables are eligible for dropping.
@@ -235,7 +223,6 @@ PgDice.drop_old_partitions('comments')
 for the `PgDice::Table`. 
   - The example `comments` table has been configured with `past: 90` tables. 
   So if there were 100 tables older than `today` it would drop up to `batch_size` tables.
-  
 
 # Validation
 
@@ -247,7 +234,7 @@ To validate that your expected number of tables exist, you can run:
 PgDice.assert_tables('comments')
 ```
 
-An [InsufficientTablesError](lib/pgdice.rb) will be raised if any conditions are not met.
+An [InsufficientTablesError](lib/pgdice/error.rb) will be raised if any conditions are not met.
 
 This will check that there are 7 future tables from now and that there are 90 past tables
 per our configuration above.
@@ -268,7 +255,6 @@ PgDice.approved_tables
 
 The [ApprovedTables](lib/pgdice/approved_tables.rb) object responds to the most common enumerable methods.
 
-
 # Miscellaneous Notes
 
 All methods for `PgDice` take a hash which will override whatever values would have been automatically supplied.
@@ -279,6 +265,12 @@ PgDice.list_droppable_partitions('comments', past: 60)
 ```
 This example would use `60` instead of the configured value of `90` from the `comments` table we configured above.
 
+# Examples
+
+1. [Here's an example on how to use PgDice in AWS](examples/aws) and the [README](examples/aws/README.md) which will guide
+you through what is going on.
+
+1. [Here's an example on how to write a config.yml for PgDice](examples/config.yml)
 
 # FAQ
 
@@ -292,7 +284,7 @@ This example would use `60` instead of the configured value of `90` from the `co
   password = config[Rails.env]["password"]
 
   "postgres://#{username}:#{password}@#{host}/#{database}"
-end
+ end
 ```
 
 1. I'm seeing off-by-one errors for my `assert_tables` calls?
@@ -303,7 +295,7 @@ end
 
 1. Full `PG::Connection` support (no more database URLs).
 1. Non time-range based partitioning. [PgParty](https://github.com/rkrage/pg_party) might be a good option!
-
+1. Hourly partitioning
 
 # Development
 
@@ -312,7 +304,6 @@ You can also run `bin/console` for an interactive prompt that will allow you to
 
 To install this gem onto your local machine, run `bundle exec rake install`.
 
-
 ## Running tests
 
 You're going to need to have postgres 10 or greater installed.
@@ -323,7 +314,6 @@ Run the following commands from your terminal. Don't run these on anything but a
 1. `createdb pgdice_test`
 1. Now you can run the tests via `guard` or `rake test`
 
-
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at 
@@ -331,11 +321,20 @@ Bug reports and pull requests are welcome on GitHub at
 to be a safe, welcoming space for collaboration, and contributors are expected to adhere to
  the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
 
-
 # License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
 
+# Disclaimer
+
+There are some features in this gem which allow you to drop database tables, due to the dangerous nature of 
+dropping database tables, please ensure you have a tested and working backup and restore strategy. 
+
+THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED. 
+By using this software you agree that the creator, maintainers, and any affiliated parties 
+CANNOT BE HELD LIABLE FOR DATA LOSS OR LOSSES OF ANY KIND.
+
+See the [LICENSE](LICENSE) for more information.
 
 # Code of Conduct
 

data/SECURITY.md

@@ -0,0 +1,15 @@
+# Security Policy
+
+## Supported Versions
+
+The versions here are currently supported for security patches.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| latest  | :white_check_mark: |
+| < 1.0   | :x:                |
+
+
+## Reporting a Vulnerability
+
+Please open a new issue

data/examples/aws/README.md

@@ -0,0 +1,28 @@
+# How can I use PgDice in production?
+
+This collection of files is how I use PgDice in production. I'll describe the architecture here so you'll have a place
+to start.
+
+1. `tasks/poll_sqs.rake` is run using some sort of process manager like systemd on the ec2 instance. I like to run
+the poll_sqs stuff on my Sidekiq instances because they are the ones who eventually handle the work anyway.
+
+1. `lib/sqs_poller.rb` is used to handle the looping logic for the rake task. It invokes `lib/sqs_listener.rb` for each 
+iteration.
+
+1. `lib/sqs_listener.rb` calls AWS SQS to receive messages and then passes each one into the `lib/sqs_listener/sqs_event_router.rb` 
+to be routed to the correct message handler.
+
+1. Inside `lib/sqs_listener/sqs_event_router.rb` the message is parsed and passed through a case statement. 
+This could be abstracted better but for now if the message has a field of `event_type` and a value of `"task"` then
+the router will send it off to the `TaskEventHandler` which in this case is 
+`lib/sqs_listener/typed_event_handler/task_event_handler.rb`
+
+1. In the `TaskEventHandler` the task is sent to a handler which responds to the task specified in the message body field `task`.
+
+1. The handler for the task (in this case, `DatabaseTasks`) handles the parameters for invoking the Sidekiq worker: `PgDiceWorker`
+
+1. Finally, the `PgDiceWorker` is called and handles invoking `PgDice` based on the parameters passed in.
+
+
+Hopefully that wasn't too confusing. There's a lot of steps in here because the system that uses PgDice handles lots
+of different types of SQS events and needs to be as resilient as possible.

data/examples/aws/cloudformation/scheduled_events.json

@@ -0,0 +1,59 @@
+{
+  "Description": "Deployment stack",
+  "Parameters": {
+    "PgDiceEnabled": {
+      "Type": "String",
+      "Description": "The ENABLED/DISABLED state of the cloudwatch scheduled events for PgDice."
+    }
+  },
+  "Resources": {
+    "PgDiceDailyAddPartitions": {
+      "DependsOn": "IncomingSQS",
+      "Type": "AWS::Events::Rule",
+      "Properties": {
+        "State":{
+          "Ref": "PgDiceEnabled"
+        },
+        "Description": " PgDice daily add partitions",
+        "Name": "PgDiceDailyAddPartitions",
+        "ScheduleExpression": "rate(1 day)",
+        "Targets": [
+          {
+            "Arn": {
+              "Fn::GetAtt": [
+                "IncomingSQS",
+                "Arn"
+              ]
+            },
+            "Id": "PgDiceDailyAddPartitionsId",
+            "Input": "{\"event_type\":\"task\",\"task\":\"add_new_partitions\"}"
+          }
+        ]
+      }
+    },
+    "PgDiceDailyDropPartitions": {
+      "DependsOn": "IncomingSQS",
+      "Type": "AWS::Events::Rule",
+      "Properties": {
+        "State":{
+          "Ref": "PgDiceEnabled"
+        },
+        "Description": " PgDice daily drop partitions",
+        "Name": "PgDiceDailyDropPartitions",
+        "ScheduleExpression": "rate(1 day)",
+        "Targets": [
+          {
+            "Arn": {
+              "Fn::GetAtt": [
+                "IncomingSQS",
+                "Arn"
+              ]
+            },
+            "Id": "PgDiceDailyDropPartitionsId",
+            "Input": "{\"event_type\":\"task\",\"task\":\"drop_old_partitions\"}"
+          }
+        ]
+      }
+    }
+  }
+}

data/examples/aws/lib/sqs_listener.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'aws-sdk-sqs'
+
+# READ_ONLY_SQS can be set to ensure we don't delete good messages
+class SqsListener
+  DEFAULT_VISIBILITY_TIMEOUT ||= 600
+  attr_reader :logger, :queue_url, :visibility_timeout
+
+  def initialize(opts = {})
+    @logger = opts[:logger] ||= Sidekiq.logger
+    @queue_url = opts[:queue_url] ||= ENV['SqsQueueUrl']
+    @sqs_client = opts[:sqs_client] ||= Aws::SQS::Client.new
+    @sqs_event_router = opts[:sqs_event_router] ||= SqsEventRouter.new(logger: logger)
+    increase_timeout_resolver = opts[:increase_timeout_resolver] ||= -> { ENV['READ_ONLY_SQS'].to_s == 'true' }
+    @visibility_timeout = calculate_visibility_timeout(increase_timeout_resolver.call)
+
+    logger.debug { "Running in environment: #{ENV['RAILS_ENV']} and using sqs queue: #{queue_url}" }
+  end
+
+  # http://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/sqs-example-get-messages-with-long-polling.html
+  def call
+    # This uses long polling to retrieve sqs events so we can process them
+    response = @sqs_client.receive_message(queue_url: queue_url,
+                                           max_number_of_messages: 10,
+                                           wait_time_seconds: 20,
+                                           visibility_timeout: visibility_timeout)
+
+    if response.messages&.size&.positive?
+      logger.debug { "The number of messages received from the queue was: #{response.messages&.size}" }
+    end
+
+    # Iterate over all the messages in the response (Response is a Struct which acts like an object with methods)
+    response.messages&.each do |message|
+      @sqs_event_router.handle_message(message)
+    end
+  end
+
+  private
+
+  def calculate_visibility_timeout(increase_timeout)
+    visibility_timeout = increase_timeout ? DEFAULT_VISIBILITY_TIMEOUT * 4 : DEFAULT_VISIBILITY_TIMEOUT
+
+    logger.info { "Visibility timeout set to: #{visibility_timeout} seconds" }
+    visibility_timeout
+  end
+end