pgdice 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 63a62f8196d2f0299a6e181001c2f1c34020683360d01347a24288a9da0bc16a
+  data.tar.gz: 2a6edd7716f95f215a222b9d47bfdae2897852c70cfff516767ab4669c167872
+SHA512:
+  metadata.gz: 190c116e139dce2e6e009703be2b38eea7a7d51efb0ebf01266abf63005e1122d55569b49d65a56dc5ce1173392e2fe24fa5cc77c93355a1497f9cf169ac1d1e
+  data.tar.gz: d81d5625a0b145b259507067c7214b3003937523965444f32631968049a3ddb686f4de6b1b711466ff76596ccbefb7dff1466d2fcf47e563e4185a170dee822d

data/.circleci/config.yml

@@ -0,0 +1,66 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+defaults: &defaults
+  working_directory: ~/repo
+  docker:
+  - image: circleci/ruby:2.5.1
+    environment:
+      DATABASE_HOST: 127.0.0.1
+      DATABASE_USERNAME: pgdice
+      PGDICE_LOG_TARGET: STDOUT
+
+  - image: circleci/postgres:10.4-alpine-ram
+    environment:
+      POSTGRES_USER: pgdice
+      POSTGRES_DB: pgdice_test
+
+jobs:
+  test:
+    <<: *defaults
+    environment:
+    - CC_TEST_REPORTER_ID: b39c211d102df8869887e0f5764fbb06114fd2430cdf6689868dec7a1261fd05
+    steps:
+    - checkout
+    - run:
+        name:  Download cc-test-reporter
+        command: |
+          mkdir -p tmp/
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./tmp/cc-test-reporter
+          chmod +x ./tmp/cc-test-reporter
+
+    - run:
+        name: Setup dependencies
+        command: |
+          bundle install --jobs 4 --retry 3
+
+    - run:
+        name: Run tests
+        command: |
+          mkdir -p /tmp/test-results
+          TEST_FILES="$(circleci tests glob "test/**/*_test.rb" | circleci tests split --split-by=timings)"
+
+          bundle exec rake test
+          ./tmp/cc-test-reporter format-coverage -t simplecov -o tmp/coverage/codeclimate.pgdice.json tmp/coverage/pgdice/.resultset.json
+
+    - store_test_results:
+       path: ~/repo/tmp/test-results
+
+    - store_artifacts:
+       path: ~/repo/tmp/test-results
+       destination: test-results
+
+    - run:
+        name: Upload coverage results to Code Climate
+        command: |
+          ./tmp/cc-test-reporter sum-coverage tmp/coverage/codeclimate.*.json -p 1 -o tmp/coverage/codeclimate.total.json
+          ./tmp/cc-test-reporter upload-coverage -i tmp/coverage/codeclimate.total.json
+
+
+workflows:
+  version: 2
+  commit:
+    jobs:
+    - test

data/.coveralls.yml

@@ -0,0 +1 @@
+repo_token: GfF90mydWLT1YZj5hFzFy9l2xyVNjMkBV

data/.gitignore

@@ -0,0 +1,13 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.idea
+Gemfile.lock
+*.log
+!tmp/.keep
+tmp/

data/.rubocop.yml

@@ -0,0 +1,6 @@
+Metrics/LineLength:
+  Max: 120
+
+Metrics/BlockLength:
+  Exclude:
+    - pgdice.gemspec

data/.ruby-gemset

@@ -0,0 +1 @@
+pgdice

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.5.1

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,74 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at scytherswings@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in pgdice.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+## Uncomment and set this to only include directories you want to watch
+# directories %w(app lib config test spec features) \
+#  .select{|d| Dir.exists?(d) ? d : UI.warning("Directory #{d} does not exist")}
+
+## Note: if you are using the `directories` clause above and you are not
+## watching the project directory ('.'), then you will want to move
+## the Guardfile to a watched dir and symlink it back, e.g.
+#
+#  $ mkdir config
+#  $ mv Guardfile config/
+#  $ ln -s config/Guardfile .
+#
+# and, you'll have to watch "config/Guardfile" instead of "Guardfile"
+
+guard :minitest, all_after_pass: true do
+  watch(%r{^lib/(.+)\.rb$}) { |m| "test/#{m[1]}_test.rb" }
+  watch(%r{^test/test_helper\.rb$}) { 'test' }
+  watch(%r{^test/.+_test\.rb$})
+end
+
+guard :rubocop, cli: %w[-D -S -a] do
+  watch(/.rubocop.yml/)
+  watch(/.+\.rb$/)
+  watch(/Rakefile/)
+  watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
+end

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Illuminus Limited Liability Company
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,262 @@
+[![CircleCI](https://circleci.com/gh/IlluminusLimited/pgdice.svg?style=shield)](https://circleci.com/gh/IlluminusLimited/pgdice)
+[![Coverage Status](https://coveralls.io/repos/github/IlluminusLimited/pgdice/badge.svg?branch=master)](https://coveralls.io/github/IlluminusLimited/pgdice?branch=master)
+[![Maintainability](https://api.codeclimate.com/v1/badges/311e005a14749bf2f826/maintainability)](https://codeclimate.com/github/IlluminusLimited/pgdice/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/311e005a14749bf2f826/test_coverage)](https://codeclimate.com/github/IlluminusLimited/pgdice/test_coverage)
+
+# PgDice
+
+PgDice is a utility that builds on top of the excellent gem
+ [https://github.com/ankane/pgslice](https://github.com/ankane/pgslice)
+ 
+PgDice is intended to be used by scheduled background jobs in frameworks like [Sidekiq](https://github.com/mperham/sidekiq)
+where logging and clear exception messages are crucial.
+
+
+## Disclaimer
+
+There are some features in this gem which allow you to drop database tables. 
+
+If you choose to use this software without a __tested and working__ backup and restore strategy in place then you 
+are a fool and will pay the price for your negligence. This software comes with no warranty 
+or any guarantees, implied or otherwise. By using this software you agree that the creator, 
+maintainers and any affiliated parties CANNOT BE HELD LIABLE FOR DATA LOSS OR LOSSES OF ANY KIND.
+
+See the [LICENSE](LICENSE) for more information.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'pgdice'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install pgdice
+
+## Usage
+
+### Configuration
+
+You must configure `PgDice` before you can use it, otherwise you won't be able to perform any manipulation actions
+on tables.
+
+This is an example config from a project using `Sidekiq` 
+```ruby
+require 'pgdice'
+PgDice.configure do |config|
+  config.logger = Sidekiq.logger # This defaults to STDOUT if you don't specify a logger
+  config.database_url = ENV['PGDICE_DATABASE_URL'] # postgresql://[user[:password]@][host][:port][/dbname][?param1=value1&...]
+  config.approved_tables = ENV['PGDICE_APPROVED_TABLES'] # Comma separated values: 'comments,posts'
+end
+```
+
+
+#### Configuration Parameters
+
+`logger` Optional: The logger to use. If you don't set this it defaults to STDOUT.
+
+`database_url` The postgres database url to connect to. This is required since `pgslice` is used to accomplish some tasks
+and it only takes a `url` currently.
+
+`approved_tables` This one is important. If you want to manipulate database tables with this gem you're going to
+need to add the base table name to this string of comma-separated values.
+
+`additional_validators` Optional: This can accept an array of `proc` or `lambda` type predicates. 
+Each predicate will be passed the `params` hash and a `logger`. These predicates are called before doing things like
+dropping tables and adding tables. 
+
+`dry_run` Optional: You can set it to either `true` or `false`. This will make PgDice print the commands but not 
+execute them.
+
+`older_than` Optional: Time object used to scope the queries on droppable tables. Defaults to 90 days ago.
+
+`table_drop_batch_size` Optional: Maximum number of tables you can drop in one query. Defaults to 7.
+
+
+#### Advanced Configuration Parameters
+
+`table_dropper` This defaults to [TableDropper](lib/pgdice/table_dropper.rb) which has a `lambda`-like interface. 
+An example use-case would be calling out to your backup system to confirm the table is backed up.
+This mechanism will be passed the `table_to_drop` and a `logger`.
+
+`pg_connection` This is a `PG::Connection` object used for the database queries made from `pgdice`.
+ By default it will be initialized from the `database_url` if left `nil`. Keep in mind the dependency 
+ `pgslice` will still establish its own connection using the `database_url` so this feature may not be very
+ useful if you are trying to only use one connection for this utility.
+ 
+ `database_connection` You can supply your own [DatabaseConnection](lib/pgdice/database_connection.rb) if you like.
+ I'm not sure why you would do this.
+ 
+ `pg_slice_manager` This is an internal wrapper around `pgslice`. [PgSliceManager](lib/pgdice/pg_slice_manager.rb)
+  This configuration lets you provide your own if you wish. I'm not sure why you would do this.
+ 
+ `partition_manager` You can supply your own [PartitionManager](lib/pgdice/partition_manager.rb) if you like.
+  I'm not sure why you would do this.
+  
+ `partition_helper` You can supply your own [PartitionHelper](lib/pgdice/partition_helper.rb) if you like.
+  I'm not sure why you would do this.
+ 
+ 
+### Converting existing tables to partitioned tables
+
+__This should only be used on smallish tables and ONLY after you have tested it on a non-production copy of your 
+production database.__
+In fact, you should just not do this in production. Schedule downtime or something and run it a few times on
+a copy of your database. Then practice restoring your database some more.
+
+
+This command will convert an existing table into 61 partitioned tables (30 past, 30 future, and one for today).
+
+For more information on what's going on in the background see 
+[https://github.com/ankane/pgslice](https://github.com/ankane/pgslice)
+
+
+```ruby
+PgDice.partition_helper.partition_table!(table_name: 'comments', 
+                                            past: 30, 
+                                            future: 30, 
+                                            column_name: 'created_at', 
+                                            period: :day)
+```
+
+If you mess up (again you shouldn't use this in production). These two methods are useful for writing tests
+that work with partitions.
+
+```ruby
+PgDice.partition_helper.undo_partitioning!(table_name: 'comments')
+```
+
+In `partition_helper` there are versions of the methods that will throw exceptions (ending in `!`) and others 
+that will return a truthy value or `false` if there is a failure.
+
+`period` can be set to one of these values: `:day`, `:month`, `:year`
+
+
+### Maintaining partitioned tables
+
+#### Adding more tables
+
+If you have existing tables that need to periodically have more tables added you can run:
+
+```ruby
+PgDice.partition_manager.add_new_partitions(table_name: 'comments', future: 30)
+```
+
+The above command would add 30 new tables and their associated indexes all based on the `period` that the
+partitioned table was defined with.
+
+
+#### Listing old tables
+
+Sometimes you just want to know what's out there and if there are tables ready to be dropped.
+
+To list all eligible tables for dropping you can run:
+```ruby
+PgDice.partition_manager.list_old_partitions(table_name: 'comments', older_than: Time.now.utc - 90*24*60*60)
+```
+
+If you have `active_support` you could do:
+```ruby
+PgDice.partition_manager.list_old_partitions(table_name: 'comments', older_than: 90.days.ago)
+```
+
+Technically `older_than` is optional and defaults to `90 days` (see the configuration section).
+It is recommended that you pass it in to be explicit, but you can rely on the configuration 
+mechanism if you so choose.
+
+
+#### Dropping old tables
+
+_Dropping tables is irreversible! Do this at your own risk!!_
+
+If you want to drop old tables (after backing them up of course) you can run:
+
+```ruby
+PgDice.partition_manager.drop_old_partitions(table_name: 'comments', older_than: Time.now.utc - 90*24*60*60)
+```
+
+If you have `active_support` you could do:
+```ruby
+PgDice.partition_manager.drop_old_partitions(table_name: 'comments', older_than: 90.days.ago)
+```
+
+This command would drop old partitions that are older than `90` days.
+
+Technically `older_than` is optional and defaults to `90 days` (see the configuration section).
+It is recommended that you pass it in to be explicit, but you can rely on the configuration 
+mechanism if you so choose.
+
+Another good reason to pass in the `older_than` parameter is if you are managing tables that
+are partiioned by different schemes or have different use-cases 
+e.g. daily vs yearly partitioned tables.
+
+
+#### Validating everything is still working
+
+If you've got background jobs creating and dropping tables you're going to want to 
+ensure they are actually working correctly. 
+
+To validate that your expected number of tables exist, you can run:
+```ruby
+PgDice.validation.assert_tables(table_name: 'comments', future: 30, past: 90)
+```
+
+An [InsufficientTablesError](lib/pgdice.rb) will be raised if any conditions are not met.
+
+This will check that the table 30 days from now exists and that there is 
+still a table from 90 days ago. The above example assumes the table was partitioned
+by day.
+
+
+## Planned Features
+
+1. Full `PG::Connection` support (no more database URLs).
+1. Custom schema support for all operations. Defaults to `public` currently.
+1. Non time-range based partitioning.
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. 
+You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the 
+version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version,
+ push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+
+### Running tests
+
+You're going to need to have postgres 10 or greater installed.
+
+Run the following commands from your terminal. Don't run these on anything but a development machine.
+
+1. `psql postgres -c "create role pgdice with createdb superuser login password 'password';"`
+1. `createdb pgdice_test`
+1. Now you can run the tests via `guard` or `rake test`
+
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at 
+[https://github.com/IlluminusLimited/pgdice](https://github.com/IlluminusLimited/pgdice). This project is intended 
+to be a safe, welcoming space for collaboration, and contributors are expected to adhere to
+ the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+
+## Code of Conduct
+
+Everyone interacting in the Pgdice project’s codebases, issue trackers, chat rooms and mailing lists is expected 
+to follow the [code of conduct](https://github.com/IlluminusLimited/pgdice/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task default: :test

data/bin/_guard-core

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application '_guard-core' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile',
+                                           Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path('bundle', __dir__)
+
+if File.file?(bundle_binstub)
+  if /This file was generated by Bundler/.match?(File.read(bundle_binstub, 300))
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('guard', '_guard-core')

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'pgdice'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require 'irb'
+IRB.start(__FILE__)

data/bin/guard

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'guard' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile',
+                                           Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path('bundle', __dir__)
+
+if File.file?(bundle_binstub)
+  if /This file was generated by Bundler/.match?(File.read(bundle_binstub, 300))
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require 'rubygems'
+require 'bundler/setup'
+
+load Gem.bin_path('guard', 'guard')

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here