table_saw 2.10.0 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2523b019080dccb116fed85c38ecb0a629cd6f8630abd65a1a30e2214c6a71b3
-  data.tar.gz: 03e53e17ef16f72e976772bc027ca4698ba8eaa7bf4498a2fae5ab7e7dfd3f7d
+  metadata.gz: 5e8e8f428d4b3d28ddec794d07a0d052e6280bfd96ee0b58735b3bcefa5a7bee
+  data.tar.gz: 64249bc2ae252e9b361c32b376f1679837ac1fd82bde70b11eda9fad66050a00
 SHA512:
-  metadata.gz: ab502f4b52e5e5214cca044d4ca450b9c885e63595423161e4ad130f85da6a5eb28fca008c77e49d70b675448ebe54a63bf6dfba465db6f47572b5d9b7fddbf3
-  data.tar.gz: 63a25f77f34251c801ba0f21aa5649988e763c1e8516dc4c1d3fc7976c2bd97c5bbc50d4fe6085b24032780ea7c93203d27c856c1304d681f6d2065a9a60f2f3
+  metadata.gz: 11b993bf2a9f723783b3f67648af15b794b807678db845b6c57b1ac8e5b57762ac41a3c1530696f5f3f99d6a7649061a22a88267aab23fb4e5cf158533f014e6
+  data.tar.gz: 84eb90d309a2dc4a8ddd04c4898c748f81f6c2f62223780b9e5d272913d2e990c840880e49e17dbe8832726b0eb6ea38f7f21970a44fd49ebd85d3b7e12b91e6

data/.github/workflows/ruby.yml

@@ -10,11 +10,12 @@ jobs:
     strategy:
       matrix:
         ruby:
-          - '2.7.2'
-          - '3.0.0'
+          - '3.0.5'
+          - '3.1.4'
+          - '3.2.2'
         activerecord:
-          - '6.0.0'
           - '6.1.0'
+          - '7.0.0'
 
     env:
       BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/activerecord_${{ matrix.activerecord }}.gemfile

data/.rubocop.yml

@@ -2,7 +2,7 @@ require: rubocop-rspec
 
 AllCops:
   NewCops: enable
-  TargetRubyVersion: 2.6
+  TargetRubyVersion: 3.0
   Exclude:
     - gemfiles/*
 
@@ -34,6 +34,9 @@ Style/EmptyMethod:
 Style/ExponentialNotation:
   Enabled: true
 
+Style/FetchEnvVar:
+  Enabled: false
+
 Style/FormatStringToken:
   EnforcedStyle: template
 

data/.tool-versions

@@ -1 +1 @@
-ruby 3.0.2
+ruby 3.2.2

data/Appraisals

@@ -1,9 +1,9 @@
 # frozen_string_literal: true
 
-appraise 'activerecord-6.0.0' do
-  gem 'activerecord', '~> 6.0', '< 6.1'
-end
-
 appraise 'activerecord-6.1.0' do
   gem 'activerecord', '~> 6.1', '< 6.2'
 end
+
+appraise 'activerecord-7.0.0' do
+  gem 'activerecord', '~> 7.0', '< 7.1'
+end

data/Gemfile

@@ -6,3 +6,12 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'appraisal'
+gem 'bundler', '~> 2.0'
+gem 'combustion', '~> 1.3'
+gem 'database_cleaner', '~> 2'
+gem 'pry'
+gem 'rake', '13.0.3'
+gem 'rspec', '~> 3.0'
+gem 'rubocop-rspec', '~> 2.3'
+gem 'scenic', '~> 1.5'
+gem 'simplecov', '~> 0.16'

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    table_saw (2.10.0)
+    table_saw (3.0.0)
       activerecord (>= 5.2)
       pg
       thor
@@ -9,131 +9,138 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actionpack (6.1.4.1)
-      actionview (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-      rack (~> 2.0, >= 2.0.9)
+    actionpack (7.0.5)
+      actionview (= 7.0.5)
+      activesupport (= 7.0.5)
+      rack (~> 2.0, >= 2.2.4)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actionview (6.1.4.1)
-      activesupport (= 6.1.4.1)
+    actionview (7.0.5)
+      activesupport (= 7.0.5)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activemodel (6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activerecord (6.1.4.1)
-      activemodel (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
-    activesupport (6.1.4.1)
+    activemodel (7.0.5)
+      activesupport (= 7.0.5)
+    activerecord (7.0.5)
+      activemodel (= 7.0.5)
+      activesupport (= 7.0.5)
+    activesupport (7.0.5)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
-    appraisal (2.4.0)
+    appraisal (2.4.1)
       bundler
       rake
       thor (>= 0.14.0)
     ast (2.4.2)
     builder (3.2.4)
     coderay (1.1.3)
-    combustion (1.3.1)
+    combustion (1.3.7)
       activesupport (>= 3.0.0)
       railties (>= 3.0.0)
       thor (>= 0.14.6)
-    concurrent-ruby (1.1.9)
+    concurrent-ruby (1.2.2)
     crass (1.0.6)
-    database_cleaner (2.0.1)
-      database_cleaner-active_record (~> 2.0.0)
-    database_cleaner-active_record (2.0.1)
+    database_cleaner (2.0.2)
+      database_cleaner-active_record (>= 2, < 3)
+    database_cleaner-active_record (2.1.0)
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
-    diff-lcs (1.4.4)
+    diff-lcs (1.5.0)
     docile (1.4.0)
-    erubi (1.10.0)
-    i18n (1.8.10)
+    erubi (1.12.0)
+    i18n (1.13.0)
       concurrent-ruby (~> 1.0)
-    loofah (2.12.0)
+    json (2.6.3)
+    loofah (2.21.3)
       crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
+      nokogiri (>= 1.12.0)
     method_source (1.0.0)
-    mini_portile2 (2.6.1)
-    minitest (5.14.4)
-    nokogiri (1.12.5)
-      mini_portile2 (~> 2.6.1)
+    mini_portile2 (2.8.2)
+    minitest (5.18.0)
+    nokogiri (1.15.2)
+      mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
-    parallel (1.20.1)
-    parser (3.0.1.1)
+    parallel (1.23.0)
+    parser (3.2.2.1)
       ast (~> 2.4.1)
-    pg (1.2.3)
-    pry (0.14.1)
+    pg (1.5.3)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    racc (1.5.2)
-    rack (2.2.3)
-    rack-test (1.1.0)
-      rack (>= 1.0, < 3)
+    racc (1.6.2)
+    rack (2.2.7)
+    rack-test (2.1.0)
+      rack (>= 1.3)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.2)
-      loofah (~> 2.3)
-    railties (6.1.4.1)
-      actionpack (= 6.1.4.1)
-      activesupport (= 6.1.4.1)
+    rails-html-sanitizer (1.5.0)
+      loofah (~> 2.19, >= 2.19.1)
+    railties (7.0.5)
+      actionpack (= 7.0.5)
+      activesupport (= 7.0.5)
       method_source
-      rake (>= 0.13)
+      rake (>= 12.2)
       thor (~> 1.0)
-    rainbow (3.0.0)
+      zeitwerk (~> 2.5)
+    rainbow (3.1.1)
     rake (13.0.3)
-    regexp_parser (2.1.1)
+    regexp_parser (2.8.0)
     rexml (3.2.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.14.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rubocop (1.51.0)
+      json (~> 2.3)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.2.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.5.0, < 2.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.28.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.5.0)
-      parser (>= 3.0.1.1)
-    rubocop-rspec (2.3.0)
-      rubocop (~> 1.0)
-      rubocop-ast (>= 1.1.0)
-    ruby-progressbar (1.11.0)
-    scenic (1.5.4)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.28.1)
+      parser (>= 3.2.1.0)
+    rubocop-capybara (2.18.0)
+      rubocop (~> 1.41)
+    rubocop-factory_bot (2.23.1)
+      rubocop (~> 1.33)
+    rubocop-rspec (2.22.0)
+      rubocop (~> 1.33)
+      rubocop-capybara (~> 2.17)
+      rubocop-factory_bot (~> 2.22)
+    ruby-progressbar (1.13.0)
+    scenic (1.7.0)
       activerecord (>= 4.0.0)
       railties (>= 4.0.0)
-    simplecov (0.21.2)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
-    simplecov_json_formatter (0.1.3)
-    thor (1.1.0)
-    tzinfo (2.0.4)
+    simplecov_json_formatter (0.1.4)
+    thor (1.2.2)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.0.0)
-    zeitwerk (2.4.2)
+    unicode-display_width (2.4.2)
+    zeitwerk (2.6.8)
 
 PLATFORMS
   ruby
@@ -152,4 +159,4 @@ DEPENDENCIES
   table_saw!
 
 BUNDLED WITH
-   2.2.22
+   2.4.13

data/README.md

@@ -2,7 +2,7 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/abd5b5451c764d3249f1/maintainability)](https://codeclimate.com/github/hasghari/table_saw/maintainability)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/abd5b5451c764d3249f1/test_coverage)](https://codeclimate.com/github/hasghari/table_saw/test_coverage)
 
-# TableSaw
+# table-saw
 
 This gem creates a PSQL dump file (data only) from a Postgres database by only dumping a subset of data defined by a 
 manifest file.
@@ -50,52 +50,188 @@ Options:
   -o, [--output=OUTPUT]       # Default value is 'output.dump'
 ```
 
-The manifest file describes which tables you want to dump from your Postgres database:
+### Manifest
+
+The manifest is a YAML file that describes the dataset to be exported to a dump file. At the top level, the manifest 
+file supports 4 nodes:
+
+- [variables](#variables)
+- [tables](#tables)
+- [has_many](#has_many)
+- [foreign_keys](#foreign_keys)
+
+The examples that follow assume you have a database set up as follows:
+```sql
+CREATE TABLE authors (
+    id bigint NOT NULL,
+    name character varying NOT NULL,
+    PRIMARY KEY (id)
+);
+
+CREATE TABLE books (
+    id bigint NOT NULL,
+    author_id bigint NOT NULL,
+    name character varying NOT NULL,
+    votes integer DEFAULT 0 NOT NULL,
+    PRIMARY KEY (id),
+    CONSTRAINT fk_author FOREIGN KEY (author_id) REFERENCES authors(id)
+);
+
+INSERT INTO authors (id, name) VALUES (1, 'Dan Brown');
+INSERT INTO authors (id, name) VALUES (2, 'J. K. Rowling');
+
+INSERT INTO books (id, author_id, name, votes) VALUES (1, 1, 'Angels and Demons', 10);
+INSERT INTO books (id, author_id, name, votes) VALUES (2, 1, 'Digital Fortress', 35);
+INSERT INTO books (id, author_id, name, votes) VALUES (3, 1, 'The Da Vinci Code', 50);
+INSERT INTO books (id, author_id, name, votes) VALUES (4, 2, 'Philosopher''s Stone', 55);
+INSERT INTO books (id, author_id, name, votes) VALUES (5, 2, 'Chamber of Secrets', 5);
+INSERT INTO books (id, author_id, name, votes) VALUES (6, 2, 'Prisoner of Azkaban', 25);
+```
+
+#### variables
+Variables allow you parameterize the queries in the manifest. You can use the `%{variable}` substitution pattern in your
+query strings:
 
 ```yaml
 variables:
-  author_id: 24
+  author_id: 2
 tables:
   - table: books
     query: "select * from books where author_id = %{author_id}"
 ```
 
-This will only fetch records from the `books` table where `author_id = 24` and will also fetch the record from the
-`authors` table where `id = 24`.
-
-Assuming there is a `chapters` table with a foreign key reference of `book_id` to the `books` table, the above manifest
-file will **not** automatically retrieve those records. If `chapters` records are also desired, there a couple ways to 
-accomplish this:
+Additionally, you can now use the `%{variable}` substitution pattern in other variables:
 
 ```yaml
 variables:
-  author_id: 24
+  author_id: '1,3,4',
+  book_ids: 'select * from books where author_id in (%{author_id})' 
+```
+
+#### tables
+This is where you list the specific tables that you want to export. If you only specify the `table` without providing a 
+`query`, then the **entire** table will be exported. However, if you specify a `query`, then only rows matching that 
+query will be exported:
+
+```yaml
 tables:
   - table: books
-    query: "select * from books where author_id = %{author_id}"
+    query: "select * from books where author_id = 2"
+```
+
+The above manifest will only export rows from the `books` table where `author_id = 2`. In addition, due to the 
+foreign key constraint defined from `books(author_id)` to `authors(id)`, table-saw will automatically export the row 
+from the `authors` table where `id = 2` in order to preserve referential integrity.
+
+The above manifest can alternatively be written as follows where exactly the same rows would be exported:
+
+```yaml
+tables:
+  - table: authors
+    query: "select * from authors where id = 2"
+    has_many:
+      - books
+```
+
+Notice we have to explicitly list the `has_many` association to the `books` table. Since `authors` does not have a
+dependency on `books` to preserve referential integrity, table-saw by design will not export the associated `books`
+rows in order to keep the output dump file as small as possible. In other words, if we eliminate the `has_many` node
+from the manifest above, table-saw will only export a single author with `id = 2`.
+
+#### has_many
+This is where we define which optional associations we want to export for each table:
+
+```yaml
+tables:
+  - table: authors
+    query: "select * from authors where id = 1"
+  - table: books
+    query: "select * from books where id = 6"
 has_many:
-  books:
-    - chapters
+  authors:
+    - books
 ```
 
-or
+The above manifest would export the following rows:
+
+* authors:
+  - `id = 1` because of the explicit query in the manifest
+  - `id = 2` because it is the `author_id` for `books` with `id = 6`
+
+* books:
+  - `id = [1, 2, 3]` due to the `has_many` association for `authors` with `id = 1`
+  - `id = [4, 5, 6]` due to the `has_many` association for `authors` with `id = 2`
+
+Now, if instead of defining the `has_many` node at the top level, we define it under the `authors` table as follows:
 
 ```yaml
-variables:
-  author_id: 24
 tables:
-  - table: chapters
-    query: "select * from chapters inner join books on books.id = chapters.book_id where books.author_id = %{author_id}"
+  - table: authors
+    query: "select * from authors where id = 1"
+    has_many:
+      - books
+  - table: books
+    query: "select * from books where id = 6"
+```
+
+The above manifest would export the following rows:
+
+* authors:
+  - `id = 1` because of the explicit query in the manifest
+  - `id = 2` because it is the `author_id` for `books` with `id = 6`
+
+* books:
+  - `id = [1, 2, 3]` due to the `has_many` association for `authors` with `id = 1`
+  - `id = 6` due to the explicit query in the manifest
+
+One potential pitfall with using `has_many` is that you end up pulling in too many associated rows when all you wanted 
+was a limited number. table-saw allows you to specify a `scope` and `limit`:
+
+```yaml
+tables:
+  - table: authors
+    query: "select * from authors where id = 1"
+  - table: books
+    query: "select * from books where id = 6"
+has_many:
+  authors:
+    - books:
+        scope: "votes > 30"
+        limit: 1
+```
+
+The above manifest would export the following rows:
+
+* authors:
+  - `id = 1` because of the explicit query in the manifest
+  - `id = 2` because it is the `author_id` for `books` with `id = 6`
+
+* books:
+  - Either `id = 2` or `id = 3` for `author_id = 1` since they both have `vote > 30` and the limit of 1 will randomly 
+    choose one of them
+  - Only `id = 4` for `author_id = 2` since it's the only book for that author with `vote > 30`
+
+#### foreign_keys
+By default, table-saw will query the Postgres `information_schema` to look up the foreign key constraints and determine
+whether it needs to export associated rows. However, if your database schema does not define foreign key constraints for
+the tables you would like to export, you can manually define them in the manifest. Assuming we had not defined any
+foreign key constraints for the `books` table, we could specify it in the manifest as follows:
+
+```yaml
+foreign_keys:
+  - from_table: books
+    from_column: author_id
+    to_table: authors
+    to_column: id
 ```
 
-The output of the 2 manifest files above are exactly the same. The dump file will contain all the relevant records from
-the `authors`, `books` and `chapters` tables.
+## Dump and Restore
 
 Once your dump file has been created, you can import the data using `psql`:
 
 ```bash
-table-saw dump -m manifest.yml -o chapters.dump
-psql -h localhost -U postgres -d library < chapters.dump
+table-saw dump -m manifest.yml -o library.dump
+psql -h localhost -U postgres -d library < library.dump
 ```
 
 ## Development

data/gemfiles/activerecord_6.1.0.gemfile

@@ -3,6 +3,15 @@
 source "https://rubygems.org"
 
 gem "appraisal"
+gem "bundler", "~> 2.0"
+gem "combustion", "~> 1.3"
+gem "database_cleaner", "~> 2"
+gem "pry"
+gem "rake", "13.0.3"
+gem "rspec", "~> 3.0"
+gem "rubocop-rspec", "~> 2.3"
+gem "scenic", "~> 1.5"
+gem "simplecov", "~> 0.16"
 gem "activerecord", "~> 6.1", "< 6.2"
 
 gemspec path: "../"