neon_postgres 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19cb8f89bec9249fa25eb7d90c9050d8a31b10682742782b2aac1e609bd35eb4
-  data.tar.gz: e65ea4e35f4f24ef679a3097bf587cd0975ebc64b055a6c4be69d34c7109c80b
+  metadata.gz: 6d2ad55de5e5434affb51e984491c49d8f55dbe05c3088589e0771e5e7ebfef9
+  data.tar.gz: 99390ca36ff8bb5246e4257d5c367853d8432a2da233dca90746b5b9cbf2a8ba
 SHA512:
-  metadata.gz: 683a99ffe0a33f88e3eddc0bc3b89545d6370a3e30effb00ae114488523396356700332e01982c016f34d4d498fb05e9b0060a0c4b5e38f9e0052b00e0905e1d
-  data.tar.gz: 904809eb087e1aee239eb4871d98f633646193b3f09c2a8db48f8704b67cc0a32397950af5f34414c63e143dc0af4e37a824294fd3d6795964a81fb376c3ea93
+  metadata.gz: c1a02ce11eb4834caefa3b4d98e96fddc3f0781f4f2790c178080b1e9b6d68bb00c925afbbbf5c0913dbe9231e457290f2db1b5dc59af8bfcc57552930c4e6a1
+  data.tar.gz: 3fab5c98423cdb3026cc7e5edc13d53d540f94d72213fad81042b4602a6431564fa022590f4eb938b41d648266a526f5be26e34157e32b889c6e1bd2409ec74b

data/Dockerfile

@@ -0,0 +1,25 @@
+FROM docker.pkg.github.com/neonlaw/codebase/base:latest
+
+ENV DATABASE_URL $DATABASE_URL
+ENV FROM_DATABASE_URL $FROM_DATABASE_URL
+ENV TO_DATABASE_URL $TO_DATABASE_URL
+
+# Install dependencies from apt-get
+RUN apt-get update -qqy &&\
+  apt-get -qqyy install ruby-full
+RUN gem install bundler
+
+## TODO use Doppler
+# RUN (curl -Ls https://cli.doppler.com/install.sh || wget -qO- https://cli.doppler.com/install.sh) | sh
+# ENTRYPOINT [ "doppler", "run", "--" ]
+
+WORKDIR /app/postgres
+
+COPY ./postgres/Gemfile .
+COPY ./postgres/Gemfile.lock .
+RUN bundle install
+
+COPY ./postgres ./
+
+ENTRYPOINT [ "bundle", "exec" ]
+CMD ["ruby", "lib/neon_postgres/inter_database_copy/copy.rb"]

data/Gemfile

@@ -12,3 +12,5 @@ gem "fixture_dependencies", "~> 1.10"
 gem "pg", "~> 1.2"
 
 gem "pry", "~> 0.14.1"
+
+gem "faker", "~> 2.17"

data/Gemfile.lock

@@ -1,15 +1,20 @@
 PATH
   remote: .
   specs:
-    neon_postgres (0.1.0)
+    neon_postgres (0.0.2)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
     coderay (1.1.3)
+    concurrent-ruby (1.1.8)
     diff-lcs (1.4.4)
+    faker (2.17.0)
+      i18n (>= 1.6, < 2)
     fixture_dependencies (1.10.0)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
     method_source (1.0.0)
     parallel (1.20.1)
     parser (3.0.1.0)
@@ -60,6 +65,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  faker (~> 2.17)
   fixture_dependencies (~> 1.10)
   neon_postgres!
   pg (~> 1.2)

data/README.md

@@ -4,6 +4,29 @@ This gem is mainly used to test the Neon Law database which is created and
 maintained by Graphile. It is also consumed by other background-job ruby gems
 that handle non-web-request related processing.
 
+## Inter Database Copy
+
+To copy data between two databases run the following script from the root of
+this codebase (`../`) from this directory.
+
+```bash
+doppler setup # project inter_db_copy, environment production
+
+# Copy the GCP credentials to your machine from the above directory
+yarn copy-gcp-credentials
+
+# These following commands start db processes so you may need to run them in
+# separate shells.
+yarn sql-proxy-production
+yarn sql-proxy-staging
+
+# Then in this directory run
+doppler run -- bundle exec ruby lib/neon_postgres/inter_database_copy/copy.rb
+```
+
+We use this ruby script to copy data from `production` to `staging` so we can
+freely showcase our software without fear of disclosing client interests.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at

data/lib/neon_postgres.rb

@@ -1,27 +1,14 @@
-require "neon_postgres/version"
 require "sequel"
 
+require "neon_postgres/version"
+require "neon_postgres/database"
+require "neon_postgres/document_sanitizer"
+require "neon_postgres/sequel_models"
+Dir["#{__dir__}/neon_postgres/inter_database_copy/*"].sort.each { |file|
+  next if /copy.rb$/.match?(file)
+  require file
+}
+
 module NeonPostgres
   class Error < StandardError; end
-
-  class Database
-    def self.connection
-      @_connection ||= Sequel.connect(
-        ENV.fetch("DATABASE_URL") {
-          "postgres://postgres:password@localhost:5432/neon_law"
-        }
-      )
-    end
-
-    def self.anonymous_connection
-      @_anonymous_connection ||= Sequel.connect(
-        ENV.fetch("DATABASE_URL") {
-          "postgres://postgres:password@localhost:5432/neon_law"
-        },
-        connect_sqls: [
-          "SET role = 'anonymous';"
-        ]
-      )
-    end
-  end
 end

data/lib/neon_postgres/database.rb

@@ -0,0 +1,76 @@
+require "sequel"
+
+module NeonPostgres
+  class Database
+    def self.connection
+      @_connection ||= Sequel.connect(
+        ENV.fetch("DATABASE_URL") {
+          "postgres://postgres:password@localhost:5432/neon_law"
+        }
+      )
+    end
+
+    def self.anonymous_connection
+      @_anonymous_connection ||= Sequel.connect(
+        ENV.fetch("DATABASE_URL") {
+          "postgres://postgres:password@localhost:5432/neon_law"
+        },
+        connect_sqls: [
+          "SET role = 'anonymous';"
+        ]
+      )
+    end
+
+    def self.portal_connection(user_id:)
+      @_portal_connection ||= Sequel.connect(
+        ENV.fetch("DATABASE_URL") {
+          "postgres://postgres:password@localhost:5432/neon_law"
+        },
+        connect_sqls: [
+          "SET role = 'portal';",
+          "SET \"person.id\" = '#{user_id}';"
+        ]
+      )
+    end
+
+    def self.lawyer_connection(user_id:)
+      @_lawyer_connection ||= Sequel.connect(
+        ENV.fetch("DATABASE_URL") {
+          "postgres://postgres:password@localhost:5432/neon_law"
+        },
+        connect_sqls: [
+          "SET role = 'lawyer';",
+          "SET \"person.id\" = '#{user_id}';"
+        ]
+      )
+    end
+
+    def self.admin_connection(user_id:)
+      @_admin_connection ||= Sequel.connect(
+        ENV.fetch("DATABASE_URL") {
+          "postgres://postgres:password@localhost:5432/neon_law"
+        },
+        connect_sqls: [
+          "SET role = 'admin';",
+          "SET \"person.id\" = '#{user_id}';"
+        ]
+      )
+    end
+
+    def self.from_connection
+      @_from_connection ||= Sequel.connect(
+        ENV.fetch("FROM_DATABASE_URL") {
+          "postgres://postgres:password@localhost:5432/production"
+        }
+      )
+    end
+
+    def self.to_connection
+      @_to_connection ||= Sequel.connect(
+        ENV.fetch("TO_DATABASE_URL") {
+          "postgres://postgres:password@localhost:5432/staging"
+        }
+      )
+    end
+  end
+end

data/lib/neon_postgres/document_sanitizer.rb

@@ -0,0 +1,46 @@
+require_relative "./database"
+require "faker"
+
+module NeonPostgres
+  class DocumentSanitizer
+    def self.sanitize(
+      document:, database_connection: NeonPostgres::Database.connection
+    )
+      new(
+        database_connection: database_connection,
+        document: document
+      ).sanitize
+    end
+
+    def initialize(database_connection:, document:)
+      @database_connection = database_connection
+      @document = document
+    end
+
+    def sanitize
+      values = documents_connection.find(id: document.fetch(:id)).first
+
+      filename = "#{Faker::Internet.uuid}.pdf"
+
+      values[:filename] = filename
+      values[:gcp_url] = "https://neon-law-staging-private-assets.storage."\
+        "googleapis.com/matters/#{Faker::Internet.uuid}/#{filename}"
+
+      documents_connection.insert_conflict(
+        target: :id,
+        update: {
+          filename: Sequel[:excluded][:filename],
+          gcp_url: Sequel[:excluded][:gcp_url]
+        }
+      ).insert(values)
+    end
+
+    private
+
+    attr_reader :database_connection, :document
+
+    def documents_connection
+      @_documents_connection ||= database_connection[:documents]
+    end
+  end
+end

data/lib/neon_postgres/inter_database_copy/copy.rb

@@ -0,0 +1,34 @@
+require_relative "../document_sanitizer"
+require_relative "./document_template_copier"
+require_relative "./matter_copier"
+require_relative "./matter_template_copier"
+require_relative "./matter_documents_copier"
+
+NeonPostgres::InterDatabaseCopy::DocumentTemplateCopier.copy
+NeonPostgres::InterDatabaseCopy::MatterTemplateCopier.copy
+NeonPostgres::InterDatabaseCopy::MatterCopier.copy
+NeonPostgres::InterDatabaseCopy::MatterDocumentsCopier.copy
+
+# Add the admin test user as a lawyer matter contact to all matters
+admin_id = NeonPostgres::Database.to_connection[:people]
+  .where(email: "admin@sink.sendgrid.com").first.fetch(:id)
+
+NeonPostgres::Database.to_connection[:matters].all.each do |matter|
+  puts "Copying matter #{matter.fetch(:id)}"
+
+  NeonPostgres::Database.to_connection[:matter_contacts].insert_conflict.insert(
+    matter_id: matter.fetch(:id),
+    contact_id: admin_id,
+    role: "lawyer"
+  )
+end
+
+# Sanitize the documents in the to connection
+NeonPostgres::Database.to_connection[:documents].all.each do |document|
+  puts "sanitizing #{document.fetch(:id)}"
+
+  NeonPostgres::DocumentSanitizer.sanitize(
+    document: document,
+    database_connection: NeonPostgres::Database.to_connection
+  )
+end

data/lib/neon_postgres/inter_database_copy/document_template_copier.rb

@@ -0,0 +1,49 @@
+require_relative "../database"
+
+module NeonPostgres
+  module InterDatabaseCopy
+    class DocumentTemplateCopier
+      def self.copy(
+        from_connection: NeonPostgres::Database.from_connection,
+        to_connection: NeonPostgres::Database.to_connection
+      )
+        new(
+          from_connection: from_connection,
+          to_connection: to_connection
+        ).copy
+      end
+
+      def initialize(from_connection:, to_connection:)
+        @from_connection = from_connection
+        @to_connection = to_connection
+      end
+
+      def copy
+        from_dataset.each do |row|
+          to_dataset
+            .insert_conflict(
+              target: :id,
+              update: {
+                name: Sequel[:excluded][:name],
+                description: Sequel[:excluded][:description],
+                abbreviation: Sequel[:excluded][:abbreviation],
+                jurisdiction: Sequel[:excluded][:jurisdiction]
+              }
+            ).insert(row)
+        end
+      end
+
+      private
+
+      attr_reader :from_connection, :to_connection
+
+      def from_dataset
+        @_from_dataset ||= from_connection[:document_templates]
+      end
+
+      def to_dataset
+        @_to_dataset ||= to_connection[:document_templates]
+      end
+    end
+  end
+end

data/lib/neon_postgres/inter_database_copy/matter_copier.rb

@@ -0,0 +1,85 @@
+require_relative "../database"
+require "faker"
+
+module NeonPostgres
+  module InterDatabaseCopy
+    class MatterCopier
+      def self.copy(
+        from_connection: NeonPostgres::Database.from_connection,
+        to_connection: NeonPostgres::Database.to_connection
+      )
+        new(
+          from_connection: from_connection,
+          to_connection: to_connection
+        ).copy
+      end
+
+      def initialize(from_connection:, to_connection:)
+        @from_connection = from_connection
+        @to_connection = to_connection
+      end
+
+      def copy
+        from_dataset.each do |row|
+          to_connection[:matter_templates].insert_conflict(
+            target: :id,
+            update: {
+              category: Sequel[:excluded][:category],
+              name: Sequel[:excluded][:name],
+              description: Sequel[:excluded][:description]
+            }
+          ).insert({
+            id: row.fetch(:matter_template_id),
+            name: row.fetch(:matter_templates_name),
+            category: row.fetch(:category),
+            description: row.fetch(:matter_templates_description)
+          })
+
+          to_connection[:people].insert_conflict(
+            target: :id,
+            update: {
+              name: Sequel[:excluded][:name],
+              email: Sequel[:excluded][:email],
+              sub: Sequel[:excluded][:sub]
+            }
+          ).insert({
+            id: row.fetch(:people_id),
+            name: Faker::Name.name,
+            email: Faker::Internet.email,
+            sub: Faker::Internet.uuid
+          })
+
+          to_connection[:matters].insert_conflict(
+            target: :name,
+            update: {
+              matter_template_id: Sequel[:excluded][:matter_template_id],
+              primary_contact_id: Sequel[:excluded][:primary_contact_id],
+              description: Sequel[:excluded][:description]
+            }
+          ).insert({
+            name: "#{row.fetch(:matter_templates_name)} Matter",
+            matter_template_id: row.fetch(:matter_template_id),
+            description: JSON.generate({
+              body: Faker::Lorem.paragraphs(number: 3)
+            }),
+            primary_contact_id: row.fetch(:primary_contact_id)
+          })
+        end
+      end
+
+      private
+
+      attr_reader :from_connection, :to_connection
+
+      def from_dataset
+        @_from_dataset ||= from_connection[:matters].graph(
+          :people,
+          id: :primary_contact_id
+        ).graph(
+          :matter_templates,
+          id: Sequel[:matters][:matter_template_id]
+        )
+      end
+    end
+  end
+end

data/lib/neon_postgres/inter_database_copy/matter_documents_copier.rb

@@ -0,0 +1,82 @@
+require_relative "../database"
+require "faker"
+
+module NeonPostgres
+  module InterDatabaseCopy
+    class MatterDocumentsCopier
+      def self.copy(
+        from_connection: NeonPostgres::Database.from_connection,
+        to_connection: NeonPostgres::Database.to_connection
+      )
+        new(
+          from_connection: from_connection,
+          to_connection: to_connection
+        ).copy
+      end
+
+      def initialize(from_connection:, to_connection:)
+        @from_connection = from_connection
+        @to_connection = to_connection
+      end
+
+      def copy
+        from_dataset.each do |row|
+          to_connection[:documents].insert_conflict(
+            target: :id,
+            update: {
+              filename: Sequel[:excluded][:filename],
+              gcp_url: Sequel[:excluded][:gcp_url],
+              document_template_id: Sequel[:excluded][:document_template_id],
+              documentable_table_name: Sequel[:excluded][
+                :documentable_table_name
+              ],
+              created_at: Sequel[:excluded][:created_at]
+            }
+          ).insert({
+            id: row.fetch(:document_id),
+            filename: row.fetch(:filename),
+            gcp_url: row.fetch(:gcp_url),
+            documentable_table_name: row.fetch(:documentable_table_name),
+            document_template_id: row.fetch(:document_template_id),
+            created_at: row.fetch(:created_at)
+          })
+
+          matter_id = to_connection[:matters].select(:id).all.sample.fetch(:id)
+          author_id = to_connection[:people].select(:id).all.sample.fetch(:id)
+
+          to_connection[:matter_documents].insert_conflict({
+            target: :document_id,
+            update: {
+              matter_id: Sequel[:excluded][:matter_id],
+              author_id: Sequel[:excluded][:author_id]
+            }
+          }).insert({
+            document_id: row.fetch(:document_id),
+            matter_id: matter_id,
+            author_id: author_id
+          })
+        end
+      end
+
+      private
+
+      attr_reader :from_connection, :to_connection
+
+      def from_dataset
+        @_from_dataset ||= from_connection[:matter_documents].graph(
+          :documents,
+          id: Sequel[:matter_documents][:document_id]
+        ).graph(
+          :matters,
+          id: Sequel[:matter_documents][:matter_id]
+        ).graph(
+          :people,
+          id: Sequel[:matter_documents][:author_id]
+        ).graph(
+          :document_templates,
+          id: Sequel[:documents][:document_template_id]
+        )
+      end
+    end
+  end
+end

data/lib/neon_postgres/inter_database_copy/matter_template_copier.rb

@@ -0,0 +1,47 @@
+require_relative "../database"
+
+module NeonPostgres
+  module InterDatabaseCopy
+    class MatterTemplateCopier
+      def self.copy(
+        from_connection: NeonPostgres::Database.from_connection,
+        to_connection: NeonPostgres::Database.to_connection
+      )
+        new(
+          from_connection: from_connection,
+          to_connection: to_connection
+        ).copy
+      end
+
+      def initialize(from_connection:, to_connection:)
+        @from_connection = from_connection
+        @to_connection = to_connection
+      end
+
+      def copy
+        from_dataset.each do |row|
+          to_dataset
+            .insert_conflict(
+              target: :name,
+              update: {
+                category: Sequel[:excluded][:category],
+                description: Sequel[:excluded][:description]
+              }
+            ).insert(row)
+        end
+      end
+
+      private
+
+      attr_reader :from_connection, :to_connection
+
+      def from_dataset
+        @_from_dataset ||= from_connection[:matter_templates]
+      end
+
+      def to_dataset
+        @_to_dataset ||= to_connection[:matter_templates]
+      end
+    end
+  end
+end

data/lib/neon_postgres/sequel_models.rb

@@ -0,0 +1,36 @@
+require_relative "./database"
+Sequel::Model.db = NeonPostgres::Database.connection
+
+module NeonPostgres
+  module Models
+    class Person < Sequel::Model
+      one_to_many :matters, key: :primary_contact_id
+      one_to_many :matter_documents, key: :author_id
+    end
+
+    class DocumentTemplate < Sequel::Model
+      one_to_many :documents
+    end
+
+    class MatterTemplate < Sequel::Model
+      one_to_many :matters
+    end
+
+    class Matter < Sequel::Model
+      many_to_one :matter_template
+      many_to_one :matter_documents
+      many_to_one :primary_contact, class: :person, key: :primary_contact_id
+    end
+
+    class Document < Sequel::Model
+      many_to_one :document_template
+      one_to_many :matter_document
+    end
+
+    class MatterDocument < Sequel::Model
+      many_to_one :document
+      many_to_one :matter
+      many_to_one :author, class: :person, key: :author_id
+    end
+  end
+end

data/lib/neon_postgres/version.rb

@@ -1,3 +1,3 @@
 module NeonPostgres
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neon_postgres
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Neon Law
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-27 00:00:00.000000000 Z
+date: 2021-05-25 00:00:00.000000000 Z
 dependencies: []
 description: |-
   This gem contains tests and sequel setup to speak to
@@ -21,11 +21,20 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
+- Dockerfile
 - Gemfile
 - Gemfile.lock
 - README.md
 - Rakefile
 - lib/neon_postgres.rb
+- lib/neon_postgres/database.rb
+- lib/neon_postgres/document_sanitizer.rb
+- lib/neon_postgres/inter_database_copy/copy.rb
+- lib/neon_postgres/inter_database_copy/document_template_copier.rb
+- lib/neon_postgres/inter_database_copy/matter_copier.rb
+- lib/neon_postgres/inter_database_copy/matter_documents_copier.rb
+- lib/neon_postgres/inter_database_copy/matter_template_copier.rb
+- lib/neon_postgres/sequel_models.rb
 - lib/neon_postgres/version.rb
 - neon_postgres.gemspec
 homepage: https://github.com/neonlaw/codebase