art_vandelay 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 26029667adccdd12b5098fb5e370050bb3bf4877dceb712cba6ce4f4032aec3f
+  data.tar.gz: '05569d49fa0ad8da29bb07947e77db6425ba12506699b073c27690a644592b52'
+SHA512:
+  metadata.gz: 1ae5ff614ad52587111bf87f10ea736d1087100ff3c782e25c4a8f8e01732f2829e7cd7640172c3e20756f20d6f9faa733b97f6fa53da39b9f442b0d796cd149
+  data.tar.gz: 66c22f9871d06bc1a1d3f4852ef57d919b78960821eabd1f9c7e2c137954db3aa2e2d2d85d31c87f5038710d6a8bc66b139c5d753bc7df9abd3bf40236249a73

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2022 Steve Polito and thoughtbot, inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,232 @@
+# 🌐 Art Vandelay
+[![GitHub Actions
+Demo](https://github.com/thoughtbot/art_vandelay/actions/workflows/ci.yml/badge.svg)](https://github.com/thoughtbot/art_vandelay/actions/workflows/ci.yml)
+
+Art Vandelay is an importer/exporter for Rails 6.0 and higher.
+
+Have you ever been on a project where, out of nowhere, someone asks you to send them a CSV of data? You think to yourself, “Ok, cool. No big deal. Just gimme five minutes”, but then that five minutes turns into a few hours. Art Vandelay can help. 
+
+**At a high level, here’s what Art Vandelay can do:**
+
+- 🕶 Automatically [filters out sensitive information](#%EF%B8%8F-configuration).
+- 🔁 Export data [in batches](#exporting-in-batches).
+- 📧 [Email](#artvandelayexportemail_csv) exported data.
+- 📥 [Import data](#-importing) from a CSV.
+
+## ✅ Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "art_vandelay", git: "https://github.com/thoughtbot/art_vandelay"
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+## ⚙️ Configuration
+
+```ruby
+# config/initializers/art_vandelay.rb
+ArtVandelay.setup do |config|
+  config.filtered_attributes = [:credit_card, :birthday]
+  config.from_address = "no-reply-export@example.com"
+  config.in_batches_of = 5000
+end
+```
+#### Default Values
+
+|Attribute|Value|Description|
+|---------|-----|-----------|
+|`filtered_attributes`|`[:passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn]`|Attributes that will be automatically filtered when exported|
+|`from_address`|`nil`|The email address used when sending an email of exports|
+|`in_batches_of`|`10000`|The number of records that will be exported into each CSV|
+
+## 🧰 Usage
+
+### 📤 Exporting
+
+```ruby
+ArtVandelay::Export.new(records, export_sensitive_data: false, attributes: [], in_batches_of: ArtVandelay.in_batches_of)
+```
+
+|Argument|Description|
+|--------|-----------|
+|`records`|An [Active Record Relation](https://api.rubyonrails.org/classes/ActiveRecord/Relation.html) or an instance of an Active Record. E.g. `User.all`, `User.first`, `User.where(...)`, `User.find_by`|
+|`export_sensitive_data`|Export sensitive data. Defaults to `false`. Can be configured with `ArtVandelay.filtered_attributes`.|
+|`attributes`|An array attributes to export. Default to all.|
+|`in_batches_of`|The number of records that will be exported into each CSV. Defaults to 10,000. Can be configured with `ArtVandelay.in_batches_of`|
+
+#### ArtVandelay::Export#csv
+
+Returns an instance of `ArtVandelay::Export::Result`.
+
+```ruby
+result = ArtVandelay::Export.new(User.all).csv
+# => #<ArtVandelay::Export::Result>
+
+csv_exports = result.result.csv_exports
+# => [#<CSV::Table>, #<CSV::Table>, ...]
+
+csv = csv_exports.first.to_a
+# => [["id", "email", "password", "created_at", "updated_at"], ["1", "user@example.com", "[FILTERED]", "2022-10-25 09:20:28 UTC", "2022-10-25 09:20:28 UTC"]]
+```
+
+##### Exporting Sensitive Data
+
+```ruby
+result = ArtVandelay::Export.new(User.all, export_sensitive_data: true).csv
+# => #<ArtVandelay::Export::Result>
+
+password = result.csv_exports.first["password"]
+# => ["bosco"]
+```
+
+##### Exporting Specific Attributes
+
+```ruby
+result = ArtVandelay::Export.new(User.all, attributes: [:email]).csv
+# => #<ArtVandelay::Export::Result>
+
+csv = result.csv_exports.first.to_a
+# => [["email"], ["george@vandelay_industries.com"]]
+```
+
+##### Exporting in Batches
+
+```ruby
+result = ArtVandelay::Export.new(User.all, in_batches_of: 100).csv
+# => #<ArtVandelay::Export::Result>
+
+csv_size = result.csv_exports.first.size
+# => 100
+```
+
+#### ArtVandelay::Export#email_csv
+
+Emails the recipient(s) CSV exports as attachments.
+
+```ruby
+email_csv(to:, from: ArtVandelay.from_address, subject: "#{model_name} export", body: "#{model_name} export")
+```
+
+|Argument|Description|
+|---------|-----|
+|`to`|An array of email addresses representing who should receive the email.|
+|`from`|The email address of the sender.|
+|`subject`|The email subject. Defaults to the following pattern: "User export"|
+|`body`|The email body. Defaults to the following pattern: "User export"|
+
+```ruby
+ArtVandelay::Export
+  .new(User.where.not(confirmed: nil))
+  .email_csv(
+    to: ["george@vandelay_industries.com", "kel_varnsen@vandelay_industries.com"],
+    from: "noreply@vandelay_industries.com",
+    subject: "List of confirmed users",
+    body: "Here's an export of all confirmed users in our database."
+  )
+# => ActionMailer::Base#mail: processed outbound mail in...  
+```
+
+### 📥 Importing
+
+```ruby
+ArtVandelay::Import.new(model_name, **options)
+```
+
+|Argument|Description|
+|--------|-----------|
+|`model_name`|The name of the model being imported. E.g. `:users`, `:user`, `"users"` or `"user"`|
+|`**options`|A hash of options. Available options are `rollback:`|
+
+#### Options
+
+|Option|Description|
+|------|-----------|
+|`rollback:`|Whether the import should rollback if any of the records fails to save.|
+
+#### ArtVandelay::Import#csv
+
+Imports records from the supplied CSV. Returns an instance of `ArtVandelay::Import::Result`.
+
+```ruby
+csv_string = CSV.generate do |csv|
+  csv << ["email", "password"]
+  csv << ["george@vandelay_industries.com", "bosco"]
+  csv << ["kel_varnsen@vandelay_industries.com", nil]
+end
+
+result = ArtVandelay::Import.new(:users).csv(csv_string)
+# => #<ArtVandelay::Import::Result>
+
+result.rows_accepted
+# => [{:row=>["george@vandelay_industries.com", "bosco"], :id=>1}]
+
+result.rows_rejected
+# => [{:row=>["kel_varnsen@vandelay_industries.com", nil], :errors=>{:password=>["can't be blank"]}}]
+```
+
+```ruby
+csv(csv_string, **options)
+```
+
+|Argument|Description|
+|--------|-----------|
+|`csv_string`|Data in the form of a CSV string.|
+|`**options`|A hash of options. Available options are `headers:` and `attributes:`|
+
+#### Options
+
+|Option|Description|
+|------|-----------|
+|`headers:`|The CSV headers. Use when the supplied CSV string does not have headers.|
+|`attributes:`|The attributes the headers should map to. Useful if the headers do not match the model's attributes.|
+
+##### Rolling back if a record fails to save
+
+```ruby
+csv_string = CSV.generate do |csv|
+  csv << ["email", "password"]
+  csv << ["george@vandelay_industries.com", "bosco"]
+  csv << ["kel_varnsen@vandelay_industries.com", nil]
+end
+
+result = ArtVandelay::Import.new(:users, rollback: true).csv(csv_string)
+# => rollback transaction
+```
+
+##### Setting headers
+
+```ruby
+csv_string = CSV.generate do |csv|
+  csv << ["george@vandelay_industries.com", "bosco"]
+end
+
+result = ArtVandelay::Import.new(:users).csv(csv_string, headers: [:email, :password])
+# => #<ArtVandelay::Import::Result>
+```
+
+##### Mapping custom headers
+
+```ruby
+csv_string = CSV.generate do |csv|
+  csv << ["email_address", "passcode"]
+  csv << ["george@vandelay_industries.com", "bosco"]
+end
+
+result = ArtVandelay::Import.new(:users).csv(csv_string, attributes: {email_address: :email, passcode: :password})
+# => #<ArtVandelay::Import::Result>
+```
+
+## 🙏 Contributing
+
+1. Run `./bin/setup`.
+2. Make your changes.
+3. Ensure `./bin/ci` passes.
+4. Create a [pull request](https://github.com/thoughtbot/art_vandelay/compare).
+
+## 📜 License
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("test/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+load "rails/tasks/statistics.rake"
+
+require "bundler/gem_tasks"

data/app/assets/config/art_vandelay_manifest.js

@@ -0,0 +1 @@
+//= link_directory ../stylesheets/art_vandelay .css

data/app/assets/stylesheets/art_vandelay/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any other CSS/SCSS
+ * files in this directory. Styles in this file should be added after the last require_* statement.
+ * It is generally better to create a new file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/art_vandelay/application_controller.rb

@@ -0,0 +1,4 @@
+module ArtVandelay
+  class ApplicationController < ActionController::Base
+  end
+end

data/app/helpers/art_vandelay/application_helper.rb

@@ -0,0 +1,4 @@
+module ArtVandelay
+  module ApplicationHelper
+  end
+end

data/app/jobs/art_vandelay/application_job.rb

@@ -0,0 +1,4 @@
+module ArtVandelay
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/mailers/art_vandelay/application_mailer.rb

@@ -0,0 +1,6 @@
+module ArtVandelay
+  class ApplicationMailer < ActionMailer::Base
+    default from: "from@example.com"
+    layout "mailer"
+  end
+end

data/app/models/art_vandelay/application_record.rb

@@ -0,0 +1,5 @@
+module ArtVandelay
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/views/layouts/art_vandelay/application.html.erb

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Art vandelay</title>
+  <%= csrf_meta_tags %>
+  <%= csp_meta_tag %>
+
+  <%= stylesheet_link_tag    "art_vandelay/application", media: "all" %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,2 @@
+ArtVandelay::Engine.routes.draw do
+end

data/lib/art_vandelay/engine.rb

@@ -0,0 +1,5 @@
+module ArtVandelay
+  class Engine < ::Rails::Engine
+    isolate_namespace ArtVandelay
+  end
+end

data/lib/art_vandelay/version.rb

@@ -0,0 +1,3 @@
+module ArtVandelay
+  VERSION = "0.1.0"
+end

data/lib/art_vandelay.rb

@@ -0,0 +1,204 @@
+require "art_vandelay/version"
+require "art_vandelay/engine"
+require "csv"
+
+module ArtVandelay
+  mattr_accessor :filtered_attributes, :from_address, :in_batches_of
+  @@filtered_attributes = [:passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn]
+  @@in_batches_of = 10000
+
+  def self.setup
+    yield self
+  end
+
+  class Error < StandardError
+  end
+
+  class Export
+    class Result
+      attr_reader :csv_exports
+
+      def initialize(csv_exports)
+        @csv_exports = csv_exports
+      end
+    end
+
+    # TODO attributes: self.filtered_attributes
+    def initialize(records, export_sensitive_data: false, attributes: [], in_batches_of: ArtVandelay.in_batches_of)
+      @records = records
+      @export_sensitive_data = export_sensitive_data
+      @attributes = attributes
+      @in_batches_of = in_batches_of
+    end
+
+    def csv
+      csv_exports = []
+
+      if records.is_a?(ActiveRecord::Relation)
+        records.in_batches(of: in_batches_of) do |relation|
+          csv_exports << CSV.parse(generate_csv(relation), headers: true)
+        end
+      elsif records.is_a?(ActiveRecord::Base)
+        csv_exports << CSV.parse(generate_csv(records), headers: true)
+      end
+
+      Result.new(csv_exports)
+    end
+
+    def email_csv(to:, from: ArtVandelay.from_address, subject: "#{model_name} export", body: "#{model_name} export")
+      if from.nil?
+        raise ArtVandelay::Error, "missing keyword: :from. Alternatively, set a value on ArtVandelay.from_address"
+      end
+
+      mailer = ActionMailer::Base.mail(to: to, from: from, subject: subject, body: body)
+      csv_exports = csv.csv_exports
+
+      csv_exports.each.with_index(1) do |csv, index|
+        if csv_exports.one?
+          mailer.attachments[file_name] = csv
+        else
+          mailer.attachments[file_name(suffix: "-#{index}")] = csv
+        end
+      end
+
+      mailer.deliver
+    end
+
+    private
+
+    attr_reader :records, :export_sensitive_data, :attributes, :in_batches_of
+
+    def file_name(**options)
+      options = options.symbolize_keys
+      suffix = options[:suffix]
+      prefix = model_name.downcase
+      timestamp = Time.current.in_time_zone("UTC").strftime("%Y-%m-%d-%H-%M-%S-UTC")
+
+      "#{prefix}-export-#{timestamp}#{suffix}.csv"
+    end
+
+    def filtered_values(attributes)
+      if export_sensitive_data
+        ActiveSupport::ParameterFilter.new([]).filter(attributes).values
+      else
+        ActiveSupport::ParameterFilter.new(ArtVandelay.filtered_attributes).filter(attributes).values
+      end
+    end
+
+    def generate_csv(relation)
+      CSV.generate do |csv|
+        csv << header
+        if relation.is_a?(ActiveRecord::Relation)
+          relation.each do |record|
+            csv << row(record.attributes)
+          end
+        elsif relation.is_a?(ActiveRecord::Base)
+          csv << row(records.attributes)
+        end
+      end
+    end
+
+    def header
+      if attributes.any?
+        model.attribute_names.select do |column_name|
+          standardized_attributes.include?(column_name)
+        end
+      else
+        model.attribute_names
+      end
+    end
+
+    def model
+      model_name.constantize
+    end
+
+    def model_name
+      records.model_name.name
+    end
+
+    def row(attributes)
+      if self.attributes.any?
+        filtered_values(attributes.slice(*standardized_attributes))
+      else
+        filtered_values(attributes)
+      end
+    end
+
+    def standardized_attributes
+      attributes.map(&:to_s)
+    end
+  end
+
+  class Import
+    class Result
+      attr_reader :rows_accepted, :rows_rejected
+
+      def initialize(rows_accepted:, rows_rejected:)
+        @rows_accepted = rows_accepted
+        @rows_rejected = rows_rejected
+      end
+    end
+
+    def initialize(model_name, **options)
+      @options = options.symbolize_keys
+      @rollback = options[:rollback]
+      @model_name = model_name
+    end
+
+    def csv(csv_string, **options)
+      options = options.symbolize_keys
+      headers = options[:headers] || true
+      attributes = options[:attributes] || {}
+      rows = build_csv(csv_string, headers)
+
+      if rollback
+        # TODO: It would be nice to still return a result object during a
+        # failure
+        active_record.transaction do
+          parse_rows(rows, attributes, raise_on_error: true)
+        end
+      else
+        parse_rows(rows, attributes)
+      end
+    end
+
+    private
+
+    attr_reader :model_name, :rollback
+
+    def active_record
+      model_name.to_s.classify.constantize
+    end
+
+    def build_csv(csv_string, headers)
+      CSV.parse(csv_string, headers: headers)
+    end
+
+    def build_params(row, attributes)
+      attributes = attributes.stringify_keys
+
+      row.to_h.stringify_keys.transform_keys do |key|
+        attributes[key] || key
+      end
+    end
+
+    def parse_rows(rows, attributes, **options)
+      options = options.symbolize_keys
+      raise_on_error = options[:raise_on_error] || false
+      result = Result.new(rows_accepted: [], rows_rejected: [])
+
+      rows.each do |row|
+        params = build_params(row, attributes)
+        record = active_record.new(params)
+
+        if raise_on_error ? record.save! : record.save
+          result.rows_accepted << {row: row.fields, id: record.id}
+        else
+          result.rows_rejected << {row: row.fields, errors: record.errors.messages}
+        end
+      end
+
+      result
+    end
+  end
+end

data/lib/tasks/art_vandelay_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :art_vandelay do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification
+name: art_vandelay
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Steve Polito
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Art Vandelay is an importer/exporter for Rails
+email:
+- stevepolito@hey.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/assets/config/art_vandelay_manifest.js
+- app/assets/stylesheets/art_vandelay/application.css
+- app/controllers/art_vandelay/application_controller.rb
+- app/helpers/art_vandelay/application_helper.rb
+- app/jobs/art_vandelay/application_job.rb
+- app/mailers/art_vandelay/application_mailer.rb
+- app/models/art_vandelay/application_record.rb
+- app/views/layouts/art_vandelay/application.html.erb
+- config/routes.rb
+- lib/art_vandelay.rb
+- lib/art_vandelay/engine.rb
+- lib/art_vandelay/version.rb
+- lib/tasks/art_vandelay_tasks.rake
+homepage: https://github.com/thoughtbot/art_vandelay
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/thoughtbot/art_vandelay
+  source_code_uri: https://github.com/thoughtbot/art_vandelay
+  changelog_uri: https://github.com/thoughtbot/art_vandelay/NEWS.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.3
+signing_key:
+specification_version: 4
+summary: Art Vandelay is an importer/exporter for Rails
+test_files: []