staging_table 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c89644e006af09c46ea8ea69f9a5687b20c88e400fbb1436615c10982ce63eb6
+  data.tar.gz: 90a9f0c62c6a63b60fb512411c2a74a26f927cb66a8407b082a8bb1e6421ca6c
+SHA512:
+  metadata.gz: fc32f9c5d72b5971002d503bf3969c8941e6675d1764d59976a366a14a95134c2ca22259b70c1a84b29c04f085df9c217c4b9451f506094d7c32150b902cc309
+  data.tar.gz: bb3e29593966192d22fc8168656100ba7880d5f0cfa14f520667dec0ad0822c0efd0212c4d4d53e4e0c1bb49fd06f3b0ad63431a78111ee2ddd535efc0f7cf31

data/.github/workflows/rbs.yml

@@ -0,0 +1,30 @@
+name: RBS
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  validate:
+    name: Validate Type Signatures
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+
+      - name: Install RBS
+        run: gem install rbs
+
+      - name: Install RBS collection
+        run: rbs collection install
+
+      - name: Validate RBS signatures
+        run: rbs -I sig validate

data/.github/workflows/test.yml

@@ -0,0 +1,124 @@
+name: Tests
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test-sqlite:
+    name: Ruby ${{ matrix.ruby }} - SQLite
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ['3.1', '3.2', '3.3']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby ${{ matrix.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Run tests with SQLite
+        run: bundle exec rspec
+
+  test-postgresql:
+    name: Ruby ${{ matrix.ruby }} - PostgreSQL
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ['3.1', '3.2', '3.3']
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: staging_table_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby ${{ matrix.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Run tests with PostgreSQL
+        run: bundle exec rspec
+        env:
+          POSTGRES_HOST: localhost
+          POSTGRES_PORT: 5432
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: staging_table_test
+
+  test-mysql:
+    name: Ruby ${{ matrix.ruby }} - MySQL
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ['3.1', '3.2', '3.3']
+
+    services:
+      mysql:
+        image: mysql:8.0
+        env:
+          MYSQL_ROOT_PASSWORD: root
+          MYSQL_DATABASE: staging_table_test
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd "mysqladmin ping -h localhost"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby ${{ matrix.ruby }}
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Run tests with MySQL
+        run: bundle exec rspec
+        env:
+          MYSQL_HOST: 127.0.0.1
+          MYSQL_PORT: 3306
+          MYSQL_USER: root
+          MYSQL_PASSWORD: root
+          MYSQL_DB: staging_table_test
+
+  lint:
+    name: Standard Ruby
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+
+      - name: Run Standard Ruby
+        run: bundle exec standardrb

data/.gitignore

@@ -0,0 +1,40 @@
+# Gem build artifacts
+*.gem
+pkg/
+
+# Bundle
+/.bundle/
+vendor/bundle
+
+# Ruby version managers
+.ruby-version
+.ruby-gemset
+.rvmrc
+
+# RSpec
+spec/examples.txt
+
+# Coverage
+/coverage/
+
+# IDE
+.idea/
+*.swp
+*.swo
+.vscode/
+
+# macOS
+.DS_Store
+
+# Logs
+*.log
+
+# Temporary files
+tmp/
+*.tmp
+
+Gemfile.lock
+
+# RBS type collection
+.gem_rbs_collection/
+rbs_collection.lock.yaml

data/.rspec

@@ -0,0 +1,3 @@
+--require spec_helper
+--format documentation
+--color

data/Gemfile

@@ -0,0 +1,14 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in staging_table.gemspec
+gemspec
+
+group :development, :test do
+  gem "rspec", "~> 3.0"
+  gem "sqlite3"
+  gem "pg"
+  gem "mysql2"
+  gem "ruby-lsp"
+  gem "ruby-lsp-rspec"
+  gem "standard"
+end

data/README.md

@@ -0,0 +1,327 @@
+# 🎭 StagingTable
+
+**The red carpet for your data before it hits the main stage.**
+
+[![Gem Version](https://badge.fury.io/rb/staging_table.svg)](https://badge.fury.io/rb/staging_table)
+[![Test Status](https://github.com/eagerworks/staging_table/actions/workflows/test.yml/badge.svg)](https://github.com/eagerworks/staging_table/actions)
+[![RBS](https://github.com/eagerworks/staging_table/actions/workflows/rbs.yml/badge.svg)](https://github.com/eagerworks/staging_table/actions/workflows/rbs.yml)
+
+Stop shoving data directly into your production tables like a savage. Give it a dressing room first! 
+
+`StagingTable` lets you bulk import data into a temporary "staging" table, validate/massage it, and *then* gracefully transfer it to your real tables using efficient SQL strategies.
+
+It's like `INSERT INTO ... SELECT` but with a Ruby DSL that makes you smile.
+
+## 🌟 Why?
+
+Importing large datasets is hard.
+- **Direct inserts** are slow and bypass validations.
+- **ActiveRecord** is safe but slow for millions of records.
+- **Raw SQL** is fast but messy and hard to maintain.
+
+**StagingTable** gives you the best of both worlds:
+1.  **🚀 Speed**: Bulk insert into a temp table (no index overhead yet).
+2.  **🛡️ Safety**: Validate or query the data *before* it touches your real table.
+3.  **🧹 Cleanliness**: Automatic cleanup of temp tables.
+4.  **🔄 Power**: Built-in support for `UPSERT` (INSERT ON CONFLICT) and duplicate handling.
+
+---
+
+## 📦 Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'staging_table'
+```
+
+And then execute:
+
+```bash
+bundle install
+```
+
+---
+
+## 🛠️ Usage
+
+### The "Happy Path" (Block Syntax)
+
+The simplest way to use StagingTable. It handles the creation and cleanup of the temporary table automatically.
+
+```ruby
+# 1. Create a staging table that mirrors the 'users' table
+StagingTable.stage(User) do |staging|
+  
+  # 2. Bulk insert data (Hashes, AR Objects, or Relations)
+  staging.insert([
+    { name: 'John Doe', email: 'john@example.com' },
+    { name: 'Jane Doe', email: 'jane@example.com' }
+  ])
+  
+  # 3. The 'staging' object is a real ActiveRecord model!
+  #    You can query it, validate it, or massage data.
+  puts "Staged count: #{staging.count}"
+  staging.where(email: nil).delete_all
+  
+  # 4. When the block exits, data is automatically transferred 
+  #    to the 'users' table using a single SQL statement.
+end
+```
+
+### 📥 Importing Data
+
+The `insert` method is flexible. Feed it whatever you have:
+
+```ruby
+StagingTable.stage(User) do |staging|
+  # 🍎 Array of Hashes
+  staging.insert([
+    { name: 'John', email: 'john@example.com' },
+    { name: 'Jane', email: 'jane@example.com' }
+  ])
+
+  # 🍊 Array of ActiveRecord objects
+  staging.insert(User.where(active: true).to_a)
+
+  # 🍇 ActiveRecord::Relation (Lazy loading)
+  staging.insert(User.where(role: 'admin'))
+end
+```
+
+For massive datasets, use `insert_from_query` to process in batches and keep memory usage low:
+
+```ruby
+StagingTable.stage(User) do |staging|
+  # Processes in batches of 1000 (configurable)
+  staging.insert_from_query(User.where(needs_migration: true))
+end
+```
+
+### ⚔️ Handling Duplicates (Upsert)
+
+Don't let duplicates crash your party. Configure the transfer strategy to handle conflicts gracefully.
+
+```ruby
+StagingTable.stage(User, 
+  transfer_strategy: :upsert,    # Default is :insert
+  conflict_target: [:email],     # Column(s) to check for conflicts
+  conflict_action: :update       # :update (overwrite) or :ignore (skip)
+) do |staging|
+  staging.insert(records)
+end
+```
+
+### 📊 Transfer Results
+
+Every transfer returns a `TransferResult` with detailed statistics:
+
+```ruby
+result = StagingTable.stage(User) do |staging|
+  staging.insert(records)
+end
+
+puts result.inserted  # => 450  (new records)
+puts result.updated   # => 50   (updated via upsert)
+puts result.skipped   # => 10   (ignored conflicts)
+puts result.total     # => 510  (total processed)
+puts result.success?  # => true (any inserts or updates?)
+
+# Also available as a hash
+result.to_h # => { inserted: 450, updated: 50, skipped: 10, total: 510 }
+```
+
+### 🪝 Callbacks
+
+Hook into the staging lifecycle to validate, transform, or log:
+
+```ruby
+StagingTable.stage(User,
+  before_insert: ->(session) {
+    Rails.logger.info "Starting import..."
+  },
+  after_insert: ->(session, records) {
+    Rails.logger.info "Staged #{records.count} records"
+  },
+  before_transfer: ->(session) {
+    # Clean up invalid data before transfer
+    session.where(email: nil).delete_all
+    session.where(status: 'banned').delete_all
+  },
+  after_transfer: ->(session, result) {
+    Rails.logger.info "Imported #{result.inserted} new, updated #{result.updated}"
+  }
+) do |staging|
+  staging.insert(records)
+end
+```
+
+### 📡 Instrumentation (ActiveSupport::Notifications)
+
+Monitor and debug your imports in production with built-in instrumentation:
+
+```ruby
+# Subscribe to transfer events
+StagingTable::Instrumentation.subscribe(:transfer) do |event|
+  Rails.logger.info "[StagingTable] Transfer to #{event.payload[:source_table]} " \
+                    "completed in #{event.duration.round(2)}ms"
+  StatsD.measure('staging_table.transfer.duration', event.duration)
+  StatsD.increment('staging_table.transfer.inserted', event.payload[:result].inserted)
+end
+
+# Subscribe to all StagingTable events
+StagingTable::Instrumentation.subscribe_all do |event|
+  Rails.logger.debug "[StagingTable] #{event.name}: #{event.duration.round(2)}ms"
+end
+```
+
+**Available Events:**
+
+| Event | Payload | Description |
+|-------|---------|-------------|
+| `staging_table.stage` | `source_model`, `source_table`, `options`, `result` | Wraps the entire staging block |
+| `staging_table.create_table` | `source_model`, `source_table`, `staging_table` | When staging table is created |
+| `staging_table.insert` | `source_model`, `source_table`, `staging_table`, `record_count`, `batch_size` | When records are inserted |
+| `staging_table.transfer` | `source_model`, `source_table`, `staging_table`, `strategy`, `staged_count`, `result` | When data is transferred |
+| `staging_table.drop_table` | `source_model`, `source_table`, `staging_table` | When staging table is dropped |
+
+You can also use standard ActiveSupport::Notifications directly:
+
+```ruby
+ActiveSupport::Notifications.subscribe('staging_table.transfer') do |event|
+  # Your monitoring code here
+end
+```
+
+### 🎛️ Manual Control
+
+Need to keep the staging table alive across multiple background jobs? We got you.
+
+**Note:** Temporary tables in PostgreSQL are session-specific. This only works if you stay in the same DB connection!
+
+```ruby
+# Create the session
+session = StagingTable::Session.new(User, excluded_columns: %w[created_at updated_at])
+session.create_table
+
+begin
+  # Insert data in chunks
+  session.insert(batch_1)
+  session.insert(batch_2)
+  
+  # Run some sanity checks
+  if session.where(status: 'banned').exists?
+    raise "Whoa there! No banned users allowed."
+  end
+  
+  # Commit to the real table
+  result = session.transfer
+  puts "Transferred #{result.total} records"
+ensure
+  # Always clean up your mess
+  session.drop_table
+end
+```
+
+---
+
+## ⚙️ Configuration
+
+Set global defaults in an initializer (e.g., `config/initializers/staging_table.rb`):
+
+```ruby
+StagingTable.configure do |config|
+  config.default_batch_size = 2000
+  config.default_transfer_strategy = :insert # or :upsert
+end
+```
+
+---
+
+## 🔍 Type Checking (RBS)
+
+This gem ships with [RBS](https://github.com/ruby/rbs) type signatures for static type checking. The signatures are located in the `sig/` directory and are validated in CI.
+
+### Using the Type Signatures
+
+If you want to type-check your own code that uses StagingTable:
+
+```bash
+# Install RBS and the collection for dependencies
+gem install rbs
+rbs collection install
+
+# Validate signatures
+rbs -I sig validate
+```
+
+### Using with Steep
+
+For full type checking with [Steep](https://github.com/soutaro/steep):
+
+```ruby
+# Gemfile
+gem 'steep', group: :development
+```
+
+```ruby
+# Steepfile
+target :lib do
+  signature "sig"
+  check "lib"
+  library "activerecord"
+  library "activesupport"
+end
+```
+
+```bash
+bundle exec steep check
+```
+
+---
+
+## 💾 Supported Databases
+
+We speak your language.
+
+| Database | Strategy |
+|----------|----------|
+| **PostgreSQL** | `CREATE TABLE ... (LIKE ... INCLUDING DEFAULTS)` + `INSERT ... ON CONFLICT` |
+| **MySQL** | `CREATE TABLE ... LIKE ...` + `INSERT ... ON DUPLICATE KEY UPDATE` |
+| **SQLite** | Copies structure from `sqlite_master` + `INSERT ... ON CONFLICT` |
+
+---
+
+## 🤝 Contributing
+
+Found a bug? Want to add support for Oracle? (Please don't, but if you must...)
+
+1.  Fork it
+2.  Create your feature branch (`git checkout -b my-new-feature`)
+3.  Commit your changes (`git commit -am 'Add some feature'`)
+4.  Push to the branch (`git push origin my-new-feature`)
+5.  Create new Pull Request
+
+### Running Tests
+
+We support the big three. Set up your environment variables for PG/MySQL or just run SQLite tests out of the box.
+
+```bash
+# Run everything
+bundle exec rake spec
+
+# Pick your poison
+bundle exec rake spec:postgresql
+bundle exec rake spec:mysql
+bundle exec rspec --tag sqlite
+```
+
+---
+
+## 🙏 Special Thanks
+
+Special thanks to [agustin-peluffo](https://github.com/agustin-peluffo) who created the first implementation for a project!
+
+---
+
+*Made with ❤️ by [eagerworks](https://eagerworks.com)*

data/Rakefile

@@ -0,0 +1,19 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "standard/rake"
+
+RSpec::Core::RakeTask.new(:spec)
+
+namespace :spec do
+  desc "Run PostgreSQL specs only"
+  RSpec::Core::RakeTask.new(:postgresql) do |t|
+    t.rspec_opts = "--tag postgresql"
+  end
+
+  desc "Run MySQL specs only"
+  RSpec::Core::RakeTask.new(:mysql) do |t|
+    t.rspec_opts = "--tag mysql"
+  end
+end
+
+task default: :spec

data/lib/staging_table/adapters/base.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module StagingTable
+  module Adapters
+    class Base
+      attr_reader :connection
+
+      def initialize(connection)
+        @connection = connection
+      end
+
+      def create_table(temp_table_name, source_table_name, options = {})
+        raise NotImplementedError
+      end
+
+      def drop_table(temp_table_name)
+        quoted_table = connection.quote_table_name(temp_table_name)
+        connection.execute("DROP TABLE IF EXISTS #{quoted_table}")
+      end
+
+      def self.for(connection)
+        adapter_name = connection.adapter_name.downcase
+        case adapter_name
+        when /postgresql/
+          Postgresql.new(connection)
+        when /mysql/
+          Mysql.new(connection)
+        when /sqlite/
+          Sqlite.new(connection)
+        else
+          raise AdapterError, "Unsupported adapter: #{adapter_name}. StagingTable supports PostgreSQL, MySQL, and SQLite adapters."
+        end
+      end
+    end
+  end
+end

data/lib/staging_table/adapters/mysql.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module StagingTable
+  module Adapters
+    class Mysql < Base
+      def create_table(temp_table_name, source_table_name, options = {})
+        # MySQL's LIKE copies structure and indexes by default
+        quoted_temp = connection.quote_table_name(temp_table_name)
+        quoted_source = connection.quote_table_name(source_table_name)
+        connection.execute("CREATE TABLE #{quoted_temp} LIKE #{quoted_source}")
+      end
+    end
+  end
+end

data/lib/staging_table/adapters/postgresql.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module StagingTable
+  module Adapters
+    class Postgresql < Base
+      def create_table(temp_table_name, source_table_name, options = {})
+        quoted_temp = connection.quote_table_name(temp_table_name)
+        quoted_source = connection.quote_table_name(source_table_name)
+        sql = "CREATE TABLE #{quoted_temp} (LIKE #{quoted_source} INCLUDING DEFAULTS"
+        sql += " INCLUDING INDEXES" if options[:include_indexes]
+        sql += ")"
+        connection.execute(sql)
+      end
+    end
+  end
+end

data/lib/staging_table/adapters/sqlite.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module StagingTable
+  module Adapters
+    class Sqlite < Base
+      def create_table(temp_table_name, source_table_name, options = {})
+        quoted_temp = connection.quote_table_name(temp_table_name)
+        connection.quote_table_name(source_table_name)
+
+        # SQLite doesn't support CREATE TABLE ... LIKE, so we copy the structure
+        # by getting the original CREATE TABLE statement and modifying it
+        create_sql = connection.select_value(
+          "SELECT sql FROM sqlite_master WHERE type='table' AND name=#{connection.quote(source_table_name)}"
+        )
+
+        if create_sql.nil?
+          raise TableError, "Source table '#{source_table_name}' does not exist"
+        end
+
+        # Replace the table name in the CREATE TABLE statement
+        new_create_sql = create_sql.sub(/CREATE\s+TABLE\s+["'`]?#{Regexp.escape(source_table_name)}["'`]?/i, "CREATE TABLE #{quoted_temp}")
+
+        connection.execute(new_create_sql)
+
+        # Copy indexes if requested
+        if options[:include_indexes]
+          copy_indexes(temp_table_name, source_table_name)
+        end
+      end
+
+      private
+
+      def copy_indexes(temp_table_name, source_table_name)
+        indexes = connection.select_all(
+          "SELECT sql FROM sqlite_master WHERE type='index' AND tbl_name=#{connection.quote(source_table_name)} AND sql IS NOT NULL"
+        )
+
+        indexes.each do |row|
+          index_sql = row["sql"]
+          next if index_sql.nil?
+
+          # Replace the table name and generate a new index name
+          new_index_sql = index_sql.gsub(/\b#{Regexp.escape(source_table_name)}\b/, temp_table_name)
+          # Make index name unique by appending temp table name suffix
+          new_index_sql = new_index_sql.sub(/INDEX\s+["'`]?(\w+)["'`]?/i) do
+            "INDEX #{$1}_#{temp_table_name.split("_").last}"
+          end
+
+          connection.execute(new_index_sql)
+        end
+      end
+    end
+  end
+end