sqlreport 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 521ae617f946f8878d3ec77d64198cdb9abe65567c8c752b731661ff62112550
-  data.tar.gz: 6115f8c4a9c3032ba983464ddd1ec8eafa853583d7dc71c0f8dce0a95c0b9192
+  metadata.gz: e9ab7d553ecd33126c38418100399abab18686a28faba11e0e4ac0d46491c01e
+  data.tar.gz: 7b77b6af58c083760be6ce350903ad58cf9ea7742084aa464a6e64173ca0a8e6
 SHA512:
-  metadata.gz: 77a80f7b76808e6d6496549709bf2e697ee7b983a08533c7515fc1918d24a68eb002fe0e95bdb1c8079a36897bf958d1fe3f8aea055db411b8656bb025e3ef85
-  data.tar.gz: 85d22692915240b6fcf6b4049d81a72a33770542e631feec663f25fd6850b51d19cffbe70621d802b8d2b666eeb2481169e3dcf28d291d787bf5249d0b03d42f
+  metadata.gz: fe9b20a35baad6c7a39fd0d5b598aea299bb6f9e8c661b1972c6ed46cd0bb45676ebdc66ee1e8fff9ad32f0af643d35cd25e0072b896a632b94d0bf51c2483e6
+  data.tar.gz: ff031385aa5b1e84973e668cda73b2ef705d736485c500f7366d07268db8b286891fcd637a3b136162f17782b0fa257256c14cbd664e997e524c0863ebff718f

data/CHANGELOG.md

@@ -1,4 +1,9 @@
-## [Unreleased]
+## [0.1.2]
+
+- Added batch processing for large datasets
+- Added streaming CSV export for memory-efficient processing
+- Added progress tracking for batch operations
+- Added ActiveRecord integration for direct model-to-CSV conversion
 
 ## [0.1.1] - 2025-03-09
 

data/README.md

@@ -10,6 +10,12 @@ This gem provides an easy way to convert SQL database queries to CSV. Below you
   - [Write result to CSV file](#write-result-to-csv-file)
   - [Get columns / headers](#get-column-headers)
   - [Get rows](#get-rows)
+- [Batch Processing](#batch-processing)
+  - [Creating a Batch Manager](#creating-a-batch-manager)
+  - [Processing Batches](#processing-batches)
+  - [Streaming to CSV](#streaming-to-csv)
+  - [Tracking Progress](#tracking-progress)
+- [ActiveRecord Integration](#activerecord-integration)
 - [Compatibility](#compatibility)
 - [Todo](#todo)
 - [Contributing](#contributing)
@@ -94,13 +100,88 @@ This gem is tested with the following Ruby versions on Linux and Mac OS X:
 
 - Ruby > 2.2.2
 
+## Batch Processing
+
+For handling large datasets, SQLreport provides batch processing capabilities that allow you to process data in chunks to avoid memory issues.
+
+### Creating a Batch Manager
+
+```ruby
+batch_manager = Sqlreport.batch_query("SELECT * FROM large_table", batch_size: 1000)
+```
+
+### Processing Batches
+
+You can process one batch at a time:
+
+```ruby
+# Get the next batch
+batch = batch_manager.next_batch
+# Process the batch
+batch.rows.each do |row|
+  # Process each row
+end
+```
+
+Or process all batches at once with a block:
+
+```ruby
+batch_manager.process_all do |batch|
+  # Process each batch
+  puts "Processing batch with #{batch.rows.count} rows"
+end
+```
+
+### Streaming to CSV
+
+For very large datasets, you can stream directly to a CSV file without loading all data into memory:
+
+```ruby
+batch_manager.stream_to_csv("large_table.csv")
+```
+
+### Tracking Progress
+
+You can track the progress of batch processing:
+
+```ruby
+batch_manager.count_total_rows # Get total row count for progress calculation
+batch_manager.next_batch
+puts "Processed #{batch_manager.processed_rows} of #{batch_manager.total_rows} rows"
+puts "Progress: #{batch_manager.progress_percentage}%"
+```
+
+## ActiveRecord Integration
+
+SQLreport can be used directly with ActiveRecord models and relations, allowing for a more fluent interface:
+
+```ruby
+# Generate a report from an ActiveRecord relation
+User.where(active: true).sqlreport.result.write_csv("active_users.csv")
+
+# Or with more options
+Post.where(published: true)
+    .order(created_at: :desc)
+    .limit(100)
+    .sqlreport
+    .result
+    .to_csv(include_headers: true, separator: ",")
+
+# Use batch processing with ActiveRecord
+User.where(created_at: 1.month.ago..Time.current)
+    .sqlreport_batch(batch_size: 500)
+    .stream_to_csv("new_users.csv")
+```
+
+This integration makes it easy to generate reports directly from your models without having to write raw SQL.
+
 ## Todo
 
-- Batch jobs
+- ~~Add simple safeguard validations~~
+- ~~Allow it to use different databases~~
+- ~~Batch jobs (for bigger tables)~~
+- ~~Tie into Rails models~~
 - Add support for multiple export options (PDF, textfile, LaTex)
-- Add safeguard validations
-- Tie into Rails models
-- Allow it to use different databases
 - ..
 
 ## Contributing

data/examples/batch_operations.rb

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "sqlreport"
+require "active_record"
+
+# Set up a test database
+ActiveRecord::Base.establish_connection(
+  adapter: "sqlite3",
+  database: ":memory:"
+)
+
+# Create a test table
+ActiveRecord::Base.connection.create_table(:test_table, force: true) do |t|
+  t.column :user_id, :integer
+  t.column :name, :string
+  t.column :json_object, :json
+end
+
+# Insert test data
+10.times do |i|
+  ActiveRecord::Base.connection.execute("INSERT INTO test_table (user_id, name) VALUES (#{i}, 'test name #{i}');")
+end
+
+# Create a model for the test table
+class TestTable < ActiveRecord::Base
+  self.table_name = "test_table" # Use the singular table name
+end
+
+puts "=== Basic Query ==="
+result = Sqlreport.query("SELECT * FROM test_table").result
+puts "Columns: #{result.columns.join(", ")}"
+puts "Rows: #{result.rows.count}"
+puts "CSV: #{result.to_csv.lines.first}"
+
+puts "\n=== Batch Query ==="
+batch_manager = Sqlreport.batch_query("SELECT * FROM test_table", batch_size: 3)
+batch_manager.count_total_rows
+puts "Total rows: #{batch_manager.total_rows}"
+
+batch = batch_manager.next_batch
+puts "First batch rows: #{batch.rows.count}"
+puts "Processed rows: #{batch_manager.processed_rows}"
+puts "Progress: #{batch_manager.progress_percentage}%"
+
+batch = batch_manager.next_batch
+puts "Second batch rows: #{batch.rows.count}"
+puts "Processed rows: #{batch_manager.processed_rows}"
+puts "Progress: #{batch_manager.progress_percentage}%"
+
+puts "\n=== ActiveRecord Integration ==="
+# The extensions are automatically included in ActiveRecord::Relation
+# by the ActiveSupport.on_load(:active_record) hook
+
+# Test the sqlreport method
+result = TestTable.where(user_id: 1..5).sqlreport.result
+puts "ActiveRecord query rows: #{result.rows.count}"
+
+# Test the sqlreport_batch method
+batch_manager = TestTable.where(user_id: 1..5).sqlreport_batch(batch_size: 2)
+batch = batch_manager.next_batch
+puts "ActiveRecord batch rows: #{batch.rows.count}"
+puts "Processed rows: #{batch_manager.processed_rows}"
+
+puts "\n=== CSV Export ==="
+# Write to CSV
+csv_file = "test_table.csv"
+result.write_csv(csv_file)
+puts "CSV file created: #{File.exist?(csv_file)}"
+puts "CSV content: #{File.read(csv_file).lines.first}"
+FileUtils.rm_f(csv_file)
+
+# Stream to CSV
+batch_csv_file = "batch_test_table.csv"
+batch_manager = TestTable.where(user_id: 1..5).sqlreport_batch(batch_size: 3)
+batch_manager.stream_to_csv(batch_csv_file)
+puts "Batch CSV file created: #{File.exist?(batch_csv_file)}"
+puts "Batch CSV content: #{File.read(batch_csv_file).lines.first}"
+FileUtils.rm_f(batch_csv_file)
+
+puts "\nAll examples completed successfully!"

data/lib/sqlreport/active_record_extension.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Sqlreport
+  # ActiveRecordExtension
+  # Extends ActiveRecord::Relation with SQLReport functionality
+  module ActiveRecordExtension
+    # Provides SQLReport functionality to ActiveRecord models
+    module RelationMethods
+      # Convert the relation to a SQLReport result
+      def sqlreport
+        # Get the SQL query from the relation
+        sql = to_sql
+
+        # Create a SQLReport result from the query
+        ::Sqlreport::Result.new(sql)
+      end
+    end
+
+    # Provides batch processing functionality to ActiveRecord models
+    module BatchMethods
+      # Convert the relation to a SQLReport batch manager
+      def sqlreport_batch(batch_size: 1000)
+        # Get the SQL query from the relation
+        sql = to_sql
+
+        # Create a SQLReport batch manager from the query
+        ::Sqlreport::BatchManager.new(sql, batch_size: batch_size)
+      end
+    end
+  end
+end
+
+# Extend ActiveRecord::Relation with SQLReport functionality
+ActiveSupport.on_load(:active_record) do
+  ActiveRecord::Relation.include(Sqlreport::ActiveRecordExtension::RelationMethods)
+  ActiveRecord::Relation.include(Sqlreport::ActiveRecordExtension::BatchMethods)
+end

data/lib/sqlreport/base.rb

@@ -7,4 +7,29 @@ module Sqlreport
   def self.query(sql_query)
     ::Sqlreport::Result.new(sql_query)
   end
+
+  def self.batch_query(sql_query, batch_size: 1000)
+    ::Sqlreport::BatchManager.new(sql_query, batch_size: batch_size)
+  end
+
+  def self.database(db_config)
+    # Return a DatabaseConnector instance
+    DatabaseConnector.new(db_config)
+  end
+
+  # DatabaseConnector
+  # Handles database connection switching
+  class DatabaseConnector
+    def initialize(db_config)
+      @db_config = db_config
+    end
+
+    def query(sql_query)
+      ::Sqlreport::Result.new(sql_query, db_config: @db_config)
+    end
+
+    def batch_query(sql_query, batch_size: 1000)
+      ::Sqlreport::BatchManager.new(sql_query, batch_size: batch_size, db_config: @db_config)
+    end
+  end
 end

data/lib/sqlreport/batch_manager.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Sqlreport
+  # BatchManager
+  # Handles batch processing of SQL queries for large datasets
+  class BatchManager
+    attr_reader :total_rows, :processed_rows, :batch_size, :current_offset
+
+    def initialize(query, batch_size: 1000, db_config: false)
+      @query = query
+      @batch_size = batch_size
+      @db_config = db_config
+      @total_rows = nil
+      @processed_rows = 0
+      @current_offset = 0
+      @complete = false
+    end
+
+    def connection
+      @connection ||= if @db_config
+                        ActiveRecord::Base.establish_connection(@db_config)
+                      else
+                        ActiveRecord::Base.connection
+                      end
+    end
+
+    def count_total_rows
+      return @total_rows if @total_rows
+
+      # Extract the FROM clause and any WHERE conditions to count total rows
+      from_clause = @query.match(/FROM\s+([^\s;]+)(\s+WHERE\s+(.+?))?(\s+ORDER BY|\s+GROUP BY|\s+LIMIT|\s*;|\s*$)/i)
+      return nil unless from_clause
+
+      table = from_clause[1]
+      where_clause = from_clause[3]
+
+      count_query = "SELECT COUNT(*) FROM #{table}"
+      count_query += " WHERE #{where_clause}" if where_clause
+
+      result = connection.exec_query(count_query)
+      @total_rows = result.rows.first.first.to_i
+    end
+
+    def next_batch
+      return nil if @complete
+
+      # Add LIMIT and OFFSET to the query
+      paginated_query = if @query.include?("LIMIT")
+                          # If query already has LIMIT, we need to handle differently
+                          raise "Batch processing not supported for queries with LIMIT clause"
+                        else
+                          "#{@query.chomp(";")} LIMIT #{@batch_size} OFFSET #{@current_offset}"
+                        end
+
+      result = Sqlreport::Result.new(paginated_query, db_config: @db_config).result
+
+      # Update state
+      rows_in_batch = result.rows.count
+      @processed_rows += rows_in_batch
+      @current_offset += @batch_size
+      @complete = rows_in_batch < @batch_size
+
+      result
+    end
+
+    def process_all
+      results = []
+      while (batch = next_batch)
+        results << batch
+        yield batch if block_given?
+      end
+      results
+    end
+
+    def stream_to_csv(path, include_headers: true, separator: ",", quote_char: '"')
+      first_batch = true
+
+      File.open(path, "w") do |file|
+        process_all do |batch|
+          if first_batch && include_headers
+            headers = batch.columns.join(separator)
+            file.puts headers
+            first_batch = false
+          elsif first_batch
+            first_batch = false
+          end
+
+          # Write rows without loading all into memory
+          batch.rows.each do |row|
+            csv_row = CSV.generate_line(row, col_sep: separator, quote_char: quote_char)
+            file.write(csv_row)
+          end
+        end
+      end
+
+      true
+    end
+
+    def progress_percentage
+      return 0 unless @total_rows && @total_rows.positive?
+
+      (@processed_rows.to_f / @total_rows * 100).round(2)
+    end
+  end
+end

data/lib/sqlreport/result.rb

@@ -1,17 +1,30 @@
 # frozen_string_literal: true
 
 require "csv"
+require "yaml"
 
 module Sqlreport
   # Result
   class Result
-    def initialize(query)
+    def initialize(query, db_config: false)
       @query = query
-      @connection = ActiveRecord::Base.connection
+      @db_config = db_config
       @response = nil
     end
 
-    def result
+    def connection
+      @connection ||= if @db_config
+                        ActiveRecord::Base.establish_connection(@db_config)
+                      else
+                        ActiveRecord::Base.connection
+                      end
+    end
+
+    def result(validate: true)
+      connection
+      validations = validate_input if validate
+      return validations if validations
+
       @response = @connection.exec_query(@query)
       self
     end
@@ -36,5 +49,14 @@ module Sqlreport
       File.write(path, data)
       true
     end
+
+    private
+
+    def validate_input
+      return "DELETE, UPDATE, DROP, RENAME, ALTER cannot be used in queries" \
+        if %w[DELETE UPDATE DROP RENAME ALTER].any? { |needle| @query.include? needle }
+
+      false
+    end
   end
 end

data/lib/sqlreport/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sqlreport
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/sqlreport.rb

@@ -6,4 +6,6 @@ require "active_support/lazy_load_hooks"
 ActiveSupport.on_load(:active_record) do
   require "sqlreport/base"
   require "sqlreport/result"
+  require "sqlreport/batch_manager"
+  require "sqlreport/active_record_extension"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sqlreport
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Gem shards
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-03-15 00:00:00.000000000 Z
+date: 2025-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -79,8 +79,11 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/batch_operations.rb
 - lib/sqlreport.rb
+- lib/sqlreport/active_record_extension.rb
 - lib/sqlreport/base.rb
+- lib/sqlreport/batch_manager.rb
 - lib/sqlreport/result.rb
 - lib/sqlreport/version.rb
 - sig/sqlreport.rbs