active_cursor 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: aa6257797955fe7432710d3282f7ec6b483ba7ec4b7ab594744ba651ab760299
+  data.tar.gz: 7584a37351f4f530adcc308ca173fcc64bc9051d1418d910177cfeb06f6ba5e9
+SHA512:
+  metadata.gz: 6880d4c1d207ccbbf5055f856850734f17953f948d06923e29bb8d5409de74e2c36deedb96888209c6dc017da940138d77cfc2ba159822c4ec3bdcd1b3417fed
+  data.tar.gz: 34b5f1209b38f6bd0dc58a271ab37866c8fc207000bd53a350f98153ac43bddc8ee74c959ab5c0f96eda5a7fe76bc190781794d1fd16de4cbb0f6c3d7656917b

data/.github/workflows/main.yml

@@ -0,0 +1,29 @@
+name: Ruby
+
+on: [push, pull_request]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:13-alpine
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3.0.0
+          bundler-cache: true
+      - name: Run the default task
+        run: bundle exec rake

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in active_cursor.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+gem "rspec", "~> 3.0"
+gem "pg"
+gem "sqlite3"
+gem "gruff"
+gem "memory_profiler"

data/Gemfile.lock

@@ -0,0 +1,66 @@
+PATH
+  remote: .
+  specs:
+    active_cursor (0.1.0)
+      activerecord (~> 6.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (6.1.4)
+      activesupport (= 6.1.4)
+    activerecord (6.1.4)
+      activemodel (= 6.1.4)
+      activesupport (= 6.1.4)
+    activesupport (6.1.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    concurrent-ruby (1.1.9)
+    diff-lcs (1.4.4)
+    gruff (0.14.0)
+      histogram
+      rmagick
+    histogram (0.2.4.1)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    memory_profiler (1.0.0)
+    minitest (5.14.4)
+    pg (1.2.3)
+    rake (13.0.6)
+    rmagick (4.2.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    sqlite3 (1.4.2)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    zeitwerk (2.4.2)
+
+PLATFORMS
+  ruby
+  x86_64-darwin-20
+
+DEPENDENCIES
+  active_cursor!
+  gruff
+  memory_profiler
+  pg
+  rake (~> 13.0)
+  rspec (~> 3.0)
+  sqlite3
+
+BUNDLED WITH
+   2.2.16

data/README.md

@@ -0,0 +1,72 @@
+# ActiveCursor
+
+This gem adds support for cursors to Active Record. This library only supports
+PostgreSQL.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'active_cursor'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install active_cursor
+
+## Why?
+
+Sometimes you need to process a huge amount of data, but loading the entire
+dataset into memory isn't possible.
+
+In those cases, you'll usually reach for Active Record's `find_each` method, which
+will only load records in batches.
+
+Unfortunately, `find_each` requires that each record in the dataset has a unique,
+integer ID. That's not always possible. Enter cursors.
+
+## Usage
+
+Extend your `ApplicationRecord` with `ActiveCursor::QueryMethods`:
+
+```ruby
+class ApplicationRecord < ActiveRecord::Base
+  extend ActiveCursor::QueryMethods
+end
+```
+
+Now, you're ready to start cursing.
+
+```ruby
+User.cursor.each { |user| ... }
+User.cursor.each_row { |attributes| ... }
+User.select(:id, :name).cursor.each_tuple { |id, name| ... }
+```
+
+By default, this will load 1,000 records at a time from the database. You can
+change that by specifying the batch size:
+
+```ruby
+User.cursor(batch_size: 10).each { |user| ... }
+```
+
+All methods return enumerables when no block is given, so you can use the full power of Ruby's Enumerable:
+
+```ruby
+User.cursor.each.find { |user| user.name == "Rick" }
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/rzane/active_cursor.

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/active_cursor.gemspec

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "lib/active_cursor/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "active_cursor"
+  spec.version       = ActiveCursor::VERSION
+  spec.authors       = ["Ray Zane"]
+  spec.email         = ["raymondzane@gmail.com"]
+
+  spec.summary       = "Adds support for cursors to Active Record."
+  spec.description   = "Efficiently iterate through massive collections in your database."
+  spec.homepage      = "https://github.com/rzane/active_cursor"
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/rzane/active_cursor"
+  spec.metadata["changelog_uri"] = "https://github.com/rzane/active_cursor/releases"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activerecord", "~> 6.0"
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/bin/benchmark

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "gruff"
+require "benchmark"
+require "memory_profiler"
+require_relative "../spec/support/database"
+
+Database.create
+Database.connect
+Database.migrate
+
+COUNTS = (1..5).map { _1 * 200_000 }
+LABELS = (1..5).to_h { [_1 - 1, _1 * 2] }
+TIME = Hash.new { |h, k| h[k] = [] }
+OBJECTS = Hash.new { |h, k| h[k] = [] }
+
+def seed(count)
+  Widget.delete_all
+  Widget.connection.execute <<~SQL
+    INSERT INTO widgets (value, name, timestamp)
+    SELECT generate_series(0, #{count}), gen_random_uuid(), CURRENT_TIMESTAMP;
+  SQL
+end
+
+def measure(name, &block)
+  memory = MemoryProfiler.report do
+    TIME[name] << Benchmark.realtime(&block)
+  end
+
+  OBJECTS[name] << memory.total_allocated
+end
+
+def chart(data:, name:, unit:, filename:)
+  chart = Gruff::Bar.new
+  chart.title = name
+  chart.labels = LABELS
+  chart.y_axis_label = unit
+  chart.x_axis_label = "Records (x100,000)"
+
+  data.each do |name, values|
+    chart.data(name, values)
+  end
+
+  chart.write(filename)
+end
+
+COUNTS.each do |count|
+  puts count
+  seed(count)
+
+  puts "* to_a"
+  measure "to_a" do
+    Widget.all.each {}
+  end
+
+  puts "* pluck"
+  measure "pluck" do
+    Widget.pluck(:id, :value, :name, :timestamp)
+  end
+
+  puts "* find_each"
+  measure "find_each" do
+    Widget.find_each {}
+  end
+
+  puts "* cursor.each"
+  measure "cursor.each" do
+    Widget.cursor.each {}
+  end
+
+  puts "* cursor.each_row"
+  measure "cursor.each_row" do
+    Widget.cursor.each_row {}
+  end
+
+  puts "* cursor.each_tuple"
+  measure "cursor.each_tuple" do
+    Widget.cursor.each_tuple {}
+  end
+end
+
+chart(
+  data: SECONDS,
+  name: "Time",
+  unit: "Seconds",
+  filename: "assets/time.png"
+)
+chart(
+  data: OBJECTS,
+  name: "Allocated Objects",
+  unit: "Objects",
+  filename: "assets/objects.png"
+)

data/bin/console

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "active_cursor"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/active_cursor.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative "active_cursor/version"
+require "active_support/core_ext/module/delegation"
+
+class ActiveCursor
+  module QueryMethods
+    def cursor(...)
+      ActiveCursor.new(all, ...)
+    end
+  end
+
+  def initialize(relation, batch_size: 1_000)
+    @relation = relation
+    @batch_size = batch_size
+  end
+
+  def each(&block)
+    return enum_for(__method__) unless block_given?
+
+    iterate do |name|
+      records = model.find_by_sql("FETCH #{batch_size} FROM #{name}")
+      records.each(&block)
+      records.length
+    end
+  end
+
+  def each_row(&block)
+    return enum_for(__method__) unless block_given?
+
+    iterate do |name|
+      result = connection.execute("FETCH #{batch_size} FROM #{name}")
+      result.each(&block)
+      result.ntuples
+    end
+  end
+
+  def each_tuple(&block)
+    return enum_for(__method__) unless block_given?
+
+    iterate do |name|
+      result = connection.execute("FETCH #{batch_size} FROM #{name}")
+      result.each_row(&block)
+      result.ntuples
+    end
+  end
+
+  private
+
+  attr_reader :relation, :batch_size
+
+  delegate :model, :connection, to: :relation
+
+  def sql
+    connection.unprepared_statement { relation.to_sql }
+  end
+
+  def iterate
+    name = "cursor_#{SecureRandom.uuid.tr("-", "_")}"
+
+    connection.transaction do
+      connection.execute("DECLARE #{name} NO SCROLL CURSOR FOR #{sql}")
+
+      begin
+        loop until yield(name) < batch_size
+      ensure
+        connection.execute("CLOSE #{name}")
+      end
+    end
+  end
+end

data/lib/active_cursor/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class ActiveCursor
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification
+name: active_cursor
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Ray Zane
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2021-08-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+description: Efficiently iterate through massive collections in your database.
+email:
+- raymondzane@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/main.yml"
+- ".gitignore"
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- README.md
+- Rakefile
+- active_cursor.gemspec
+- assets/objects.png
+- assets/time.png
+- bin/benchmark
+- bin/console
+- bin/setup
+- lib/active_cursor.rb
+- lib/active_cursor/version.rb
+homepage: https://github.com/rzane/active_cursor
+licenses: []
+metadata:
+  homepage_uri: https://github.com/rzane/active_cursor
+  source_code_uri: https://github.com/rzane/active_cursor
+  changelog_uri: https://github.com/rzane/active_cursor/releases
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
+summary: Adds support for cursors to Active Record.
+test_files: []