created_id 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 6aea8a225fcdd235ca56719e20de06dbaa6d4c143dc6d2658e68d95fc59c6d5b
+  data.tar.gz: 656d93bb31b73415cbeeba237e2c868bb936cb26942e4b03c1cc778e911cf825
+SHA512:
+  metadata.gz: 85cce04a40800a43b39754ad7686341ba940a1e6fae1e5de90a34dbc6f3c83f346208486d8fbd084dd00b4ff8541df54cd7ef073f894175522fb1bd56e7a0cf3
+  data.tar.gz: 42c29b10692f1dbfbae7af08b075d1eca605c06ca450d78024794a4965c76cc20bf2d3c0117d7cee324f2f1eee2976fc27261d6dc89ac86ab20961d5ee0b6319

data/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## 1.0.0
+
+### Added
+
+- Initial release.

data/MIT_LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2023 Brian Durand
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,150 @@
+# Created ID
+
+[![Continuous Integration](https://github.com/bdurand/created_id/actions/workflows/continuous_integration.yml/badge.svg)](https://github.com/bdurand/created_id/actions/workflows/continuous_integration.yml)
+[![Ruby Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://github.com/testdouble/standard)
+
+The gem is designed to optimize queries for ActiveRecord models that filter by the `created_at` timestamp. It can make queries more efficient by pre-calculating the ranges of id's for specific dates.
+
+The use case this code is designed to solve is when you have a large table with an auto-populated `created_at` column where you want to run queries that filter on that column. In most cases, simply adding an index on the `created_at` column will work just fine., However, once you start constructing more complex queries or adding joins and your table grows very large, the index can become less effective and not even be used at all.
+
+For instance, suppose you have a `Task` model backed by these tables:
+
+```ruby
+create_table :tasks do |t|
+  t.string :status, index: true
+  t.bigint, :user_id, index: true
+  t.datetime :created_at, index: true
+  t.string :description
+end
+
+create_table :users do |t|
+  t.string :name
+  t.string :group_name, index: true
+end
+
+class Task < ApplicationRecord
+  belongs_to :user
+end
+
+class User < ApplicationRecord
+  has_many :tasks
+end
+```
+
+And now suppose you want to count the tasks completed by users in the "public" group within the last day:
+
+```ruby
+Task.joins(:users)
+  .where(status: "completed", users: { group_name: "public" })
+  .where(created_at: [24.hours.ago...Time.current])
+  .count
+```
+
+This will construct a SQL query like this:
+
+```sql
+SELECT COUNT(*)
+FROM tasks
+INNER JOIN users ON users.id = tasks.user_id
+WHERE tasks.status = 'completed'
+  AND users.group_name = 'public'
+  AND tasks.created_at >= ?
+  AND tasks.created_at < ?
+```
+
+The query optimizer will have it's choice of several indexes to use to figure out the best query plan. The most important choice will be the first step of the plan to reduce the number of rows that the query needs to look at. Depending on the shape of your data, the query optimizer may decide to simply filter by `status` or `user_id` and then perform a table scan on all the rows to filter by `created_at`, not using the index on that column at all.
+
+This gem solves for this case by keeping track of the range ids created in each hour in a separate table. When you query on the `created_at` column, it will then look up the possible id range and add that to the query, so the SQL becomes:
+
+```sql
+SELECT COUNT(*)
+FROM tasks
+INNER JOIN users ON users.id = tasks.user_id
+WHERE tasks.status = 'completed'
+  AND users.group_name = 'public'
+  AND tasks.created_at >= ?
+  AND tasks.created_at < ?
+  AND tasks.id >= ?
+  AND tasks.id < ?
+```
+
+Because the `id` column is the primary key, it will always be indexed and the query optimizer will generally make better decisions about how to filter the query rows. You won't even need the index on `created_at` since the primay key would always be preferred.
+
+Another good use case is if you have some periodic tasks to calculate daily stats for some large tables. You will be able to make these queries more efficient without having to add an index on the `created_at` column that's only used on one query per day.
+
+## Usage
+
+Run the generator to create the database table
+
+```
+  rails created_id_engine:install:migrations
+```
+
+Next, include the `CreatedId` module into your models. Note that any model you wish to include this module in must have a numeric primary key.  If the model is subclassed you will need to include the `CreatedId` module in the parent model.
+
+```ruby
+class Task < ApplicationRecord
+  include CreatedId
+
+  belongs_to :user
+end
+```
+
+Now when you want to query by a range on the `created_at` column, you can use the `created_after`, `created_before`, or `created_between` scopes on the model.
+
+```ruby
+Task.where(status: "completed").created_after(24.hours.ago)
+
+Task.where(user_id: 1000).created_before(7.days.ago)
+
+Task.created_between(25.hour.ago, 24.hours.ago)
+```
+
+You'll then need to set up a periodic task to store the id ranges for your models. For each model that includes `CreatedId`, you need to run the `index_ids_for` once per hour. This task should be run shortly after the top of the hour.
+
+```ruby
+Task.index_ids_for(1.hour.ago)
+```
+
+Finally, you'll need to run a script to calculate the id ranges for all of your existing data.
+
+```ruby
+first_time = Task.first.created_at.utc
+time = Time.utc(first_time.year, first_time.month, first_time.day, first_time.hour)
+while time < Time.now
+  Task.index_ids_for(time)
+  time += 3600
+end
+```
+
+Don't worry if the id range for a specific hour does not get recorded, the queries will still work and they can be re-calcuated at any time. Queries will just be a bit less efficient if the ranges don't exist because queries will be given a large span of ids to filter on.
+
+There is an additional requirement for using this gem that you do not change the `created_at` value after a row is inserted since this can mess up the assumption about the correlation between ids and `created_at` timestamps. An error will be thrown if you try to change a record's timestamp after the id range has been created. The query logic can handle small variations between id order and timestamp order (i.e. if id 1000 has a timestamp a few seconds after id 1001).
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "created_id"
+```
+
+And then execute:
+```bash
+$ bundle
+```
+
+Or install it yourself as:
+```bash
+$ gem install created_id
+```
+
+## Contributing
+
+Open a pull request on GitHub.
+
+Please use the [standardrb](https://github.com/testdouble/standard) syntax and lint your code with `standardrb --fix` before submitting.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/created_id.gemspec

@@ -0,0 +1,34 @@
+Gem::Specification.new do |spec|
+  spec.name = "created_id"
+  spec.version = File.read(File.expand_path("../VERSION", __FILE__)).strip
+  spec.authors = ["Brian Durand"]
+  spec.email = ["bbdurand@gmail.com"]
+
+  spec.summary = "Mechanism for optimizing ActiveRecord queries against the created_at column on tables."
+  spec.homepage = "https://github.com/bdurand/created_id"
+  spec.license = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  ignore_files = %w[
+    .
+    Appraisals
+    Gemfile
+    Gemfile.lock
+    Rakefile
+    bin/
+    gemfiles/
+    spec/
+  ]
+  spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| ignore_files.any? { |path| f.start_with?(path) } }
+  end
+
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activerecord", ">= 5.0"
+
+  spec.add_development_dependency "bundler"
+
+  spec.required_ruby_version = ">= 2.5"
+end

data/db/migrate/20230403140000_create_created_ids.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class CreateCreatedIds < ActiveRecord::Migration[5.0]
+  def up
+    create_table :created_ids do |t|
+      t.string :class_name, null: false, limit: 100
+      t.datetime :hour, null: false
+      t.bigint :min_id, null: false
+      t.bigint :max_id, null: false
+    end
+
+    add_index :created_ids, [:class_name, :hour], unique: true
+  end
+end

data/lib/created_id/engine.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module CreatedId
+  class Engine < Rails::Engine
+    config.before_eager_load do
+      require_relative "id_range"
+    end
+  end
+end

data/lib/created_id/id_range.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module CreatedId
+  # This model stores the id ranges for other models by the hour. It is not meant to be
+  # accessed directly.
+  class IdRange < ActiveRecord::Base
+    self.table_name = "created_ids"
+
+    scope :for_class, ->(klass) { where(class_name: klass.base_class.name) }
+    scope :created_before, ->(time) { where(arel_table[:hour].lteq(time)) }
+    scope :created_after, ->(time) { where(arel_table[:hour].gteq(time)) }
+
+    before_validation :set_hour
+
+    validates :class_name, presence: true, length: {maximum: 100}
+    validates :hour, presence: true
+    validates :min_id, presence: true, numericality: {only_integer: true, greater_than_or_equal_to: 0}
+    validates :max_id, presence: true, numericality: {only_integer: true, greater_than_or_equal_to: 0}
+    validates_uniqueness_of :hour, scope: :class_name
+
+    class << self
+      # Get the minimum id for a class created in a given hour.
+      #
+      # @param klass [Class] The class to get the minimum id for.
+      # @param time [Time] The hour to get the minimum id for.
+      # @return [Integer] The minimum id for the class created in the given hour.
+      def min_id(klass, time)
+        for_class(klass).created_before(time).order(hour: :desc).first&.min_id || 0
+      end
+
+      # Get the maximum id for a class created in a given hour.
+      #
+      # @param klass [Class] The class to get the maximum id for.
+      # @param time [Time] The hour to get the maximum id for.
+      # @return [Integer] The maximum id for the class created in the given hour.
+      def max_id(klass, time)
+        id = for_class(klass).created_after(CreatedId.coerce_hour(time)).order(hour: :asc).first&.max_id
+
+        unless id
+          col_limit = klass.columns.detect { |c| c.name == klass.primary_key }.limit
+          id = if col_limit && col_limit > 0
+            ((256**col_limit) / 2) - 1
+          else
+            klass.base_class.unscoped.maximum(:id).to_i
+          end
+        end
+
+        id
+      end
+
+      # Get the minimum and maximum ids for a model created in a given hour. This
+      # method is used in indexing the ranges.
+      #
+      # @param klass [Class] The class to get the id range for.
+      # @param time [Time] The hour to get the id range for.
+      # @return [Array<Integer>] The minimum and maximum ids for the class created in the given hour.
+      def id_range(klass, time)
+        klass = klass.base_class
+        hour = CreatedId.coerce_hour(time)
+        next_hour = hour + 3600
+        prev_hour = hour - 3600
+
+        finder = klass.unscoped.where(created_at: (hour...next_hour))
+
+        prev_id = CreatedId::IdRange.min_id(self, prev_hour)
+        if prev_id
+          finder = finder.where(klass.arel_table[:id].gt(prev_id)) if prev_id > 0
+
+          next_id = CreatedId::IdRange.min_id(self, next_hour + 3600)
+          if next_id
+            finder = finder.where(klass.arel_table[:id].lt(next_id)) if next_id > prev_id
+          end
+        end
+
+        [finder.minimum(:id), finder.maximum(:id)]
+      end
+
+      # Save the minimum and maximum ids for a class created in a given hour.
+      #
+      # @param klass [Class] The class to save the id range for.
+      # @param time [Time] The hour to save the id range for.
+      # @param min_id [Integer] The minimum id for the class created in the given hour.
+      # @param max_id [Integer] The maximum id for the class created in the given hour.
+      # @return [void]
+      def save_created_id(klass, time, min_id, max_id)
+        record = find_or_initialize_by(class_name: klass.base_class.name, hour: CreatedId.coerce_hour(time))
+        record.min_id = min_id
+        record.max_id = max_id
+        record.save!
+      end
+    end
+
+    private
+
+    def set_hour
+      self.hour = CreatedId.coerce_hour(hour) if hour && hour_changed?
+    end
+  end
+end

data/lib/created_id/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module CreatedId
+  VERSION = File.read(File.expand_path("../../VERSION", __dir__)).chomp.freeze
+end

data/lib/created_id.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require_relative "created_id/version"
+require_relative "created_id/engine" if defined?(Rails::Engine)
+
+module CreatedId
+  extend ActiveSupport::Concern
+  class CreatedAtChangedError < StandardError
+  end
+
+  class << self
+    # Coerce a time to the beginning of the hour in UTC.
+    def coerce_hour(time)
+      time = time.to_time.utc
+      Time.utc(time.year, time.month, time.day, time.hour)
+    end
+  end
+
+  included do
+    unless defined?(ActiveRecord) && self < ActiveRecord::Base
+      raise ArgmentError, "CreatedId can only be included in ActiveRecord models"
+    end
+
+    # Require here so we don't mess up loading the activerecord gem.
+    require_relative "created_id/id_range"
+
+    scope :created_after, ->(time) { where(arel_table[:created_at].gteq(time).and(arel_table[primary_key].gteq(CreatedId::IdRange.min_id(self, time)))) }
+    scope :created_before, ->(time) { where(arel_table[:created_at].lt(time).and(arel_table[primary_key].lteq(CreatedId::IdRange.max_id(self, time)))) }
+    scope :created_between, ->(time_1, time_2) { created_after(time_1).created_before(time_2) }
+
+    before_save :verify_created_at_created_id!, if: :created_at_changed?
+  end
+
+  class_methods do
+    # Index the id range for the records created in the given hour.
+    #
+    # @param time [Time] The hour to store the id range for. The value will be coerced to the beginning of the hour.
+    # @return [void]
+    def index_ids_for(time)
+      min_id, max_id = CreatedId::IdRange.id_range(self, time)
+      if min_id && max_id
+        CreatedId::IdRange.save_created_id(self, time, min_id, max_id)
+      end
+    end
+  end
+
+  private
+
+  # Verify that the created_at value is within the range of the created_ids for that time period.
+  #
+  # @return [void]
+  # @raise [CreatedId::CreatedAtChangedError] If the created_at value is outside the range of the created_ids for that time period.
+  def verify_created_at_created_id!
+    # This is the normal case where created at is set to the current time on insert.
+    return if id.nil? && created_at_was.nil?
+
+    new_hour = CreatedId.coerce_hour(created_at || Time.now)
+    range = CreatedId::IdRange.for_class(self.class).find_by(hour: new_hour)
+
+    if range && (id < range.min_id || id > range.max_id)
+      raise CreatedAtChangedError, "created_at cannot be changed outside of the range of the created_ids for that time period"
+    end
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: created_id
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Brian Durand
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-04-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description:
+email:
+- bbdurand@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- MIT_LICENSE.txt
+- README.md
+- VERSION
+- created_id.gemspec
+- db/migrate/20230403140000_create_created_ids.rb
+- lib/created_id.rb
+- lib/created_id/engine.rb
+- lib/created_id/id_range.rb
+- lib/created_id/version.rb
+homepage: https://github.com/bdurand/created_id
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.5'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.12
+signing_key:
+specification_version: 4
+summary: Mechanism for optimizing ActiveRecord queries against the created_at column
+  on tables.
+test_files: []