ar-ondemand 1.1.4 → 1.1.5

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OTQ1OGY0YzBjZGU4MjkzZTUxMzEzZTg1YzU2NjBiNzc2YmU1ZGQ5Yg==
+    NDhlMjQ2ZTkzNjRkMTc0MmJiMGM3Y2VkYjQ1NDcwZDVlYTA4ZGQzNQ==
   data.tar.gz: !binary |-
-    MjM1YmU4MmQ3YjA5NDhkOWNmYTI5NTkwMTRmMzhlMmYwODRlMTZlZg==
+    Mzk5ZDY1OTdmZjg5YzEzZmI2YTNkNDNhMGI0MTliYzQ2Nzc0ZjU1Mw==
 SHA512:
   metadata.gz: !binary |-
-    ZTRiODA1ZTM1ZDNiMjI2MzY3MjJhYjA3NmFjODVlZDQwZjE5M2NhZjA0NzBh
-    MzhmNTgwMGJkNWMzMTc2YTM1YWZhMjFjYzdmODNiMWMwYzExODE3YjVmODE3
-    NjlkOTQwZGExODZkMDFjODMyMWFkMjE3ODczYTI0MTY5NmJjMmU=
+    OTRlYjk5NzhhZjU4MmVkOGViMGFjOTkyYWY2MjQ2ZGUxMmE3ZTFlMzE1NWZm
+    YTQ1ZGFhZWI0NDZlYmUwMDUzYjA0OWIwMGJjMzM1OWRkMTcwNDliODIwZWVl
+    NzZhN2QzMDc1MWMwY2UxMWI0ZTNhNThjMTFhODBkZjk4MTVjZWI=
   data.tar.gz: !binary |-
-    NGQ4NmY3MzUyNmI4NTZlY2JmODZmM2VkZmMyOThkMDVmNjJjM2MwZDBlNjY3
-    MDdiOWI2YjFlOGY3ZTg0MDg2ZDk0MzQzNzBlYWQwMWU5NWY0NDM2MTgyZGU1
-    NGQ1MDcyZjY1ZjMyYTE3OTBjZTVmOGU5ZTg1MzIzZTliYjA5MDQ=
+    ZDRmOTExOTU0Y2E0ZmI0YjQ2YmUxZGVkOGVhMWU5YTlmNDU4N2FkM2JlYTVj
+    N2Q0MWU1ZDI2NzViNjY0YTU2ZTU5ZTE2NzAyMmEyNGMzZjE5NTZiNjAyNDZh
+    NmI1YjFkZWJiZDQ2YTdkNDY2OGQyMDNmOGQwMjYyYjk1YmI0ZjY=

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# 1.1.4
+
+## [misc]
+* Added `for_streaming`
+* Added `raw_results`
+* Refactoring of namespaces
+* By default include in ActiveRecord::Base and ActiveRecord::Relation

data/CONTRIBUTORS.md

@@ -0,0 +1 @@
+* Steve Frank <lardcanoe@gmail.com>

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ar-ondemand (1.1.4)
+    ar-ondemand (1.1.5)
       activerecord (>= 3.2)
       activesupport (>= 3.2)
 

data/LICENSE.md

@@ -0,0 +1,18 @@
+The MIT License (MIT)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,4 +1,144 @@
 # ar-ondemand
 
+The `ar-ondemand` gem adds functionality to ActiveRecord to help deal with AR's bloat.
+
 [![Gem Version](https://badge.fury.io/rb/ar-ondemand.svg)](http://badge.fury.io/rb/ar-ondemand)
 [![](https://ci.solanolabs.com:443/cloudhealthtech/ar-ondemand/badges/170027.png?badge_token=bd73a19d5421a68f29e22ad15ad080cbabc56ba7)](https://ci.solanolabs.com:443/cloudhealthtech/ar-ondemand/suites/170027)
+
+# Getting Started
+
+```
+require 'ar-ondemand'
+```
+
+Please note that this library has been written for our needs, and even though it has gotten significant usage in 
+production environments, it hasn't seen the myriad ways others use and abuse ActiveRecord, so please experiment
+locally first.
+
+It has been used with ActiveRecord 3.2, MRI 1.9.3, JRuby 1.7 and the MySQL adapter.
+
+# Functionality
+
+## on_demand
+
+This was the original impetus for the gem. The issue was that we had to compare ~500k records between the source 
+dataset and our database, and we had no idea which records were new, changed or deleted. We'd preload everything from
+the database, but due to ActiveRecord's massive bloat, we'd constantly run into OOM exceptions. To get around that,
+the concept of a [lightweight ActiveRecord object](https://github.com/CloudHealth/ar-ondemand/blob/master/lib/ar-ondemand/record.rb)
+was introduced that had the absolute bare minimum needed to handle comparing the source data with the database.
+
+With this new type of object, we could easily interate over the 500k records extremely quickly and determine what has
+changed. The on-demand aspect comes when `.save` is called. If changes were noticed, it 
+[secretly](https://github.com/CloudHealth/ar-ondemand/blob/master/lib/ar-ondemand/record.rb#L67) instantiates an actual
+ActiveRecord model so that all the real functionality you'd expect, such as validation and callbacks, occurs.
+
+### Usage
+
+```
+assets = Widget.on_demand :identifier, {customer_id: 1, account_id: 42}
+source.each do |dso|
+  w = assets[dso[:identifier]]
+  w.name = dso[:name] 
+  w.foo = dso[:foo] 
+  w.bar = dso[:bar]
+  ar_obj = w.save
+  # If new or changed, ar_obj will be an ActiveRecord instance, and ar_obj.id will now be set
+end
+```
+
+## for_reading
+
+How often do you just need to load a bunch of objects and read some properties? Pretty often, right? Now, have you ever
+looked at how much memory ActiveRecord itself is consuming, as well as how much extra time it takes to create all the
+instances of the objects compared to how long it took to extract from the database? It's bonkers! The `for_reading`
+method makes it easy to get access to ActiveRecord-like functionality at 100th the cost.
+
+### Usage
+
+Let's say you have some Widget's. Instead of:
+
+```
+Widget.where(customer_id: 1).each { |r| ... }
+```
+
+use `for_reading` and get a significant speed boost, and use far less memory:
+
+```
+Widget.where(customer_id: 1).for_reading.each { |r| ... }
+```
+
+The big limitation is that you can't use `.includes` so you have to be writing a query for just that class.
+
+### Batch Results
+
+```
+Widget.where(customer_id: 1).for_reading(batch_size: 50000).each { |b| b.each { |r| } }
+```
+
+### raw_results
+
+This is just a nice little helper to get the raw database results, which you'd get by calling `ActiveRecord::Base.connection.select_all`
+but for some reason you can't call that on a model.
+
+```
+ActiveRecord::Base.connection.select_all "select * from widgets"
+ActiveRecord::Base.connection.select_all "select * from widgets where customer_id = 1"
+ActiveRecord::Base.connection.select_all "select * from widgets where customer_id = 1 limit 100000"
+```
+
+use:
+
+```
+Widget.raw_results
+Widget.where(customer_id: 1).raw_results
+Widget.where(customer_id: 1).limit(100_000).raw_results
+```
+
+## for_streaming
+
+Another use case was needing to iterate through 30,000,000 records. The code that needed this data was not setup to work
+with batching, so the concept of streaming results was introduced. It simply uses an Enumerator to hide the fact that
+batching is actually happening behind the scenes. The downside is that Enumerator made it ~10% slower. If speed matters,
+change your code to use batching.
+
+### Usage
+
+```
+def run
+  get_objects.each do |r|
+    do_something r
+  end
+end
+
+def get_objects
+  Widget.where(customer_id: 1).for_streaming
+end
+```
+
+Additional usage:
+
+```
+Widget.where(customer_id: 1).for_streaming(batch_size: 100_000).each { |r| }
+Widget.where(customer_id: 1).for_streaming(for_reading: true).each { |r| }
+Widget.where(customer_id: 1).for_streaming(for_reading: true, batch_size: 1_000_000).each { |r| }
+```
+
+## delete_all_by_pk
+
+Deleting many records or even a few records in a massive table can be an expensive operation, and can even lock up
+your table during the duration of the delete, as well as perform a complete table scan. A common pattern to deal with 
+this is querying the table first to find the primary keys that meet the criteria and then doing a delete specifying
+the primary keys as the `where` condition. This function does that all for you.
+
+### Usage
+
+```
+Widget.delete_all_by_pk
+Widget.where(Widget[:customer_id].eq(1).and(Widget[:usage].gt(42))).delete_all_by_pk
+```
+
+If you know you could be deleting millions, then we recommend batching the deletes:
+
+```
+Widget.where(Widget[:customer_id].eq(1).and(Widget[:usage].gt(42))).delete_all_by_pk(batch_size: 250_000)
+```

data/lib/ar-ondemand/result.rb

@@ -60,7 +60,11 @@ module ActiveRecord
         # TODO: Is using HashWithIndifferentAccess[] more efficient?
         h = {}
         @col_indexes.each_pair do |k, v|
-          h[k] = @column_types[k].type_cast rec[v]
+          if @column_types[k]
+            h[k] = @column_types[k].type_cast rec[v]
+          else
+            h[k] = rec[v]
+          end
         end
         h.with_indifferent_access
       end

data/lib/ar-ondemand/version.rb

@@ -1,3 +1,3 @@
 module ArOnDemand
-  VERSION = '1.1.4'
+  VERSION = '1.1.5'
 end

data/spec/app/models/widget.rb

@@ -0,0 +1,3 @@
+class Widget < ActiveRecord::Base
+
+end

data/spec/db/schema.rb

@@ -1,11 +1,19 @@
 ActiveRecord::Schema.define do
-  create_table "audit_records", :force => true do |t|
-    t.integer  "customer_id"
-    t.integer  "model_id"
-    t.integer  "model_type_id"
-    t.string   "action",        :limit => 6
-    t.text     "description"
-    t.datetime "created_at",                 :null => false
-    t.datetime "updated_at",                 :null => false
+  create_table 'audit_records', force: true do |t|
+    t.integer  'customer_id'
+    t.integer  'model_id'
+    t.integer  'model_type_id'
+    t.string 'action', limit: 6
+    t.text     'description'
+    t.datetime 'created_at', null: false
+    t.datetime 'updated_at', null: false
+  end
+
+  create_table 'widgets', force: true do |t|
+    t.integer  'customer_id'
+    t.text     'identifier'
+    t.text     'description'
+    t.datetime 'created_at', null: false
+    t.datetime 'updated_at', null: false
   end
 end

data/spec/factories/widget.rb

@@ -0,0 +1,9 @@
+FactoryGirl.define do
+  sequence(:identifier) { |n| "w-#{n}" }
+
+  factory :widget do
+    customer_id 1
+    identifier
+    description { (0...50).map { ('a'..'z').to_a[rand(26)] }.join }
+  end
+end

data/spec/lib/on_demand_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+require 'ar-ondemand'
+
+describe 'OnDemand' do
+  before(:each) do
+    (1..25).each do
+      create(:widget)
+    end
+  end
+
+  context 'Existing' do
+    it 'should get asset' do
+      assets = ::Widget.where(customer_id: 1).on_demand :identifier, customer_id: 1
+      ids = assets.ids
+      expect(ids.length).to eql(25)
+
+      x = assets['w-1']
+      expect(x.identifier).to eql('w-1')
+      expect(x.id).to eql(1)
+      expect(x).to be_an_instance_of(::ActiveRecord::OnDemand::Record)
+    end
+  end
+
+  context 'Update Existing' do
+    it 'saving should get instance of a Widget' do
+      assets = ::Widget.where(customer_id: 1).on_demand :identifier, customer_id: 1
+      x = assets['w-26']
+      expect(x.identifier).to eql('w-26')
+      expect(x).to be_an_instance_of(::ActiveRecord::OnDemand::Record)
+
+      x.customer_id = 2
+      obj = x.save
+      expect(obj).to be_an_instance_of(::Widget)
+    end
+  end
+
+  context 'Create' do
+    it 'should create asset' do
+      assets = ::Widget.where(customer_id: 1).on_demand :identifier, customer_id: 1
+      x = assets['w-999']
+      expect(x.identifier).to eql('w-999')
+      expect(x.id).to eql(nil)
+      expect(x.persisted?).to eql(false)
+      expect(x).to be_an_instance_of(::Widget)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ar-ondemand
 version: !ruby/object:Gem::Version
-  version: 1.1.4
+  version: 1.1.5
 platform: ruby
 authors:
 - Steve Frank
@@ -48,8 +48,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG.md
+- CONTRIBUTORS.md
 - Gemfile
 - Gemfile.lock
+- LICENSE.md
 - README.md
 - ar-ondemand.gemspec
 - lib/ar-ondemand.rb
@@ -62,12 +65,15 @@ files:
 - lib/ar-ondemand/version.rb
 - solano.yml
 - spec/app/models/audit_record.rb
+- spec/app/models/widget.rb
 - spec/db/schema.rb
 - spec/db/seeds.rb
 - spec/factories/audit_record.rb
+- spec/factories/widget.rb
 - spec/lib/delete_all_by_pk_spec.rb
 - spec/lib/for_reading_spec.rb
 - spec/lib/for_streaming_spec.rb
+- spec/lib/on_demand_spec.rb
 - spec/spec_helper.rb
 - spec/support/active_record.rb
 homepage: https://github.com/CloudHealth/ar-ondemand