bitfields 0.1.5 → 0.2.0

data/README.markdown

@@ -13,8 +13,8 @@ e.g. true-false-false = 1, false-true-false = 2,  true-false-true = 5 (1,2,4,8,.
 
  - records changes `user.chamges == {:seller => [false, true]}`
  - adds scopes `User.seller.stupid.first` (deactivate with `bitfield ..., :scopes => false`)
- - builds sql `User.bitfield_sql(:insane => true, :stupid => false) == 'users.my_bits IN (2, 3)'` (2 and 1+2)
- - builds not-index-using sql with `bitfield ... ,:query_mode => :bit_operator` and `User.bitfield_sql(:insane => true, :stupid => false) == '(users.my_bits & 3) = 1'`, always slower than IN() sql, since it will not use an existing index (tested for up to 64 values)
+ - builds sql `User.bitfield_sql(:insane => true, :stupid => false) == '(users.my_bits & 3) = 1'`
+ - builds index-using sql with `bitfield ... ,:query_mode => :in_list` and `User.bitfield_sql(:insane => true, :stupid => false) == 'users.my_bits IN (2, 3)'` (2 and 1+2), often slower than :bit_operator sql especially for high number of bits
  - builds update sql `User.set_bitfield_sql(:insane => true, :stupid => false) == 'my_bits = (my_bits | 6) - 4'`
  - **faster sql than any other bitfield lib** through combination of multiple bits into a single sql statement
  - gives access to bits `User.bitfields[:my_bits][:stupid] == 4`
@@ -22,7 +22,7 @@ e.g. true-false-false = 1, false-true-false = 2,  true-false-true = 5 (1,2,4,8,.
 Install
 =======
 As Gem: ` sudo gem install bitfields `  
-Or as Rails plugin: ` script/plugin install git://github.com/grosser/bitfields.git `
+Or as Rails plugin: ` rails plugin install git://github.com/grosser/bitfields.git `
 
 ### Migration
 ALWAYS set a default, bitfield queries will not work for NULL
@@ -38,6 +38,14 @@ Update all users
 Delete the shop when a user is no longer a seller
     before_save :delete_shop, :if => lambda{|u| u.changes['seller'] == [true, false]}
 
+TIPS
+====
+ - Never do: "#{bitfield_sql(...)} AND #{bitfield_sql(...)}", merge both into one hash
+ - bit_operator is faster in most cases, use :query_mode => :in_list sparingly
+ - standard mysql integer is 4 byte -> 32 bitfields
+
+![performance](http://chart.apis.google.com/chart?chtt=bit-operator+vs+IN+--+with+index&chd=s:CEGIKNPRUW,DEHJLOQSVX,CFHKMPSYXZ,DHJMPSVYbe,DHLPRVZbfi,FKOUZeinsx,FLQWbglqw2,HNTZfkqw19,BDEGHJLMOP,BDEGIKLNOQ,BDFGIKLNPQ,BDFGILMNPR,BDFHJKMOQR,BDFHJLMOQS,BDFHJLNPRT,BDFHJLNPRT&chxt=x,y&chxl=0:|100K|200K|300K|400K|500K|600K|700K|800K|900K|1000K|1:|0|1441.671ms&cht=lc&chs=600x500&chdl=2bits+%28in%29|3bits+%28in%29|4bits+%28in%29|6bits+%28in%29|8bits+%28in%29|10bits+%28in%29|12bits+%28in%29|14bits+%28in%29|2bits+%28bit%29|3bits+%28bit%29|4bits+%28bit%29|6bits+%28bit%29|8bits+%28bit%29|10bits+%28bit%29|12bits+%28bit%29|14bits+%28bit%29&chco=0000ff,0000ee,0000dd,0000cc,0000bb,0000aa,000099,000088,ff0000,ee0000,dd0000,cc0000,bb0000,aa0000,990000,880000)
+
 TODO
 ====
  - convenient named scope `User.with_bitfields(:xxx=>true, :yy=>false)`

data/VERSION

@@ -1 +1 @@
-0.1.5
+0.2.0

data/benchmark/{1.rb → bit_operator_vs_in.rb}

@@ -1,5 +1,5 @@
-bit_counts = [2,3,4,5,6,7,8,9,10,11,12,13,14]
-record_counts = (1..20).to_a.map{|i| i * 50_000 }
+bit_counts = [2,3,4,6,8,10,12,14]
+record_counts = (1..10).to_a.map{|i| i * 100_000 }
 use_index = true
 database = ARGV[0]
 
@@ -90,15 +90,6 @@ def test_speed(bit_counts, query_mode)
   Hash[result]
 end
 
-#determines a unique color based on a name
-def color_for(name)
-  name = name.inspect + "some randomness for fun"
-  hash = (name.hash % 1_000_000_000).to_s #get a hash of a constant size
-  colors = [hash[0..1], hash[2..3], hash[4..5]].map{|c| c.to_f / 100.0 * 16} #use 3 parts of the hash to get numbers from 0 to 15.99
-  palette = ('0'..'9').to_a + ('a'..'f').to_a #hex values 0..f
-  colors.map{|c| palette[c.floor].to_s * 2} * '' #each color is duplicated and the joined
-end
-
 connect(database)
 create_model_table(bit_counts, use_index)
 class User < ActiveRecord::Base
@@ -108,10 +99,10 @@ create_model_fields(bit_counts)
 puts "creating test data"
 last = 0
 
+# collect graph data
 graphs = {:bit => {}, :in => {}}
-data = record_counts.map do |record_count|
+record_counts.each do |record_count|
   create(bit_counts, record_count-last)
-  puts User.count
   last = record_count
 
   puts "testing with #{record_count} records -- bit_operator"
@@ -119,26 +110,26 @@ data = record_counts.map do |record_count|
 
   puts "testing with #{record_count} records -- in_list"
   graphs[:in][record_count] = test_speed(bit_counts, :in_list)
-
 end
 
-colors = {:bit => '00xx00', :in => 'xx0000'}
+# print them
+colors = {:bit => 'xx0000', :in => '0000xx'}
 alpha_num = (('0'..'9').to_a + ('a'..'f').to_a).reverse
-title = "bit-operator vs IN() -- #{use_index ? 'with' : 'without'} index"
+title = "bit-operator vs IN -- #{use_index ? 'with' : 'without'} index"
 url = GoogleChart::LineChart.new('600x500', title, false) do |line|
-  max = 0
+  max_y = 0
   graphs.each do |type, line_data|
     bit_counts.each do |bit_count|
       data = record_counts.map{|rc| line_data[rc][bit_count] }
       name = "#{bit_count}bits (#{type})"
-      color = colors[type].sub('xx', alpha_num[bit_count]*2)
+      color = colors[type].sub('xx', alpha_num[bit_counts.index(bit_count)]*2)
       line.data(name, data, color)
-      max = [data.max, max].max
+      max_y = [data.max, max_y].max
     end
   end
 
   line.axis :x, :labels => record_counts.map{|c|"#{c/1000}K"}
-  line.axis :y, :labels => ['0', "%.3f" % [max*100]]
+  line.axis :y, :labels => ['0', "%.3fms" % [max_y*1000]]
 end.to_url
 
 puts url

data/bitfields.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{bitfields}
-  s.version = "0.1.5"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Michael Grosser"]
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
     "README.markdown",
     "Rakefile",
     "VERSION",
-    "benchmark/1.rb",
+    "benchmark/bit_operator_vs_in.rb",
     "bitfields.gemspec",
     "lib/bitfields.rb",
     "spec/bitfields_spec.rb",

data/lib/bitfields.rb

@@ -71,7 +71,7 @@ module Bitfields
     private
 
     def bitfield_sql_by_column(column, bit_values, options={})
-      mode = options[:query_mode] || (bitfield_options[column][:query_mode] || :in_list)
+      mode = options[:query_mode] || (bitfield_options[column][:query_mode] || :bit_operator)
       case mode
       when :in_list then
         max = (bitfields[column].values.max * 2) - 1

data/spec/bitfields_spec.rb

@@ -150,23 +150,23 @@ describe Bitfields do
 
   describe :bitfield_sql do
     it "includes true states" do
-      User.bitfield_sql(:insane => true).should == 'users.bits IN (2,3,6,7)' # 2, 1+2, 2+4, 1+2+4
+      User.bitfield_sql({:insane => true}, :query_mode => :in_list).should == 'users.bits IN (2,3,6,7)' # 2, 1+2, 2+4, 1+2+4
     end
 
     it "includes invalid states" do
-      User.bitfield_sql(:insane => false).should == 'users.bits IN (0,1,4,5)' # 0, 1, 4, 4+1
+      User.bitfield_sql({:insane => false}, :query_mode => :in_list).should == 'users.bits IN (0,1,4,5)' # 0, 1, 4, 4+1
     end
 
     it "can combine multiple fields" do
-      User.bitfield_sql(:seller => true, :insane => true).should == 'users.bits IN (3,7)' # 1+2, 1+2+4
+      User.bitfield_sql({:seller => true, :insane => true}, :query_mode => :in_list).should == 'users.bits IN (3,7)' # 1+2, 1+2+4
     end
 
     it "can combine multiple fields with different values" do
-      User.bitfield_sql(:seller => true, :insane => false).should == 'users.bits IN (1,5)' # 1, 1+4
+      User.bitfield_sql({:seller => true, :insane => false}, :query_mode => :in_list).should == 'users.bits IN (1,5)' # 1, 1+4
     end
 
     it "combines multiple columns into one sql" do
-      sql = MultiBitUser.bitfield_sql(:seller => true, :insane => false, :one => true, :four => true)
+      sql = MultiBitUser.bitfield_sql({:seller => true, :insane => false, :one => true, :four => true}, :query_mode => :in_list)
       sql.should == 'users.bits IN (1,5) AND users.more_bits IN (5,7)' # 1, 1+4 AND 1+4, 1+2+4
     end
 
@@ -174,7 +174,7 @@ describe Bitfields do
       u1 = MultiBitUser.create!(:seller => true, :one => true)
       u2 = MultiBitUser.create!(:seller => true, :one => false)
       u3 = MultiBitUser.create!(:seller => false, :one => false)
-      MultiBitUser.all(:conditions => MultiBitUser.bitfield_sql(:seller => true, :one => false)).should == [u2]
+      MultiBitUser.all(:conditions => MultiBitUser.bitfield_sql({:seller => true, :one => false}, :query_mode => :in_list)).should == [u2]
     end
 
     describe 'with bit operator mode' do

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: bitfields
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 23
   prerelease: 
   segments: 
   - 0
-  - 1
-  - 5
-  version: 0.1.5
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Michael Grosser
@@ -31,7 +31,7 @@ files:
 - README.markdown
 - Rakefile
 - VERSION
-- benchmark/1.rb
+- benchmark/bit_operator_vs_in.rb
 - bitfields.gemspec
 - lib/bitfields.rb
 - spec/bitfields_spec.rb