surus 0.3.2 → 0.4.0

data/Guardfile

@@ -1,7 +1,7 @@
 # A sample Guardfile
 # More info at https://github.com/guard/guard#readme
 
-guard 'rspec', :version => 2 do
+guard 'rspec' do
   watch(%r{^spec/.+_spec\.rb$})
   watch(%r{^lib/surus/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
   watch('spec/spec_helper.rb')  { "spec" }

data/README.md

@@ -7,14 +7,15 @@ Surus accelerates ActiveRecord with PostgreSQL specific types and
 functionality. It enables indexed searching of serialized arrays and hashes.
 It also can control PostgreSQL synchronous commit behavior. By relaxing
 PostgreSQL's durability guarantee, transaction commit rate can be increased by
-50% or more. 
+50% or more. It also directly generate JSON in PostgreSQL which can be
+substantially faster than converting ActiveRecord objects to JSON.
 
 # Installation
 
     gem install surus
-    
+
 Or add to your Gemfile.
-    
+
     gem 'surus'
 
 # Hstore
@@ -25,10 +26,10 @@ type that can be indexed for fast searching.
     class User < ActiveRecord::Base
       serialize :properties, Surus::Hstore::Serializer.new
     end
-    
+
     User.create :properties => { :favorite_color => "green", :results_per_page => 20 }
     User.create :properties => { :favorite_colors => ["green", "blue", "red"] }
-    
+
 Even though the underlying hstore can only use strings for keys and values
 (and NULL for values) Surus can successfully maintain type for integers,
 floats, bigdecimals, dates, and any value that YAML can serialize. It does
@@ -46,32 +47,32 @@ Hstores can be searched with helper scopes.
     User.hstore_has_key(:properties, "favorite_color")
     User.hstore_has_all_keys(:properties, "favorite_color", "gender")
     User.hstore_has_any_keys(:properties, "favorite_color", "favorite_artist")
-    
+
 Hstore is a PostgreSQL extension. You can generate a migration to install it.
 
     rails g surus:hstore:install
     rake db:migrate
-    
-    
+
+
 Read more in the [PostgreSQL hstore documentation](http://www.postgresql.org/docs/9.1/static/hstore.html).
-    
+
 # Array
 
 Ruby arrays can be serialized to PostgreSQL arrays. Surus includes support
 for text, integer, float, and decimal arrays.
 
     class User < ActiveRecord::Base
-      serialize :permissions, Surus::Array::TextSerializer.new
+
       serialize :favorite_integers, Surus::Array::IntegerSerializer.new
       serialize :favorite_floats, Surus::Array::FloatSerializer.new
       serialize :favorite_decimals, Surus::Array::DecimalSerializer.new
     end
-    
+
     User.create :permissions => %w{ read_notes write_notes, manage_topics },
       :favorite_integers => [1, 2, 3],
       :favorite_floats => [1.3, 2.2, 3.1],
       :favorite_decimals => [BigDecimal("3.14"), BigDecimal("4.23"]
-    
+
 Arrays can be searched with helper scopes.
 
     User.array_has(:permissions, "admin")
@@ -92,15 +93,32 @@ entire session or per transaction.
       User.synchronous_commit false
       @user.save
     end # This transaction can return before the data is written to the drive
-    
+
     # synchronous_commit returns to its former value outside of the transaction
     User.synchronous_commit # -> true
-    
+
     # synchronous_commit can be turned off permanently
     User.synchronous_commit false
-    
+
 Read more in the [PostgreSQL asynchronous commit documentation](http://www.postgresql.org/docs/9.1/interactive/wal-async-commit.html).
 
+# JSON
+
+PostgreSQL 9.2 added `row_to_json` and `array_to_json` functions. These
+functions can be used to build JSON very quickly. Unfortunately, they are
+somewhat cumbersome to use. The `find_json` and `all_json` methods are easy to
+use wrappers around the lower level PostgreSQL functions that closely mimic
+the Rails `to_json` interface.
+
+    User.find_json 1
+    User.find_json 1, columns: [:id, :name, :email]
+    Post.find_json 1, include: :author
+    User.find_json(user.id, include: {posts: {columns: [:id, :subject]}})
+    User.all_json
+    User.where(admin: true).all_json
+    User.all_json(columns: [:id, :name, :email], include: {posts: {columns: [:id, :subject]}})
+    Post.all_json(include: [:forum, :post])
+
 # Benchmarks
 
 Using PostgreSQL's hstore enables searches to be done quickly in the database.
@@ -126,6 +144,18 @@ Arrays are also searchable.
                    user     system      total        real
     Surus      0.120000   0.040000   0.160000 (  0.531735)
 
+JSON generation is with all_json and find_json is substantially faster than to_json.
+
+    jack@hk-47~/dev/surus$ ruby -I lib -I bench bench/json_generation.rb
+    Generating test data... Done.
+                                                                  user     system      total        real
+    find_json: 1 record 500 times                             0.140000   0.010000   0.150000 (  0.205195)
+    to_json:   1 record 500 times                             0.240000   0.010000   0.250000 (  0.287435)
+    find_json: 1 record with 3 associations 500 times         0.480000   0.010000   0.490000 (  0.796025)
+    to_json:   1 record with 3 associations 500 times         1.130000   0.050000   1.180000 (  1.500837)
+    all_json:  50 records with 3 associations 20 times        0.030000   0.000000   0.030000 (  0.090454)
+    to_json:   50 records with 3 associations 20 times        1.350000   0.020000   1.370000 (  1.710151)
+
 Disabling synchronous commit can improve commit speed by 50% or more.
 
     jack@moya:~/work/surus$ ruby -I lib -I bench bench/synchronous_commit.rb
@@ -146,12 +176,12 @@ Disabling synchronous commit can improve commit speed by 50% or more.
     disabled per transaction         0.970000   0.850000   1.820000 (  2.478290)
     enabled / single transaction     0.890000   0.500000   1.390000 (  1.693629)
     disabled / single transaction    0.820000   0.450000   1.270000 (  1.554767)
-    
+
 Many more benchmarks are in the bench directory. Most accept parameters to
 adjust the amount of test data.
-    
+
 ## Running the benchmarks
-    
+
 1. Create a database
 2. Configure bench/database.yml to connect to it.
 3. Load bench/database_structure.sql into your bench database.
@@ -159,7 +189,7 @@ adjust the amount of test data.
    the include paths for lib and bench)
 
 
-    
+
 # License
 
 MIT

data/bench/benchmark_helper.rb

@@ -16,17 +16,17 @@ end
 
 class EavMasterRecord < ActiveRecord::Base
   has_many :eav_detail_records
-  
+
   def properties
     @properties ||= eav_detail_records.each_with_object({}) do |d, hash|
       hash[d.key] = d.value
     end
   end
-  
+
   def properties=(value)
     @properties = value
   end
-  
+
   after_save :persist_properties
   def persist_properties
     eav_detail_records.clear
@@ -54,6 +54,24 @@ end
 class NarrowRecord < ActiveRecord::Base
 end
 
+class User < ActiveRecord::Base
+  has_many :posts, foreign_key: :author_id
+end
+
+class Forum < ActiveRecord::Base
+  has_many :posts
+end
+
+class Post < ActiveRecord::Base
+  belongs_to :forum
+  belongs_to :author, class_name: 'User'
+  has_and_belongs_to_many :tags
+end
+
+class Tag < ActiveRecord::Base
+  has_and_belongs_to_many :posts
+end
+
 def clean_database
   YamlKeyValueRecord.delete_all
   SurusKeyValueRecord.delete_all
@@ -62,4 +80,8 @@ def clean_database
   YamlArrayRecord.delete_all
   WideRecord.delete_all
   NarrowRecord.delete_all
+  Post.destroy_all # destroy instead of delete so it removes join records in posts_tags
+  Forum.delete_all
+  User.delete_all
+  Tag.delete_all
 end

data/bench/database_structure.sql

@@ -90,3 +90,58 @@ CREATE TABLE narrow_records(
 );
 
 CREATE INDEX ON narrow_records (a);
+
+
+
+DROP TABLE IF EXISTS users CASCADE;
+
+CREATE TABLE users(
+  id serial PRIMARY KEY,
+  name varchar NOT NULL,
+  email varchar NOT NULL
+);
+
+
+
+DROP TABLE IF EXISTS forums CASCADE;
+
+CREATE TABLE forums(
+  id serial PRIMARY KEY,
+  name varchar NOT NULL
+);
+
+
+
+DROP TABLE IF EXISTS posts CASCADE;
+
+CREATE TABLE posts(
+  id serial PRIMARY KEY,
+  forum_id integer NOT NULL REFERENCES forums,
+  author_id integer NOT NULL REFERENCES users,
+  subject varchar NOT NULL,
+  body varchar NOT NULL
+);
+
+CREATE INDEX ON posts(forum_id);
+CREATE INDEX ON posts(author_id);
+
+
+
+DROP TABLE IF EXISTS tags CASCADE;
+
+CREATE TABLE tags(
+  id serial PRIMARY KEY,
+  name varchar NOT NULL UNIQUE
+);
+
+
+
+DROP TABLE IF EXISTS posts_tags CASCADE;
+
+CREATE TABLE posts_tags(
+  post_id integer NOT NULL REFERENCES posts,
+  tag_id integer NOT NULL REFERENCES tags,
+  PRIMARY KEY (post_id, tag_id)
+);
+
+CREATE INDEX ON posts_tags(tag_id);

data/bench/json_generation.rb

@@ -0,0 +1,61 @@
+require 'benchmark_helper'
+require 'faker'
+require 'factory_girl'
+FactoryGirl.find_definitions
+
+print "Generating test data... "
+
+clean_database
+
+Forum.transaction do
+  forums = FactoryGirl.create_list :forum, 4
+  users = FactoryGirl.create_list :user, 20
+  tags = FactoryGirl.create_list :tag, 30
+  50.times do
+    FactoryGirl.create :post, forum: forums.sample, author: users.sample, tags: tags.sample(rand(0..5))
+  end
+end
+
+puts "Done."
+
+num_short_iterations = 500
+num_long_iterations = 20
+
+Benchmark.bm(55) do |x|
+  first_post = Post.first
+  x.report("find_json: 1 record #{num_short_iterations} times") do
+    num_short_iterations.times do
+      Post.find_json first_post.id
+    end
+  end
+
+  x.report("to_json:   1 record #{num_short_iterations} times") do
+    num_short_iterations.times do
+      Post.find(first_post.id).to_json
+    end
+  end
+
+  x.report("find_json: 1 record with 3 associations #{num_short_iterations} times") do
+    num_short_iterations.times do
+      Post.find_json first_post.id, include: [:author, :forum, :tags]
+    end
+  end
+
+  x.report("to_json:   1 record with 3 associations #{num_short_iterations} times") do
+    num_short_iterations.times do
+      Post.includes(:author, :forum).find(first_post.id).to_json include: [:author, :forum, :tags]
+    end
+  end
+
+  x.report("all_json:  50 records with 3 associations #{num_long_iterations} times") do
+    num_long_iterations.times do
+      Post.all_json include: [:author, :forum, :tags]
+    end
+  end
+
+  x.report("to_json:   50 records with 3 associations #{num_long_iterations} times") do
+    num_long_iterations.times do
+      Post.includes(:author, :forum).all.to_json include: [:author, :forum, :tags]
+    end
+  end
+end

data/lib/surus.rb

@@ -9,3 +9,11 @@ require 'surus/array/decimal_serializer'
 require 'surus/array/scope'
 require 'surus/synchronous_commit/connection'
 require 'surus/synchronous_commit/model'
+require 'surus/json/query'
+require 'surus/json/row_query'
+require 'surus/json/array_agg_query'
+require 'surus/json/model'
+require 'surus/json/association_scope_builder'
+require 'surus/json/belongs_to_scope_builder'
+require 'surus/json/has_many_scope_builder'
+require 'surus/json/has_and_belongs_to_many_scope_builder'

data/lib/surus/json/array_agg_query.rb

@@ -0,0 +1,9 @@
+module Surus
+  module JSON
+    class ArrayAggQuery < Query
+      def to_sql
+        "select array_to_json(coalesce(array_agg(row_to_json(t)), '{}')) from (#{subquery_sql}) t"
+      end
+    end
+  end
+end

data/lib/surus/json/association_scope_builder.rb

@@ -0,0 +1,28 @@
+module Surus
+  module JSON
+    class AssociationScopeBuilder
+      attr_reader :outside_scope
+      attr_reader :association
+
+      def initialize(outside_scope, association)
+        @outside_scope = outside_scope
+        @association = association
+      end
+
+      def outside_class
+        @outside_scope.klass
+      end
+
+      delegate :connection, to: :outside_class
+      delegate :quote_table_name, :quote_column_name, to: :connection
+
+      def conditions
+        association.options[:conditions]
+      end
+
+      def order
+        association.options[:order]
+      end
+    end
+  end
+end

data/lib/surus/json/belongs_to_scope_builder.rb

@@ -0,0 +1,13 @@
+module Surus
+  module JSON
+    class BelongsToScopeBuilder < AssociationScopeBuilder
+      def scope
+        association_scope = association
+          .klass
+          .where("#{quote_column_name association.active_record_primary_key}=#{quote_column_name association.foreign_key}")
+        association_scope = association_scope.where(conditions) if conditions
+        association_scope
+      end
+    end
+  end
+end

data/lib/surus/json/has_and_belongs_to_many_scope_builder.rb

@@ -0,0 +1,36 @@
+module Surus
+  module JSON
+    class HasAndBelongsToManyScopeBuilder < AssociationScopeBuilder
+      def scope
+        association_scope = association
+          .klass
+          .joins("JOIN #{join_table} ON #{join_table}.#{association_foreign_key}=#{association_table}.#{association_primary_key}")
+          .where("#{outside_class.quoted_table_name}.#{association_primary_key}=#{join_table}.#{foreign_key}")
+        association_scope = association_scope.where(conditions) if conditions
+        association_scope = association_scope.order(order) if order
+        association_scope
+      end
+
+      def join_table
+        connection.quote_table_name association.options[:join_table]
+      end
+
+      def association_foreign_key
+        connection.quote_column_name association.association_foreign_key
+      end
+
+      def foreign_key
+        connection.quote_column_name association.foreign_key
+      end
+
+      def association_table
+        connection.quote_table_name association.klass.table_name
+      end
+
+      def association_primary_key
+        connection.quote_column_name association.association_primary_key
+      end
+
+    end
+  end
+end

data/lib/surus/json/has_many_scope_builder.rb

@@ -0,0 +1,22 @@
+module Surus
+  module JSON
+    class HasManyScopeBuilder < AssociationScopeBuilder
+      def scope
+        association_scope = association
+          .klass
+          .where("#{outside_primary_key}=#{association_foreign_key}")
+        association_scope = association_scope.where(conditions) if conditions
+        association_scope = association_scope.order(order) if order
+        association_scope
+      end
+
+      def outside_primary_key
+        "#{outside_class.quoted_table_name}.#{connection.quote_column_name association.active_record_primary_key}"
+      end
+
+      def association_foreign_key
+        "#{connection.quote_column_name association.foreign_key}"
+      end
+    end
+  end
+end