light_record 0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 68b18b821e94f95075db330967d3cb14537ef4b3
+  data.tar.gz: 6463993d97d10f007faee6c14fda80c2efad8c1e
+SHA512:
+  metadata.gz: fc8ebbe93aa163cfc93d1368f3cbfa6c90fa0893d9476336f908aea7f135c76e105515555b19226f2df6c856c3be86047625ad6a0039ebd4baa107d077fccbd4
+  data.tar.gz: 4735d36300f2747a7233e728d05dcd143cd6103b361ded30293351c6754015744c7894609b68147c54d00b94b669fbe710501bcc9da6ceb1aeac948f2e41f78b

data/lib/light_record.rb

@@ -0,0 +1,193 @@
+module LightRecord
+  extend self
+
+  # Create LightRecord class based on klass argument
+  # Used internally by scope#light_records and scope#light_records_each
+  #
+  # @param klass [Class] ActiveRecord model
+  # @param fields [Array] list of fields, will be used to define attribute methods
+  # @return new class base on klass argument but extended with LightRecord methods
+  #
+  def build_for_class(klass, fields)
+    new_klass = Class.new(klass) do
+      self.table_name = klass.table_name
+      self.inheritance_column = nil
+
+      extend LightRecord::RecordAttributes
+
+      define_fields(fields)
+
+      def initialize(data)
+        @data = data
+        @attributes = @data
+        @readonly = true
+      end
+
+      def read_attribute_before_type_cast(attr_name)
+        @data[attr_name.to_sym]
+      end
+
+      def self.subclass_from_attributes?(attrs)
+        false
+      end
+
+      def has_attribute?(attr_name)
+        @attributes.has_key?(attr_name.to_sym)
+      end
+
+      def read_attribute(attr_name)
+        @attributes[attr_name.to_sym]
+      end
+
+      def _read_attribute(attr_name)
+        @attributes.fetch(attr_name.to_sym) { |n| yield n if block_given? }
+      end
+
+      def [](attr_name)
+        @attributes[attr_name.to_sym]
+      end
+    
+      def attributes
+        @attributes
+      end
+
+      # to avoid errors when try saving data
+      def remember_transaction_record_state
+        @_start_transaction_state ||= {}
+        super
+      end
+    end
+
+    if klass.const_defined?(:LightRecord, false)
+      new_klass.send(:include, klass::LightRecord)
+    elsif klass.superclass.const_defined?(:LightRecord, false)
+      new_klass.send(:include, klass.superclass::LightRecord)
+    end
+
+    new_klass
+  end
+
+  # ActiveRecord extension for class methods
+  # Defines klass.define_fields
+  # Overrides klass.column_names and klass.define_attribute_methods
+  module RecordAttributes
+    def define_fields(fields)
+      @fields ||= []
+
+      fields.each do |field|
+        field = field.to_sym unless field.is_a?(Symbol)
+        @fields << field
+        define_method(field) do
+          @data[field]
+        end
+
+        # to avoid errors when try saving data
+        define_method("#{field}=") do |value|
+          @data[field] = value
+        end
+      end
+
+      # ActiveRecord make method :id refers to primary key, even there is no column "id"
+      if !fields.include?(:id) && !fields.include?("id") && primary_key.present?
+        define_method(:id) do
+          @data[self.class.primary_key.to_sym]
+        end
+      end
+    end
+
+    # used in Record#respond_to?
+    def define_attribute_methods
+    end
+
+    # Active record keep it as strings, but I keep it as symbols
+    def column_names
+      @fields.map(&:to_s)
+    end
+  end
+
+  # Create LightRecord class based on klass argument
+  def base_extended(klass)
+    @base_extended ||= {}
+    if @base_extended[klass]
+      return @base_extended[klass]
+    end
+
+    @base_extended[klass] = LightRecord.build_for_class(klass, klass.column_names)
+  end
+
+  module RelationMethods
+
+    # Executes query and return array of light object (model class extended by LightRecord)
+    def light_records(options = {})
+      client = connection.instance_variable_get(:@connection)
+      sql = self.to_sql
+
+      options = {
+        stream: false, symbolize_keys: true, cache_rows: false, as: :hash,
+        database_timezone: ActiveRecord::Base.default_timezone
+      }
+      result = _light_record_execute_query(connection, sql, options)
+
+      klass = LightRecord.build_for_class(self.klass, result.fields)
+
+      if options[:set_const]
+        self.klass.const_set(:"LR_#{Time.now.to_i}", klass)
+      end
+
+      need_symbolize_keys = defined?(PG::Result) && result.is_a?(PG::Result)
+
+      records = []
+      result.each do |row|
+        row.symbolize_keys! if need_symbolize_keys
+        records << klass.new(row)
+      end
+
+      return records
+    end
+
+    # Same as `#light_records` but iterates through result set and call block for each object
+    # this uses less memroy because it creates objects one-by-one
+    # it uses stream feature of mysql client
+    def light_records_each(options = {})
+      conn = ActiveRecord::Base.connection_pool.checkout
+      sql = self.to_sql
+
+      options = {
+        stream: true, symbolize_keys: true, cache_rows: false, as: :hash,
+        database_timezone: ActiveRecord::Base.default_timezone
+      }
+      result = _light_record_execute_query(conn, sql, options)
+
+      klass = LightRecord.build_for_class(self.klass, result.fields)
+
+      if options[:set_const]
+        self.klass.const_set(:"LR_#{Time.now.to_i}", klass)
+      end
+
+      need_symbolize_keys = defined?(PG::Result) && result.is_a?(PG::Result)
+
+      result.each do |row|
+        row.symbolize_keys! if need_symbolize_keys
+        yield klass.new(row)
+      end
+    ensure
+      ActiveRecord::Base.connection_pool.checkin(conn)
+    end
+
+    private def _light_record_execute_query(connection, sql, options)
+      client = connection.instance_variable_get(:@connection)
+
+      if client.class.to_s == 'PG::Connection'
+        connection.execute(sql, "LightRecord - #{self.klass}")
+      else
+        connection.send(:log, sql, "LightRecord - #{self.klass}") do
+          client.query(sql, options)
+        end
+      end
+    end
+
+  end
+
+end
+
+ActiveRecord::Relation.send(:include, LightRecord::RelationMethods)

data/spec/light_record_spec.rb

@@ -0,0 +1,117 @@
+require_relative './test_helper'
+require_relative './prepare_db'
+
+#ActiveRecord::Base.logger = Logger.new(STDOUT)
+
+class ARQuestion < ActiveRecord::Base
+  self.table_name =  "sample"
+  self.primary_key = "policy_id"
+end
+
+class ARQuestion_wLR < ActiveRecord::Base
+  self.table_name =  "sample"
+  self.primary_key = "policy_id"
+
+  module LightRecord
+    def light_included?
+      true
+    end
+  end
+end
+
+describe "LightRecord" do
+
+  it "should use different connection with #light_records_each" do
+    # Should not raise error
+    iterated = false
+    ARQuestion.all.light_records_each do |sample|
+      iterated = true
+      ARQuestion.first
+      break
+    end
+
+    assert(iterated)
+  end
+
+  it "should create class from AR object" do
+    klass = LightRecord.base_extended(ARQuestion)
+    assert(klass < ARQuestion)
+
+    assert_equal(LightRecord.base_extended(ARQuestion).object_id, klass.object_id)
+
+    assert_equal(klass.column_names, ARQuestion.column_names)
+  end
+
+  it "should be serializable" do
+    records = ARQuestion.limit(10).light_records
+    assert_equal(records.to_json, ARQuestion.limit(10).to_a.to_json)
+  end
+
+  it "should be serializable" do
+    light = ARQuestion.limit(1).light_records.first
+    dark  = ARQuestion.find(light.id)
+
+    assert_equal(light.attributes, dark.attributes.symbolize_keys)
+  end
+
+  it "should include LightRecord submodule if present" do
+    record = ARQuestion_wLR.limit(1).light_records.first
+    assert(record.light_included?)
+
+    klass = LightRecord.base_extended(ARQuestion_wLR)
+    assert_includes(klass.ancestors, ARQuestion_wLR::LightRecord)
+  end
+
+  it "should work with #respond_to?" do
+    rand_sql_fn = ENV['DB'] == 'postgres' ? "random" : "rand"
+    light = ARQuestion.select("*, #{rand_sql_fn}() as extra_column_from_sql").limit(1).light_records.first
+
+    assert_respond_to(light, :extra_column_from_sql)
+    ARQuestion.column_names.each do |column_name|
+      assert_respond_to(light, column_name)
+    end
+  end
+
+  it "sgould have primary key" do
+    light = ARQuestion.limit(1).light_records.first
+    dark  = ARQuestion.find(light.id)
+
+    assert_equal(light.id, light[ARQuestion.primary_key])
+    assert_equal(light.id, dark.id)
+  end
+
+  it "should be read-only" do
+    record = ARQuestion_wLR.limit(1).light_records.first
+
+    assert(record.readonly?)
+
+    assert_raises(ActiveRecord::ReadOnlyRecord) do
+      record.update_attributes(ARQuestion.column_names.first => "bla bla lba")
+    end
+  end
+
+  it "should be not as a new record" do
+    record = ARQuestion_wLR.limit(1).light_records.first
+    refute(record.new_record?)
+  end
+
+  it "should add extra params in query" do
+    record = ARQuestion_wLR.where(policy_id: 119736).light_records.first
+    assert_equal(record.id, 119736)
+  end
+
+  if ActiveRecord::Base.connection.is_a?(ActiveRecord::ConnectionAdapters::Mysql2Adapter)
+    it "should select datetime in UTC" do
+      # Active Record will set :database_timezone on first query
+      # But if no query run on that connection then it will be blank
+      # This code is simulating that behaviour
+      ActiveRecord::Base.connection_pool.connections.each do |conn|
+        client = ActiveRecord::Base.connection.instance_variable_get(:@connection)
+        client.query_options.delete(:database_timezone)
+      end
+
+      record = ARQuestion_wLR.select("now() as time").light_records.first
+      assert(record.time.gmt?, "Time is not UTC")
+    end
+  end
+end

data/spec/prepare_db.rb

@@ -0,0 +1,132 @@
+require "mysql2"
+require "active_record"
+
+if ENV['DB'] == 'postgres'
+  ActiveRecord::Base.establish_connection(
+    host: '127.0.0.1',
+    adapter: 'postgresql',
+    encoding: 'unicode',
+    database: 'light_record',
+    pool: 5,
+    username: ENV['DB_USER'] || ENV["USER"],
+    password: '',
+    reconnect: true
+  )
+else
+  ActiveRecord::Base.establish_connection(
+    adapter: 'mysql2',
+    database: 'light_record',
+    host: 'localhost',
+    username: 'root',
+    password: '',
+    pool: 5
+  )
+end
+
+require 'active_record/connection_adapters/mysql2_adapter'
+require 'active_record/connection_adapters/postgresql_adapter'
+
+module TestDB
+
+  SAMPLE_TABLE = 'sample'
+
+  def self.init
+    db = ActiveRecord::Base.connection
+
+    if db.data_source_exists?(SAMPLE_TABLE)
+      puts "Testing table already exists"
+      return
+    end
+
+    if db.is_a? ActiveRecord::ConnectionAdapters::Mysql2Adapter
+      db.execute(%{
+        CREATE TABLE `#{SAMPLE_TABLE}` (
+          `policy_id` int(11) NOT NULL,
+          `statecode` varchar(255) DEFAULT NULL,
+          `county` varchar(255) DEFAULT NULL,
+          `eq_site_limit` varchar(255) DEFAULT NULL,
+          `hu_site_limit` varchar(255) DEFAULT NULL,
+          `fl_site_limit` varchar(255) DEFAULT NULL,
+          `fr_site_limit` varchar(255) DEFAULT NULL,
+          `tiv_2011` varchar(255) DEFAULT NULL,
+          `tiv_2012` varchar(255) DEFAULT NULL,
+          `eq_site_deductible` float DEFAULT NULL,
+          `hu_site_deductible` varchar(255) DEFAULT NULL,
+          `fl_site_deductible` int(11) DEFAULT NULL,
+          `fr_site_deductible` int(11) DEFAULT NULL,
+          `point_latitude` varchar(255) DEFAULT NULL,
+          `point_longitude` varchar(255) DEFAULT NULL,
+          `line` varchar(255) DEFAULT NULL,
+          `construction` varchar(255) DEFAULT NULL,
+          `point_granularity` int(11) DEFAULT NULL,
+          PRIMARY KEY (`policy_id`)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
+      })
+    elsif db.is_a? ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
+      db.execute(%{
+        CREATE TABLE #{SAMPLE_TABLE} (
+          policy_id integer NULL,
+          statecode varchar(255) NULL,
+          county varchar(255) NULL,
+          eq_site_limit varchar(255) NULL,
+          hu_site_limit varchar(255) NULL,
+          fl_site_limit varchar(255) NULL,
+          fr_site_limit varchar(255) NULL,
+          tiv_2011 varchar(255) NULL,
+          tiv_2012 varchar(255) NULL,
+          eq_site_deductible real NULL,
+          hu_site_deductible varchar(255) NULL,
+          fl_site_deductible real NULL,
+          fr_site_deductible real NULL,
+          point_latitude varchar(255) NULL,
+          point_longitude varchar(255) NULL,
+          line varchar(255) DEFAULT NULL,
+          construction varchar(255) NULL,
+          point_granularity real NULL,
+          PRIMARY KEY(policy_id)
+        );
+      })
+    end
+
+    import_csv(db)
+
+  ensure
+    #db.execute(%{drop table #{SAMPLE_TABLE}})
+  end
+
+  def self.import_csv(db)
+    require 'csv'
+
+    index = 0
+    headers = nil
+    rows_to_insert = []
+
+    puts "Importing testing data"
+
+    CSV.foreach(File.dirname(__FILE__) + "/../test_data/FL_insurance_sample.csv", headers: true) do |row|
+      headers ||= row.headers
+
+      row_values = []
+      headers.size.times do |i|
+        row_values << %{'#{db.quote_string(row.field(i))}'}
+      end
+
+      rows_to_insert << row_values
+
+      if index % 3000 == 0 && index > 0 || index == 36633
+        puts "Row #{index}"
+
+        values = rows_to_insert.map {|r| '(' + r.join(", ") + ')' }.join(', ')
+
+        db.execute(%{
+          insert into #{SAMPLE_TABLE} (#{headers.join(", ")}) values #{values}
+        })
+        rows_to_insert.clear
+      end
+
+      index += 1
+    end
+  end
+end
+
+TestDB.init

data/spec/test_helper.rb

@@ -0,0 +1,17 @@
+require 'bundler/setup'
+require "minitest/reporters"
+
+Minitest::Reporters.use!(
+  Minitest::Reporters::DefaultReporter.new(color: true)
+)
+
+def MiniTest.filter_backtrace(bt)
+  bt
+end
+
+require "mysql2"
+require "active_record"
+require 'light_record'
+require 'looksee'
+
+require "minitest/autorun"

metadata

@@ -0,0 +1,51 @@
+--- !ruby/object:Gem::Specification
+name: light_record
+version: !ruby/object:Gem::Version
+  version: '0.2'
+platform: ruby
+authors:
+- Pavel Evstigneev
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-10-03 00:00:00.000000000 Z
+dependencies: []
+description: 
+email:
+- pavel.evst@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/light_record.rb
+- spec/light_record_spec.rb
+- spec/prepare_db.rb
+- spec/test_helper.rb
+homepage: http://github.com/paxa/light_record
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 3
+summary: Getting process memory
+test_files:
+- spec/light_record_spec.rb
+- spec/prepare_db.rb
+- spec/test_helper.rb