active_data_frame 0.1.8 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3b36c2b8ec3874ed2b924e3f53b7d9ee35adda6a
-  data.tar.gz: 6607e3ca1023d6bb28162c773b9852a13f1f2271
+SHA256:
+  metadata.gz: 4dc73f14e73f597961f8b8092c29090dd962efe8f8d5adc1f5e4550fbd3c4d57
+  data.tar.gz: e6c9e266e39898c063f7e0c02b94856970c64f73e80fbacccfca924b259562e7
 SHA512:
-  metadata.gz: 7e05379d9e8c5d91adc2c237dac5fdba7cc739a129bdb377119bbfcac9cfdc24ad5c4c5529d4a5c957d1e13366fb800971a0d5196bf7d8bf14286b3bd88fd259
-  data.tar.gz: 7fb1f15a7d9e66920e938f6a5ec9fcf11a56a4beacac2c85b8f208c10528a318ececc708130c82722bcdf23fd54335bb8e5cbcd9f10cee9c3c0ef60c4f95c51b
+  metadata.gz: 5aa335d4cd4f887212b9ee3e46a96ab172d682af1f35380bcc51f160b1938faa21f82472d7baad0c49f2c49477b8567a2e8c45b5a50cc8f11be8673a60495982
+  data.tar.gz: dd246279e53dbb9d41e9894c842c1b62f8d4f7c2e724f7db2ca058983d22c25ecf4c1e24fa13d4fa3582e4fa8f92a4466facdbeaee52f238146f891ec2c4227d

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Wouter Coppieters
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/active_data_frame.gemspec

@@ -20,16 +20,16 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.13"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "pry-byebug", "~> 3.4.0", '>= 3.4.0'
-  spec.add_development_dependency 'pry', '~> 0.10.2', '>= 0.10.0'
+  spec.add_development_dependency 'bundler', '~> 2.4'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'pry'
   spec.add_development_dependency 'pg'
   spec.add_development_dependency 'sqlite3'
   spec.add_development_dependency 'mysql2'
   spec.add_development_dependency 'minitest', '~>5.11'
   spec.add_development_dependency 'minitest-reporters', '~> 1.1', '>= 1.1.0'
   spec.add_development_dependency 'minitest-around', '0.4.1'
-  spec.add_runtime_dependency     'activerecord', '~> 5.0'
+  spec.add_runtime_dependency     'activerecord', '~> 7.0'
   spec.add_runtime_dependency     'rmatrix', '~> 0.1.20', '>=0.1.20'
 end

data/lib/active_data_frame/data_frame_proxy.rb

@@ -76,7 +76,7 @@ module ActiveDataFrame
     end
 
     def method_missing(name, *args, &block)
-      if name.to_s.ends_with?(?=)
+      if name.to_s.end_with?(?=)
         is_assignment = true
         name = name.to_s.gsub(/=$/,'').to_sym
       end
@@ -92,7 +92,7 @@ module ActiveDataFrame
       ranges.map do |range|
         case range
         when Range then range
-        when Fixnum then range..range
+        when Integer then range..range
         else raise "Unexpected index for data frame proxy #{range}, expecting either a Range or an Integer"
         end
       end

data/lib/active_data_frame/database.rb

@@ -16,7 +16,7 @@ module ActiveDataFrame
         unless sql.empty?
           ActiveRecord::Base.transaction do
             ActiveDataFrame::DataFrameProxy.suppress_logs do
-              case ActiveRecord::Base.connection_config[:adapter]
+              case ActiveRecord::Base.connection_db_config.adapter
               when 'sqlite3'.freeze
                 ActiveRecord::Base.connection.raw_connection.execute_batch sql
               when 'mysql2'
@@ -59,25 +59,56 @@ module ActiveDataFrame
       flush! unless self.batching
     end
 
-    def bulk_upsert(updates, inserts)
+    def bulk_upsert(upserts, scope=nil)
       Database.batch do
-        updates.group_by(&:keys).transform_values{|v| v.map(&:values) }.each do |columns, rows|
-          update = rows.map{|df_id, period_index, *values| [period_index, [values, df_id]] }
-          bulk_update(update, columns - [:data_frame_id, :period_index])
-        end
-        inserts.group_by(&:keys).transform_values{|v| v.map(&:values) }.each do |columns, rows|
-          insert = rows.map{|df_id, period_index, *values| [period_index, [values, df_id]] }
-          bulk_insert(insert, columns - [:data_frame_id, :period_index])
+        case ActiveRecord::Base.connection_db_config.adapter
+        when 'postgresql'.freeze
+          upserts.group_by(&:keys).each do |columns, value_list|
+            columns = columns - [:data_frame_id, :period_index]
+            inserts = ''
+            value_list.each do |row|
+              df_id, period_index, *values = row.values
+              inserts <<  "(#{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}, #{df_id}, #{period_index}, '#{data_frame_type.name}'),"
+            end
+            sql = %Q{
+              INSERT INTO #{block_type.table_name} (#{columns.join(',')}, data_frame_id, period_index, data_frame_type)
+              VALUES #{inserts[0..-2]}
+              ON CONFLICT(data_frame_id, period_index, data_frame_type) DO UPDATE
+              SET #{columns.map{|c| "#{c} = excluded.#{c} "}.join(',')}
+            }
+            Database.execute sql
+          end
+        when 'mysql2'.freeze
+          upserts.group_by(&:keys).each do |columns, rows|
+            update = rows.map(&:values).map{|df_id, period_index, *values| [period_index, [values, df_id]] }
+            bulk_update(update, columns - [:data_frame_id, :period_index])
+          end
+        else
+          all_update_indices     = scope[].pluck(:data_frame_id, :period_index)
+          grouped_update_indices = all_update_indices.group_by(&:first).transform_values{|value| Set.new(value.map!(&:last)) }
+          updates, inserts = upserts.partition{|upsert| grouped_update_indices[upsert[:data_frame_id]]&.include?(upsert[:period_index]) }
+          updates.group_by(&:keys).each do |columns, rows|
+            update = rows.map(&:values).map{|df_id, period_index, *values| [period_index, [values, df_id]] }
+            bulk_update(update, columns - [:data_frame_id, :period_index])
+          end
+          inserts.group_by(&:keys).each do |columns, rows|
+            insert = rows.map(&:values).map{|df_id, period_index, *values| [period_index, [values, df_id]] }
+            bulk_insert(insert, columns - [:data_frame_id, :period_index])
+          end
         end
       end
     end
+
     ##
-    # Update block data for all blocks in a single call
+    # Fast update block data for all blocks in a single call.
+    # Uses UPDATE + SET in PostgreSQL
+    # Uses INSERT ON CONFLICT for MySQL (Upsert)
+    # Uses UPDATE with CASE on others
     ##
     def bulk_update(existing, columns=block_type::COLUMNS)
       existing.each_slice(ActiveDataFrame.update_max_batch_size) do |existing_slice|
         # puts "Updating slice of #{existing_slice.length}"
-        case ActiveRecord::Base.connection_config[:adapter]
+        case ActiveRecord::Base.connection_db_config.adapter
         when 'postgresql'.freeze
           #
           # PostgreSQL Supports the fast setting of multiple update values that differ
@@ -143,6 +174,7 @@ module ActiveDataFrame
       end
     end
 
+
     def bulk_delete(id, indices)
       indices.each_slice(ActiveDataFrame.delete_max_batch_size) do |slice|
         # puts "Deleting slice of #{slice.length}"
@@ -152,20 +184,30 @@ module ActiveDataFrame
 
     ##
     # Insert block data for all blocks in a single call
+    # PostgreSQL uses COPY, others use multi-statement insert
     ##
     def bulk_insert(new_blocks, columns=block_type::COLUMNS)
       new_blocks.each_slice(ActiveDataFrame.insert_max_batch_size) do |new_blocks_slice|
-        # puts "Inserting slice of #{new_blocks_slice.length}"
-        inserts = ''
-        new_blocks_slice.each do |period_index, (values, df_id)|
-          inserts << \
-          case ActiveRecord::Base.connection_config[:adapter]
-          when 'postgresql', 'mysql2' then "(#{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}, #{df_id}, #{period_index}, '#{data_frame_type.name}'),"
-          else "(#{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}, #{df_id}, #{period_index}, '#{data_frame_type.name}'),"
+        if ActiveRecord::Base.connection_db_config.adapter == 'postgresql'
+          copy_statement = "COPY #{block_type.table_name} (#{columns.join(',')},data_frame_id,period_index,data_frame_type) FROM STDIN CSV"
+          db_conn = ActiveRecord::Base.connection.raw_connection
+          db_conn.copy_data(copy_statement) do
+            new_blocks_slice.each do |period_index, (values, df_id)|
+              db_conn.put_copy_data((values + [df_id, period_index, data_frame_type.name]).join(',') << "\n")
+            end
+          end
+        else
+          inserts = ''
+          new_blocks_slice.each do |period_index, (values, df_id)|
+            inserts << \
+            case ActiveRecord::Base.connection_db_config.adapter
+            when 'mysql2' then "(#{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}, #{df_id}, #{period_index}, '#{data_frame_type.name}'),"
+            else "(#{values.map{|v| v.inspect.gsub('"',"'") }.join(',')}, #{df_id}, #{period_index}, '#{data_frame_type.name}'),"
+            end
           end
+          sql = "INSERT INTO #{block_type.table_name} (#{columns.join(',')}, data_frame_id, period_index, data_frame_type) VALUES #{inserts[0..-2]}"
+          Database.execute sql
         end
-        sql = "INSERT INTO #{block_type.table_name} (#{columns.join(',')}, data_frame_id, period_index, data_frame_type) VALUES #{inserts[0..-2]}"
-        Database.execute sql
       end
     end
   end

data/lib/active_data_frame/row.rb

@@ -13,7 +13,7 @@ module ActiveDataFrame
     end
 
     def self.set_all(scope, block_type, data_frame_type, from, values, trim: false)
-      if trim || ActiveRecord::Base.connection_config[:adapter] === 'mysql2'
+      if trim || ActiveRecord::Base.connection_db_config.adapter === 'mysql2'
         case values
         when Hash then scope.where(id: values.keys)
                             .each{|instance| Row.new(block_type, data_frame_type, instance)
@@ -31,9 +31,7 @@ module ActiveDataFrame
       bounds                 = get_bounds(from, to, block_type)
       scope                  = block_type.where(data_frame_type: data_frame_type.name, data_frame_id: rows.select(:id))
       scope                  = scope.where(data_frame_id: values.keys) if values.kind_of?(Hash)
-      all_update_indices     = scope.where(period_index: bounds.from.index..bounds.to.index).order(data_frame_id: :asc, period_index: :asc).pluck(:data_frame_id, :period_index)
-      grouped_update_indices = all_update_indices.group_by(&:first).transform_values{|value| Set.new(value.map!(&:last)) }
-      instance_ids           = rows.pluck(:id)
+      instance_ids           = rows.loaded? ? rows.map(&:id) : rows.pluck(:id)
       instance_ids           &= values.keys if values.kind_of?(Hash)
       upserts = to_enum(:iterate_bounds, [bounds], block_type).flat_map do |index, left, right, cursor, size|
         instance_ids.map do |instance_id|
@@ -42,13 +40,12 @@ module ActiveDataFrame
         end
       end
 
-      update, insert = upserts.partition{|upsert| grouped_update_indices[upsert[:data_frame_id]]&.include?(upsert[:period_index]) }
-      Database.for_types(block: block_type, df: data_frame_type).bulk_upsert(update, insert)
+      Database.for_types(block: block_type, df: data_frame_type).bulk_upsert(upserts, ->{scope.where(period_index: bounds.from.index..bounds.to.index)})
       values
     end
 
     def set(from, values, trim: false)
-      if trim || ActiveRecord::Base.connection_config[:adapter] === 'mysql2'
+      if trim || ActiveRecord::Base.connection_db_config.adapter === 'mysql2'
         patch(from, values)
       else
         upsert(from, values)
@@ -58,16 +55,10 @@ module ActiveDataFrame
     def upsert(from, values)
       to             = (from + values.length) - 1
       bounds         = get_bounds(from, to)
-      update_indices = Set.new(scope.where(period_index: bounds.from.index..bounds.to.index).order(period_index: :asc).pluck(:period_index))
-      # Detect blocks in bounds:
-      # - If existing and covered, do an update without load
-      # - If existing and uncovered, do a small write (without load)
-      # - If not existing, insert!
       upserts = to_enum(:iterate_bounds, [bounds]).map do |index, left, right, cursor, size|
         [[:data_frame_id, self.instance.id], [:period_index, index], *(left.succ..right.succ).map{|v| :"t#{v}" }.zip(values[cursor...cursor + size])].to_h
       end
-      update, insert = upserts.partition{|upsert| update_indices.include?(upsert[:period_index]) }
-      database.bulk_upsert(update, insert)
+      database.bulk_upsert(upserts, ->{ scope.where(period_index: bounds.from.index..bounds.to.index)})
       values
     end
 
@@ -99,7 +90,6 @@ module ActiveDataFrame
         end
       end
 
-
       database.bulk_delete(self.instance.id, deleted_indices) unless deleted_indices.size.zero?
       database.bulk_update(existing)       unless existing.size.zero?
       database.bulk_insert(new_blocks)     unless new_blocks.size.zero?

data/lib/active_data_frame/table.rb

@@ -104,7 +104,7 @@ module ActiveDataFrame
         ActiveRecord::Base.connection.execute(as_sql)
       end
 
-      case ActiveRecord::Base.connection_config[:adapter]
+      case ActiveRecord::Base.connection_db_config.adapter
       when 'postgresql'.freeze
         res.each_row {|pi, data_frame_id, *values| existing_blocks[pi][data_frame_id] = values }
       when 'mysql2'.freeze

data/lib/active_data_frame/version.rb

@@ -1,3 +1,3 @@
 module ActiveDataFrame
-  VERSION = "0.1.8"
+  VERSION = "0.1.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: active_data_frame
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.11
 platform: ruby
 authors:
 - Wouter Coppieters
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-08-12 00:00:00.000000000 Z
+date: 2023-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,68 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '2.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.13'
+        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: pry-byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.4.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.4.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 3.4.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.4.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.10.2
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.10.2
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.10.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -174,14 +162,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '7.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '7.0'
 - !ruby/object:Gem::Dependency
   name: rmatrix
   requirement: !ruby/object:Gem::Requirement
@@ -212,6 +200,7 @@ files:
 - ".gitignore"
 - CODE_OF_CONDUCT.md
 - Gemfile
+- LICENSE
 - README.md
 - Rakefile
 - active_data_frame-0.1.1.gem
@@ -237,7 +226,7 @@ files:
 homepage: https://github.com/wouterken/active_data_frame
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -252,9 +241,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.4.6
+signing_key:
 specification_version: 4
 summary: An active data frame helper
 test_files: []