ruby_px 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cfa0ddb5f455e289afe1507164ab005e0896e7e091531a052472f36cfdfe11d3
-  data.tar.gz: 3acf6d18b2ae29ddceab2c0859d4fe79be33a61a54be7f93632a014ebaf5dee7
+  metadata.gz: 9abe10d6ad1aed7bb9d28535671457cad398d7c17cab9c9bc49376dd46be5a3a
+  data.tar.gz: 95e9b594074e45eaa34e319c2ef58fc73edf4e5f6ceee9ae7c67bb2919407425
 SHA512:
-  metadata.gz: c2439c7a3698ae78f288866cb91933325dbd48ec88a2bbc6b54353e6fe4567e768b7a2c6febb326c2e6566f1f4633eca4b6bb4d7e5ea471dacae21dbe6818f66
-  data.tar.gz: 5482d50e2be5fc9fc4ab6c1b2d40bdf7e2b4b1ea8f2f528ef13d2c575cd9630208ff6e188ba3e3e8c3f17eb4e005787c7c64095c4619d77b0ed23125a5865d80
+  metadata.gz: ff283ea31198ae6fd560b591ac6c1d7dc649a148a61a3a4570953c380c22f4c667fe15a569454df1649d744845b62734051d89863e7e8c523e2f270ea77b3762
+  data.tar.gz: a36341f8e244874a2d588b0e28319c43da1f9f8b2488007ea79f1cccf16be8db544bf59839d6c1dc8f495e0581e2f3341922c41df9ef6a3d907f57748a0e31f2

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 0.7.0 (2021-03-04)
+
+- Performance in large datasets [#9](https://github.com/PopulateTools/ruby_px/pull/9)
+
 ## 0.6.0 (2020-04-28)
 
 - Nice badge with the current version

data/lib/ruby_px/dataset.rb

@@ -4,6 +4,8 @@ require 'open-uri'
 
 module RubyPx
   class Dataset
+    require 'ruby_px/dataset/data'
+
     attr_reader :headings, :stubs
 
     METADATA_RECORDS = %w[TITLE UNITS SOURCE CONTACT LAST-UPDATED CREATION-DATE].freeze
@@ -15,7 +17,7 @@ module RubyPx
       @headings = []
       @stubs = []
       @values = {}
-      @data = []
+      @data = Data.new
 
       parse_resource(resource_uri)
     end
@@ -82,7 +84,7 @@ module RubyPx
           offset += (d ? p * d : p)
         end
 
-        @data[offset]
+        @data.at(offset)
 
       # Return an array of options
       elsif options.length == dimensions.length - 1

data/lib/ruby_px/dataset/data.rb

@@ -0,0 +1,48 @@
+module RubyPx
+  class Dataset
+    class Data
+
+      CHUNK_SIZE = 5_000
+      attr_accessor :current_chunk_index
+
+      def initialize
+        @current_chunk_index = 0
+      end
+
+      def at index
+        chunk_index = index/CHUNK_SIZE
+        index_inside_chunk = index%CHUNK_SIZE
+
+        get_chunk(chunk_index)[index_inside_chunk]
+      end
+
+      def concat array
+        current_chunk.concat(array)
+        if current_chunk.size > CHUNK_SIZE
+          excess = current_chunk.pop(current_chunk.size-CHUNK_SIZE)
+          self.current_chunk_index += 1
+          concat(excess)
+        end
+      end
+
+      def indexes_count
+        self.current_chunk_index+1
+      end
+
+      private
+
+
+      def current_chunk
+        current = instance_variable_get("@chunk_#{self.current_chunk_index}")
+        return current if current
+
+        instance_variable_set("@chunk_#{self.current_chunk_index}", [])
+      end
+
+      def get_chunk chunk_index
+        instance_variable_get("@chunk_#{chunk_index}")
+      end
+
+    end
+  end
+end

data/ruby_px.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = 'ruby_px'
-  spec.version       = '0.6.0'
+  spec.version       = '0.7.0'
   spec.authors       = ['Fernando Blat']
   spec.email         = ['fernando@blat.es']
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby_px
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Fernando Blat
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-20 00:00:00.000000000 Z
+date: 2021-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -86,6 +86,7 @@ files:
 - bin/setup
 - lib/ruby_px.rb
 - lib/ruby_px/dataset.rb
+- lib/ruby_px/dataset/data.rb
 - ruby_px.gemspec
 homepage: https://github.com/PopulateTools/ruby_px
 licenses: