ruby_px 0.2.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 44941ebae6202b2580d1303054c643bd3e713971
-  data.tar.gz: 4d9b8e3bc4dc2ae1156366473137d68618f31b05
+SHA256:
+  metadata.gz: 9abe10d6ad1aed7bb9d28535671457cad398d7c17cab9c9bc49376dd46be5a3a
+  data.tar.gz: 95e9b594074e45eaa34e319c2ef58fc73edf4e5f6ceee9ae7c67bb2919407425
 SHA512:
-  metadata.gz: 915d7fa2b750f11a241d25b2188210e4cbcceb2041a6c501fe78df65f480e21bce23bf172be39d1972d910ad05fea61277706fe543124a2e2bd9d5371a5aebcf
-  data.tar.gz: 37b7bdc3faa735d97cbd04cc341e8811a49f3340b69a8e25ed21cd150e7a0520ced342c5d1fc7b7f81da786f13c941fa48c43b122f4d91cb3ff070b2451c0361
+  metadata.gz: ff283ea31198ae6fd560b591ac6c1d7dc649a148a61a3a4570953c380c22f4c667fe15a569454df1649d744845b62734051d89863e7e8c523e2f270ea77b3762
+  data.tar.gz: a36341f8e244874a2d588b0e28319c43da1f9f8b2488007ea79f1cccf16be8db544bf59839d6c1dc8f495e0581e2f3341922c41df9ef6a3d907f57748a0e31f2

data/.gitignore

@@ -8,3 +8,4 @@
 /spec/reports/
 /tmp/
 .byebug_history
+.ruby-version

data/.travis.yml

@@ -1,4 +1,5 @@
 language: ruby
 rvm:
-  - 2.2.3
-before_install: gem install bundler -v 1.10.6
+  - 2.7.1
+  - 2.6.3
+before_install: gem install bundler

data/CHANGELOG.md

@@ -0,0 +1,14 @@
+# CHANGELOG
+
+## 0.7.0 (2021-03-04)
+
+- Performance in large datasets [#9](https://github.com/PopulateTools/ruby_px/pull/9)
+
+## 0.6.0 (2020-04-28)
+
+- Nice badge with the current version
+- Added Changelog!
+- Updated dependencies
+- Fix multilingual VALUES and equals signs in values [#7](https://github.com/PopulateTools/ruby_px/pull/7)
+- Use Rubocop to check syntax
+

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in ruby_px.gemspec

data/README.md

@@ -1,5 +1,6 @@
 # RubyPx
 
+[![Gem Version](https://badge.fury.io/rb/ruby_px.svg)](https://badge.fury.io/rb/ruby_px)
 [![Build Status](https://travis-ci.org/PopulateTools/ruby_px.svg?branch=master)](https://travis-ci.org/PopulateTools/ruby_px)
 
 Work with PC-Axis files using Ruby.
@@ -82,7 +83,6 @@ dataset.data('edad (año a año)' => 'Total', 'sexo' => 'Ambos sexos')
 
 ## TODO
 
-- Allow to receive an URL as an argument
 - Refactor
 - Test the gem with more files
 - Speed-up the parsing time

data/Rakefile

@@ -1,6 +1,8 @@
-require "bundler/gem_tasks"
-require "rspec/core/rake_task"
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/console

@@ -1,7 +1,8 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-require "bundler/setup"
-require "pc-axis/dataset"
+require 'bundler/setup'
+require_relative '../lib/ruby_px'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.
@@ -10,5 +11,5 @@ require "pc-axis/dataset"
 # require "pry"
 # Pry.start
 
-require "irb"
+require 'irb'
 IRB.start

data/lib/ruby_px.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'active_support/all'
 
 module RubyPx

data/lib/ruby_px/dataset.rb

@@ -1,23 +1,25 @@
+# frozen_string_literal: true
+
+require 'open-uri'
+
 module RubyPx
   class Dataset
+    require 'ruby_px/dataset/data'
+
     attr_reader :headings, :stubs
 
-    METADATA_RECORDS = ['TITLE','UNITS','SOURCE','CONTACT','LAST-UPDATED','CREATION-DATE']
+    METADATA_RECORDS = %w[TITLE UNITS SOURCE CONTACT LAST-UPDATED CREATION-DATE].freeze
     HEADING_RECORD = 'HEADING'
     STUB_RECORD = 'STUB'
 
-    def initialize(file)
-      if !File.readable?(file)
-        raise "File #{file} not readable"
-      end
-
+    def initialize(resource_uri)
       @metadata = {}
       @headings = []
       @stubs = []
       @values = {}
-      @data = []
+      @data = Data.new
 
-      parse_file(file)
+      parse_resource(resource_uri)
     end
 
     def title
@@ -54,8 +56,10 @@ module RubyPx
 
     def data(options)
       # Validate parameters
-      options.each do |k,v|
-        raise "Invalid value #{v} for dimension #{k}" unless dimension(k).include?(v)
+      options.each do |k, v|
+        unless dimension(k).include?(v)
+          raise "Invalid value #{v} for dimension #{k}"
+        end
       end
 
       # Return a single value
@@ -67,20 +71,20 @@ module RubyPx
 
         # positions are i, j, k
         positions = (stubs + headings).map do |dimension_name|
-          self.dimension(dimension_name).index(options[dimension_name])
+          dimension(dimension_name).index(options[dimension_name])
         end
 
         # dimension_sizes are from all dimensions except the first one
         dimension_sizes = (stubs + headings)[1..-1].map do |dimension_name|
-          self.dimension(dimension_name).length
+          dimension(dimension_name).length
         end
 
         positions.each_with_index do |p, i|
           d = dimension_sizes[i..-1].reduce(&:*)
-          offset += (d ? p*d : p)
+          offset += (d ? p * d : p)
         end
 
-        return @data[offset]
+        @data.at(offset)
 
       # Return an array of options
       elsif options.length == dimensions.length - 1
@@ -91,30 +95,31 @@ module RubyPx
           result << data(options.merge(missing_dimension => dimension_value))
         end
 
-        return result
+        result
       else
-        raise "Not implented yet, sorry"
+        raise 'Not implented yet, sorry'
       end
     end
 
     def inspect
-      "#<#{self.class.name}:#{self.object_id}>"
+      "#<#{self.class.name}:#{object_id}>"
     end
 
     private
 
-    def parse_file(file)
-      File.foreach(file) do |line|
+    def parse_resource(resource_uri)
+      open(resource_uri).each_line do |line|
         parse_line(line.chomp)
       end
-      return true
+
+      true
     end
 
     def parse_line(line)
-      @line = line
+      @line = line.force_encoding('utf-8').encode('utf-8')
 
       if @current_record.nil?
-        key, value = line.scan(/[^\=]+/)
+        key, value = line.split('=', 2)
         set_current_record(key)
       else
         value = line
@@ -123,27 +128,25 @@ module RubyPx
       return if @current_record.nil? || value.nil?
 
       if @type == :data
-        value = value.split(' ')
+        value = value.split(/[\ ;,\t]/).delete_if(&:blank?).each(&:strip)
 
-        add_value_to_bucket(bucket,value) unless value == [';']
+        add_value_to_bucket(bucket, value) unless value == [';']
       else
         # First format: "\"20141201\";"
         if value =~ /\A\"([^"]+)\";\z/
           value = value.match(/\A\"([^"]+)\";\z/)[1]
           add_value_to_bucket(bucket, value.strip)
 
-          # Second format: "Ambos sexos","Hombres","Mujeres";
+        # Second format: "Ambos sexos","Hombres","Mujeres";
         elsif value =~ /\"([^"]+)\",?/
-          value = value.split(/\"([^"]+)\",?;?/).delete_if{ |s| s.blank? }.each(&:strip)
+          value = value.split(/\"([^"]+)\",?;?/).delete_if(&:blank?).each(&:strip)
           add_value_to_bucket(bucket, value)
         end
       end
 
       # If we see a ; at the end of the line, close out the record so we
       # expect a new record.
-      if line[-1..-1] == ";"
-        @current_record = nil
-      end
+      @current_record = nil if line[-1..-1] == ';'
     end
 
     def set_current_record(key)
@@ -156,7 +159,7 @@ module RubyPx
                         elsif key == STUB_RECORD
                           @type = :stubs
                           key
-                        elsif key =~ /\AVALUES/
+                        elsif key =~ /\AVALUES/ && key !~ /\[\w\w\]/
                           @type = :values
                           key.match(/\"([^"]+)\"/)[1]
                         elsif key =~ /\ADATA/
@@ -175,22 +178,20 @@ module RubyPx
       elsif @type == :headings || @type == :stubs
         bucket << value
         bucket.flatten!
-      else
-        if bucket.is_a?(Hash)
-          if value.is_a?(Array)
-            value = value.map(&:strip)
-          elsif value.is_a?(String)
-            value.strip!
-          end
-          if bucket[@current_record].nil?
-            bucket[@current_record] = value
-          else
-            bucket[@current_record].concat([value])
-            bucket[@current_record].flatten!
-          end
+      elsif bucket.is_a?(Hash)
+        if value.is_a?(Array)
+          value = value.map(&:strip)
+        elsif value.is_a?(String)
+          value.strip!
+        end
+        if bucket[@current_record].nil?
+          value = Array.wrap(value) if @type == :values
+          bucket[@current_record] = value
+        else
+          bucket[@current_record].concat([value])
+          bucket[@current_record].flatten!
         end
       end
     end
-
   end
 end

data/lib/ruby_px/dataset/data.rb

@@ -0,0 +1,48 @@
+module RubyPx
+  class Dataset
+    class Data
+
+      CHUNK_SIZE = 5_000
+      attr_accessor :current_chunk_index
+
+      def initialize
+        @current_chunk_index = 0
+      end
+
+      def at index
+        chunk_index = index/CHUNK_SIZE
+        index_inside_chunk = index%CHUNK_SIZE
+
+        get_chunk(chunk_index)[index_inside_chunk]
+      end
+
+      def concat array
+        current_chunk.concat(array)
+        if current_chunk.size > CHUNK_SIZE
+          excess = current_chunk.pop(current_chunk.size-CHUNK_SIZE)
+          self.current_chunk_index += 1
+          concat(excess)
+        end
+      end
+
+      def indexes_count
+        self.current_chunk_index+1
+      end
+
+      private
+
+
+      def current_chunk
+        current = instance_variable_get("@chunk_#{self.current_chunk_index}")
+        return current if current
+
+        instance_variable_set("@chunk_#{self.current_chunk_index}", [])
+      end
+
+      def get_chunk chunk_index
+        instance_variable_get("@chunk_#{chunk_index}")
+      end
+
+    end
+  end
+end

data/ruby_px.gemspec

@@ -1,34 +1,35 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
-  spec.name          = "ruby_px"
-  spec.version       = '0.2.0'
-  spec.authors       = ["Fernando Blat"]
-  spec.email         = ["ferblape@gmail.com"]
+  spec.name          = 'ruby_px'
+  spec.version       = '0.7.0'
+  spec.authors       = ['Fernando Blat']
+  spec.email         = ['fernando@blat.es']
 
-  spec.summary       = %q{Read PC-Axis files using Ruby}
-  spec.description   = %q{Read PC-Axis files using Ruby}
-  spec.homepage      = "https://github.com/PopulateTools/ruby_px"
-  spec.license       = "MIT"
+  spec.summary       = 'Read PC-Axis files using Ruby'
+  spec.description   = 'Read PC-Axis files using Ruby'
+  spec.homepage      = 'https://github.com/PopulateTools/ruby_px'
+  spec.license       = 'MIT'
 
   # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
   # delete this section to allow pushing this gem to any host.
   if spec.respond_to?(:metadata)
-    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+    spec.metadata['allowed_push_host'] = 'https://rubygems.org'
   else
-    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+    raise 'RubyGems 2.0 or newer is required to protect against public gem pushes.'
   end
 
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  spec.bindir        = "exe"
+  spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_runtime_dependency "activesupport", "~> 4.2", ">= 4.2.5"
+  spec.add_runtime_dependency 'activesupport', '>= 6.0'
 
-  spec.add_development_dependency "bundler", "~> 1.10"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.4"
+  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rspec', '~> 3.9'
 end

metadata

@@ -1,88 +1,82 @@
 --- !ruby/object:Gem::Specification
 name: ruby_px
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Fernando Blat
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-13 00:00:00.000000000 Z
+date: 2021-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.5
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '4.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.5
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.10'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.9'
 description: Read PC-Axis files using Ruby
 email:
-- ferblape@gmail.com
+- fernando@blat.es
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rspec"
-- ".ruby-version"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE.txt
@@ -92,6 +86,7 @@ files:
 - bin/setup
 - lib/ruby_px.rb
 - lib/ruby_px/dataset.rb
+- lib/ruby_px/dataset/data.rb
 - ruby_px.gemspec
 homepage: https://github.com/PopulateTools/ruby_px
 licenses:
@@ -113,10 +108,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Read PC-Axis files using Ruby
 test_files: []
-has_rdoc: 

data/.ruby-version

@@ -1 +0,0 @@
-2.2.3