data_modeler 0.0.0 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ab07761ab1c6a95194902cb445c9af4f63f13ad
-  data.tar.gz: '0812c7257cb9b151790b1c95f7c796e4b94b8bbc'
+  metadata.gz: 2b665a6495480b44b7d337d0a1acf6a7ad826262
+  data.tar.gz: 85ab09bec546d8bb18b5f3c0ac3f0a48b926cd86
 SHA512:
-  metadata.gz: 74fab45b2f6293fd7250ae07dddbe688bcd77dc1c2a258388fd63aa300abf824b850075533a676c6b115424d1615a4665ba2b5ba61cb14ba700f40fe1d80fc13
-  data.tar.gz: 6b8484b312d6a6fc907d00491f3e9db4a921e68a5d07eeca462c9e428f5fddced203be1c168445b24be2742fbdfddefc728e90d1e0af4d9d514cce2f76e346aa
+  metadata.gz: 05d5e05210191c51be3b50fe1db36787d8db0ab6659cdf902d6b41f8d3d86155afb39f714b1a843321a4c39fdd967b3dfc050f0caccebe574301fc281c450e96
+  data.tar.gz: 5b565eb79f326a0e169a496d12e9b98a8b432bc23caa67df74ac0d03ebeb7772ead78840fb982e0b3bb8e1ffccb503e13a623a6eee9b335f0ffef125d6286091

data/data_modeler.gemspec

@@ -22,10 +22,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   # Debug
-  spec.add_development_dependency 'pry', '~> 0'
-  spec.add_development_dependency 'pry-nav', '~> 0'
-  spec.add_development_dependency 'pry-stack_explorer', '~> 0'
-  spec.add_development_dependency 'pry-rescue', '~> 0'
+  spec.add_development_dependency 'pry', '~> 0.10'
+  spec.add_development_dependency 'pry-nav', '~> 0.2'
+  spec.add_development_dependency 'pry-stack_explorer', '~> 0.4'
+  spec.add_development_dependency 'pry-rescue', '~> 1.4'
 
   # Test
   spec.add_development_dependency 'bundler', '~> 1.14'

data/lib/data_modeler.rb

@@ -1,5 +1,5 @@
 require "data_modeler/version"
-
-module DataModeler
-  # Your code goes here...
-end
+require "data_modeler/exceptions"
+require "data_modeler/dataset_helper"
+require "data_modeler/dataset"
+require "data_modeler/dataset_gen"

data/lib/data_modeler/dataset.rb

@@ -0,0 +1,107 @@
+
+# Create complex inputs and targets for the model from the given data
+# @note checks to validate if enough data is present (given ntimes, tspread
+#     and look_ahead) should be done on the caller (typically DatasetGen)
+class DataModeler::Dataset
+
+  attr_reader :data, :input_series, :target_series, :first_idx, :end_idx,
+              :ntimes, :tspread, :look_ahead, :target_idx, :input_idxs,
+              :nrows
+
+  # @param data [Hash-like] the data, in an object that can be
+  #     accessed by keys and return a time series per each key.
+  #     It is required to include and be sorted by a series named `time`,
+  #     and for all series to have equal length.
+  # @param inputs [Array] data key accessors for input series
+  # @param targets [Array] data key accessors for target series
+  # @param first_idx [Integer] index where the dataset starts on data
+  # @param end_idx [Integer] index where the dataset ends on data
+  # @param ntimes [Integer] number of lines/times/datapoints to be
+  #     used to construct the input
+  # @param tspread [Numeric] distance (in `time`!) between the `ntimes`
+  #     lines/times/datapoints used to construct the input
+  # @param look_ahead [Numeric] distance (in `time`!) between the
+  #     most recent line/time/datapoint used for the input and
+  #     the target -- i.e., how far ahead the model is trained to predict
+  # @note we expect Datasets indices to be used with left inclusion but
+  #     right exclusion, i.e. targets are considered in the range `[from,to)`
+  def initialize data, inputs:, targets:, first_idx:, end_idx:,
+      ntimes:, tspread:, look_ahead:
+    @data = data
+    @input_series = inputs
+    @target_series = targets
+    @first_idx = first_idx
+    @end_idx = end_idx
+    @ntimes = ntimes
+    @nrows = data[:time].size
+    @tspread = tspread
+    @look_ahead = look_ahead
+    @target_idx = first_idx
+    @input_idxs = init_inputs
+  end
+
+  # TODO: make sure constructor requirements are unnecessary for static models
+  # TODO: check if enough data / minimum_target
+  # TODO: the check in `#init_target` should go in the `ds_gen`
+
+  # Builds inputs for the model
+  # @return [Array]
+  def inputs
+    input_idxs.flat_map do |idx|
+      input_series.collect do |s|
+        data[s][idx]
+      end
+    end
+  end
+
+  # Builds targets for the model
+  # @return [Array]
+  def targets
+    target_series.collect do |s|
+      data[s][target_idx]
+    end
+  end
+
+  # Returns the next pair [inputs, targets]
+  # @return [Array]
+  def peek
+    raise StopIteration if target_idx >= end_idx
+    [inputs, targets]
+  end
+
+  # Returns the next pair [inputs, targets] and increments the target
+  # @return [Array]
+  def next
+    peek.tap do
+      @target_idx += 1
+      @input_idxs = init_inputs
+    end
+  end
+
+  include DataModeler::IteratingBasedOnNext # `#each` and `#to_a` based on `#next`
+
+  # Overloaded comparison for easier testing
+  def == other
+    self.class == other.class &&
+      data.object_id == other.data.object_id &&
+      (instance_variables - [:@data]).all? do |var|
+        self.instance_variable_get(var) == other.instance_variable_get(var)
+      end
+  end
+
+  private
+
+  include DataModeler::ConvertingTimeAndIndices # `#time` and `#idx`
+
+  def init_inputs
+    if target_idx < end_idx
+      # build list of incremental time buffers
+      bufs = ntimes.times.collect { |n| look_ahead + n * tspread }
+      # reverse it and subtract from the target's time
+      times = bufs.reverse.collect { |s| time(target_idx) - s }
+      # now you have the list of times at which each pointer should point
+      times.collect &method(:idx)
+    end
+  end
+
+end

data/lib/data_modeler/dataset_gen.rb

@@ -0,0 +1,105 @@
+
+# Create train and test datasets for the training.
+# @note: this diagram should help understanding how it works
+# ----------------------------------------> data (time)
+#   v- this is the input+look_ahead window for first training target
+# |win|train1|t1|       -> train starts after window, test after training
+#        |train2|t2|    -> train starts after window + 1 tset
+#           |train3|t3| -> train starts after window + 2 tset
+# Note how the test sets line up. This allows the testing results plots
+# to be continuous, no model is tested on data on which *itself* has been
+# trained, and all data is used multiple times
+class DataModeler::DatasetGen
+
+  attr_reader :data, :ds_args, :first_idx, :train_size, :test_size, :nrows
+
+  # @param data [Hash-like] the data, in an object that can be
+  #     accessed by keys and return a time series per each key.
+  #     It is required to include and be sorted by a series named `time`,
+  #     and for all series to have equal length.
+  # @param ds_args [Hash] parameters for the Datasets: inputs, targets,
+  #     first_idx, end_idx, ntimes. Check class Dataset for details.
+  # @train_size: how many points to predict for each training set
+  # @test_size: how many points to predict for each test set
+  def initialize data, ds_args:, train_size:, test_size:, min_nruns: 1
+    @data = data
+    @ds_args = ds_args
+    @first_idx = first_idx
+    @train_size = train_size
+    @test_size = test_size
+    @local_nrun = 1 # used to iterate over nruns with #next
+
+    @nrows = data[:time].size
+    validate_enough_data_for min_nruns
+  end
+
+  # Builds training set for the training
+  # @param nrun [Integer] will build different train+test for each run
+  # @return [Dataset]
+  # @raise [NoDataLeft] when there's not enough data left for a full train+test
+  def train nrun
+    first = min_eligible_trg + (nrun-1) * test_size
+    last = first + train_size
+    # make sure there's enough data for both train and test
+    raise NoDataLeft unless last + test_size < nrows
+    DataModeler::Dataset.new data, ds_args.merge(first_idx: first, end_idx: last)
+  end
+
+  # Builds test set for the training
+  # @param nrun [Integer] will build different train+test for each run
+  # @return [Dataset]
+  # @note we already checked pre-training there's enough data for the test too
+  def test nrun
+    first = min_eligible_trg + (nrun-1) * test_size + train_size
+    last = first + test_size
+    DataModeler::Dataset.new data, ds_args.merge(first_idx: first, end_idx: last)
+  end
+
+  # Returns the next pair [trainset, testset]
+  # @return [Array<Dataset, Dataset>]
+  def peek
+    [self.train(@local_nrun), self.test(@local_nrun)]
+  end
+
+  # TODO: @local_nrun is an ugly hack, refactor it!
+
+  # Returns the next pair [trainset, testset] and increments the counter
+  # @return [Array<Dataset, Dataset>]
+  def next
+    peek.tap { @local_nrun += 1 }
+  end
+
+  include DataModeler::IteratingBasedOnNext # `#each` and `#to_a` based on `#next`
+
+  # I want `#to_a` to return an array of arrays rather than an array of dataset
+
+  # @return [Array<Array[Dataset]>]
+  alias_method :to_ds_a, :to_a
+  # @return [Array<Array<Array<...>>]
+  def to_a
+    to_ds_a.collect do |run|
+      run.collect &:to_a
+    end
+  end
+
+  private
+
+  include DataModeler::ConvertingTimeAndIndices # `#time` and `#idx`
+
+  # Find the index of the first element in the data eligible as target for training
+  # @return [Integer] the index of the first eligible target
+  def min_eligible_trg
+    @min_eligible_trg ||= idx(time(0) +
+      # minimum time span required as input for the first target
+      ds_args[:look_ahead] + (ds_args[:ntimes]-1) * ds_args[:tspread]
+    )
+  end
+
+  # Check if there is enough data to build `min_nruns` train + test sets
+  # @raise [NotEnoughDataError] if `not enough minerals` (cit.)
+  # @note remember the schema: need to check for `|win|train1|t1|t2|...|tn|`
+  def validate_enough_data_for min_nruns
+    min_data_size = min_eligible_trg + train_size + min_nruns * test_size
+    raise NotEnoughDataError if nrows < min_data_size
+  end
+end

data/lib/data_modeler/dataset_helper.rb

@@ -0,0 +1,44 @@
+module DataModeler
+  # Converts between time and indices for referencing data lines
+  module ConvertingTimeAndIndices
+    # Returns the time for a given index
+    # @param [Integer] idx row index
+    # @return [kind_of_time]
+    def time idx
+      data[:time][idx]
+    end
+
+    # Returns the index for a given time
+    # @param [time] time
+    # @return [Integer] row index
+    def idx time
+      # TODO: optimize with `from:`
+      # TODO: test corner case when index not found
+      # find index of first above time
+      idx = data[:time].index { |t| t > time }
+      # if index not found: all data is below time, "first above" is outofbound
+      idx ||= nrows
+      # if first above time is 0: there is no element with that time
+      raise TimeNotFoundError, "Time not found: #{time}" if idx.zero?
+      # return index of predecessor (last below time)
+      idx-1
+    end
+  end
+
+  # Provides each (which can return an `Iterator`) and `to_a` based on `#next`
+  module IteratingBasedOnNext
+    # Yields on each [inputs, targets] pair.
+    # @return [nil, Iterator] `block_given? ? nil : Iterator`
+    def each
+      return enum_for(:each) unless block_given?
+      loop { yield self.next }
+      nil
+    end
+
+    # @return [Array]
+    def to_a
+      each.to_a
+    end
+
+  end
+end

data/lib/data_modeler/exceptions.rb

@@ -0,0 +1,12 @@
+class DataModeler::Dataset
+  # Exception: the requested `time` is not present in the data
+  class TimeNotFoundError < StandardError; end
+end
+
+class DataModeler::DatasetGen
+  # Exception: the `data` is not sufficient for the training setup
+  class NotEnoughDataError < StandardError; end
+  # Exception: not enough `data` left to build another train+test
+  # @note being subclassed from `StopIteration`, it will break loops
+  class NoDataLeft < StopIteration; end
+end

data/lib/data_modeler/version.rb

@@ -1,3 +1,5 @@
+# Main gem module
 module DataModeler
-  VERSION = "0.0.0"
+  # Version number
+  VERSION = "0.1.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: data_modeler
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.0
 platform: ruby
 authors:
 - Giuseppe Cuccu
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-10 00:00:00.000000000 Z
+date: 2017-05-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -16,56 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: pry-nav
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: pry-stack_explorer
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.4'
 - !ruby/object:Gem::Dependency
   name: pry-rescue
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.4'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.4'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -128,6 +128,10 @@ files:
 - bin/setup
 - data_modeler.gemspec
 - lib/data_modeler.rb
+- lib/data_modeler/dataset.rb
+- lib/data_modeler/dataset_gen.rb
+- lib/data_modeler/dataset_helper.rb
+- lib/data_modeler/exceptions.rb
 - lib/data_modeler/version.rb
 homepage: https://github.com/giuse/data_modeler
 licenses: