ssstats 0.2.1

data/lib/ssstats.rb

@@ -0,0 +1,129 @@
+
+class Ssstats
+
+  # TODO 2-sigma stats
+  CURRENTLY_AVAILABLE = %i[qty min max sum avg std]
+
+  def initialize
+    @stats = {}
+    @schema = {}
+  end
+
+  def <<(datum)
+    stat_schema datum, @schema, @stats
+  end
+
+  def schema
+    clone @schema
+  end
+
+  def stats
+    clone @stats
+  end
+
+  def stat(key)
+    @stats.transform_values{|col| col[key]}.compact
+  end
+
+  CURRENTLY_AVAILABLE.each do |calc|
+    define_method calc do
+      stat calc
+    end
+  end
+
+  private
+
+  def stat_schema(datum, schema, stats, path=[])
+    stat_add path.join('.'), datum
+
+    if datum.respond_to? :keys
+      (schema || {}).tap do |schema|
+        datum.keys.each do |key|
+          key_s = key.to_s
+          key_s += "'"  while schema[key_s] && class_of(schema[key_s]) != class_of(datum[key])
+          schema[key_s] = stat_schema(datum[key], schema[key_s], stats, path + [key_s])
+        end
+      end
+    elsif datum.respond_to? :each
+      (schema || []).tap do |schema|
+        datum.each do |el|
+          el_schema = schema.find{|e| class_of(e) == class_of(el)}
+          new_schema = stat_schema(el, el_schema, stats, path + [class_of(datum)])
+          schema << new_schema  unless el_schema
+        end
+      end
+    else
+      if datum.class.respond_to? :new
+        datum.class.new
+      else
+        rep_class = datum.class
+        while rep_class  # NOTE this complication is due to cases like the Fixnum's
+          return send rep_class.name, 0  if Kernel.respond_to? rep_class.name  # NOTE the case of Numeric's
+          rep_class = rep_class.superclass
+        end
+        fail "Can't represent #{datum} (#{datum.class.name})"
+      end
+    end
+  end
+
+  def stat_add(key, val)
+    stat_key = [key, class_of(val)].join('.')
+    val_stat = @stats[stat_key] ||= {}
+    stat_count val_stat
+
+    if val.respond_to? :length
+      len_stat = @stats[[stat_key, 'length'].join '.'] ||= {}
+      stat_count len_stat
+      num_stat len_stat, val.length
+    end
+
+    num_stat val_stat, val  if val.is_a? Numeric
+  end
+
+  def num_stat(stat, num)
+    num_stat_min_max stat, num
+    num_stat_sum stat, num
+    num_stat_avg_std stat, num
+  end
+
+  def stat_count(stat)
+    stat[:qty] ||= 0
+    stat[:qty] += 1
+  end
+
+  def num_stat_min_max(stat, num)
+    stat[:min] ||= num
+    stat[:min] = num  if num < stat[:min]
+    stat[:max] ||= num
+    stat[:max] = num  if num > stat[:max]
+  end
+
+  def num_stat_sum(stat, num)
+    stat[:sum] ||= 0
+    stat[:sum] += num
+  end
+
+  def num_stat_avg_std(stat, num)
+    # NOTE ref https://www.johndcook.com/blog/standard_deviation
+    stat[:avg] ||= 0.0
+    avg_was = stat[:avg]
+    stat[:avg] += (num - avg_was) / stat[:qty]  # NOTE qty is not zero here
+    stat[:delta_squares_sum] ||= 0.0
+    stat[:delta_squares_sum] += (num - avg_was) * (num - stat[:avg])
+    stat[:std] = Math.sqrt(stat[:delta_squares_sum] / stat[:qty])
+  end
+
+  def clone(thing)
+    Marshal.load Marshal.dump thing
+  end
+
+  def class_of(datum)
+    if datum.respond_to? :keys
+      Hash
+    elsif datum.respond_to? :each
+      Array
+    else
+      datum.class
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require 'bundler/setup'
+require 'byebug'
+require 'ssstats'
+
+RSpec.configure do |config|
+  # Enable flags like --only-failures and --next-failure
+  config.example_status_persistence_file_path = '.rspec_status'
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

data/spec/ssstats_spec.rb

@@ -0,0 +1,53 @@
+RSpec.describe Ssstats do
+
+  it "has a version number" do
+    expect(Ssstats::VERSION).not_to be nil
+  end
+
+  it "stats (and schemes) unstructured data" do
+    subject << {weather: {temperature: 32.0}, score: {'Real' => 2, 'Barcelona' => 2}}
+    subject << {weather: "freezing", score: {'Real' => 1, 'Barcelona' => 0}}
+
+    expect(subject.schema).to(
+      eq "weather" => {"temperature" => 0.0}, "weather'" => "", "score" => {"Real" => 0, "Barcelona" => 0}
+    )
+    expect(subject.avg).to(
+      eq ".Hash.length" => 2.0, "weather.Hash.length" => 1.0, "weather.temperature.Float" => 32.0,
+         "score.Hash.length" => 2.0, "score.Real.Integer" => 1.5, "score.Barcelona.Integer" => 1.0, "weather'.String.length" => 8.0
+    )
+    Ssstats::CURRENTLY_AVAILABLE.each do |calc|
+      expect(subject.stat calc).to_not be_empty, "#{calc} be calculated"
+    end
+  end
+
+  it "collects useful statistics off simply (and loosely) structured data" do
+    subject << {did: '6978a678d79e', ts: 1234567890, sd: [{os: 123, hz: 55.66}, {os: 101, hz: 56.56}, {os: 77, hz: 54.45}, {os: 39, hz: 57.99}]}
+    subject << {did: '6978a678d79e', ts: 1234568901, sd: [{os: 104, hz: 53.56}, {os: 73, hz: 52.25}]}
+    subject << {did: '6978a678d79e', ts: 1234569012, sd: []}
+    subject << {did: '6978a678d79e', ts: 1234569999, sd: [{os: 0, error: 404}]}
+
+    expect(subject.schema).to(
+      eq 'did' => "", 'ts' => 0, 'sd' => [{'os' => 0, 'hz' => 0.0, 'error' => 0}]
+    )
+
+    expect(subject.std).to(
+      eq '.Hash.length' => 0.0, 'ts.Integer' => 746.7002410606281, 'sd.Array.length' => 1.479019945774904, 'sd.Array.Hash.length' => 0.0, 'sd.Array.os.Integer' => 39.17152235062235,
+         'sd.Array.hz.Float' => 1.9022391776243297, 'sd.Array.error.Integer' => 0.0, 'did.String.length' => 0.0
+    )
+  end
+
+  it "collects useful statistics off oddly structured data" do
+    subject << {uid: '6978a678d79e', ts: 1234567890, sdHz: [55.66, 56.56, 54.45, 57.99], sdOs: [123, 101, 77, 39]}
+    subject << {uid: '6978a678d79e', ts: 1234568901, sdHz: [53.56, 52.25], sdOs: [104, 73]}
+    subject << {uid: '6978a678d79e', ts: 1234569012, sdHz: [], sdOs: []}
+
+    expect(subject.schema).to(
+      eq 'uid' => "", 'ts' => 0, 'sdHz' => [0.0], 'sdOs' => [0]
+    )
+
+    expect(subject.std).to(
+      eq '.Hash.length' => 0.0, 'ts.Integer' => 504.79104587938167, 'sdHz.Array.length' => 1.632993161855452, 'sdHz.Array.Float' => 1.9022391776243297,
+         'sdOs.Array.length' => 1.632993161855452, 'sdOs.Array.Integer' => 27.00874344026804, 'uid.String.length' => 0.0
+    )
+  end
+end

data/ssstats.gemspec

@@ -0,0 +1,21 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'ssstats/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'ssstats'
+  spec.version       = Ssstats::VERSION
+  spec.authors       = ["Costa Shapiro", "Owlytics Healthcare (former BandManage)"]
+  spec.email         = ['costa@mouldwarp.com', 'contact@owlytics.com']
+  spec.summary       = "Stupid Simple Statistics (for unstructured data)"
+  spec.description   = "Consumes simple data structures, produces elementary statistics"
+  spec.homepage      = "https://github.com/bandmanage/ssstats"
+
+  spec.files         = `find . -print0`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+end

metadata

@@ -0,0 +1,208 @@
+--- !ruby/object:Gem::Specification
+name: ssstats
+version: !ruby/object:Gem::Version
+  version: 0.2.1
+platform: ruby
+authors:
+- Costa Shapiro
+- Owlytics Healthcare (former BandManage)
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-08-13 00:00:00.000000000 Z
+dependencies: []
+description: Consumes simple data structures, produces elementary statistics
+email:
+- costa@mouldwarp.com
+- contact@owlytics.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- "./.git/COMMIT_EDITMSG"
+- "./.git/FETCH_HEAD"
+- "./.git/HEAD"
+- "./.git/ORIG_HEAD"
+- "./.git/config"
+- "./.git/description"
+- "./.git/hooks/applypatch-msg.sample"
+- "./.git/hooks/commit-msg.sample"
+- "./.git/hooks/post-update.sample"
+- "./.git/hooks/pre-applypatch.sample"
+- "./.git/hooks/pre-commit.sample"
+- "./.git/hooks/pre-push.sample"
+- "./.git/hooks/pre-rebase.sample"
+- "./.git/hooks/pre-receive.sample"
+- "./.git/hooks/prepare-commit-msg.sample"
+- "./.git/hooks/update.sample"
+- "./.git/index"
+- "./.git/info/exclude"
+- "./.git/logs/HEAD"
+- "./.git/logs/refs/heads/master"
+- "./.git/logs/refs/heads/minor-name-changes"
+- "./.git/logs/refs/remotes/origin/basic-stats-basic-structs"
+- "./.git/logs/refs/remotes/origin/init"
+- "./.git/logs/refs/remotes/origin/master"
+- "./.git/logs/refs/remotes/origin/minor-name-changes"
+- "./.git/logs/refs/stash"
+- "./.git/objects/05/207ded3952d840ba340c85a6f6c96b22ca1c67"
+- "./.git/objects/06/61209566f12727ab3976b2a747ac4f1a470dbb"
+- "./.git/objects/07/0ef73c7375d71e0c339fc4aead20a94bdce9cd"
+- "./.git/objects/09/3a47fc9464be57d4e6cd75bdeca68529dfa266"
+- "./.git/objects/0b/13becb0672026236c3f08a689e114cabf3e822"
+- "./.git/objects/0b/b8805e9d0480adc26c683bf7a1bd3eb2665e63"
+- "./.git/objects/11/57e944f6067e26491cef7d109d112788bb191e"
+- "./.git/objects/1a/067207e458b64430a6a79e21886f343334b3fc"
+- "./.git/objects/1a/3673f33c56f4cc23a54287689848536642748e"
+- "./.git/objects/20/c351e1ee1bdc479818659d65258270c7d617a2"
+- "./.git/objects/20/d8723a745139ddfb87fd27e1178f88aaf0e93c"
+- "./.git/objects/24/8f7300de1cd8580c21601c27a8315d52adab40"
+- "./.git/objects/28/2eebe389bbf8cf3a8d3cf134122efaca57e269"
+- "./.git/objects/29/aa78fd51aa4d90e1735db9320829cebbc6f774"
+- "./.git/objects/2f/2a216e86ef1e3e0c17a18948e9ddfcb7824fbe"
+- "./.git/objects/2f/f19a188c1bb17bd2eb90c50c7a13310caa63e4"
+- "./.git/objects/32/a211703f0ae31329162799c7b44aaebf8cfef5"
+- "./.git/objects/33/c40c2e3408720afaa18f5a6ed09de12c7c56c6"
+- "./.git/objects/34/57a6355860054942ca5f713eba623b5add5a4b"
+- "./.git/objects/34/fe522e62971aa628f2d607e22a5df151d7594c"
+- "./.git/objects/36/8ffaf5d41725272eac6de6afe1d7a4abc6a4c8"
+- "./.git/objects/39/80ee7e8608ec095f751b30e4ccb4d4e3d2f063"
+- "./.git/objects/39/91d58f78b5ddb1216fe120b834b184822a0d77"
+- "./.git/objects/3b/55cedd0b036421b5cc56809db567a93652bd27"
+- "./.git/objects/42/ee185379834da4a07947cdabf2594d877ede05"
+- "./.git/objects/45/635f8c395eb61f91834254b3d6504ce99a0c59"
+- "./.git/objects/46/cba8f26c2c2d78c370601a75ebf431fcc128bb"
+- "./.git/objects/4b/4957b2c5b28c05f73bafb25c3779c8687acfa4"
+- "./.git/objects/4c/60da0b7246a7fa15887a4e0d70771d5f79361a"
+- "./.git/objects/4c/ca36fa8ee4f353e76fd3c835c93f88c9b3f894"
+- "./.git/objects/4d/d92d2a28eb8e40a86c3ff35fcde1e77dd51ecd"
+- "./.git/objects/4e/34db983e3d5bf8d6e38ce35b51ade3e72d5840"
+- "./.git/objects/51/418c10bd4cc59a50e23a740dfa16d5663fb5b9"
+- "./.git/objects/52/1691fc61347c01b77918b60ec1dc228cd0b6f3"
+- "./.git/objects/52/846381a80ccbb768e15771050dd9aea4b88f26"
+- "./.git/objects/55/bb31194c235e81c92bd1a6ac27235c038cd2fc"
+- "./.git/objects/57/4bd5b259664d1fc66dd90d88fc62f20e695cfe"
+- "./.git/objects/58/8b61fa66a0e289c2344d1a2c78edaab3da59e9"
+- "./.git/objects/5a/88ced498186dae91e7bf0065ab8948c2e2140a"
+- "./.git/objects/5d/be59d06419f3d3631db6097dbc62b5741a533e"
+- "./.git/objects/5d/fefea9b6eb5dbb84b561b7a196aa4c6d6aaa58"
+- "./.git/objects/5f/a988a61b5cd5eb62a1f33a8170e3d0cec76be8"
+- "./.git/objects/60/02dcf4e9b0e86485b11091469ef77f061be228"
+- "./.git/objects/61/626cd3240ac7cef12dae40aa6bbbd014ad31dd"
+- "./.git/objects/64/3c305e1d46a4969ddefabbfe59f9f20400817c"
+- "./.git/objects/67/301489447605605ba8f6816c6e1bac7ec58f3e"
+- "./.git/objects/68/75867b1f8adc25281dc23edfb9f70cfc2ad30d"
+- "./.git/objects/68/9db29b73abed60c8d0d63124898091062dd106"
+- "./.git/objects/69/8a251ce3adf8a0b32ba74b122e0c2f99173922"
+- "./.git/objects/6a/10a18940a6d0239fd4d72d01ae681759251be4"
+- "./.git/objects/6a/5392570ca6a567e7b261f9c5399fa0dc44a113"
+- "./.git/objects/6f/7243614d7ed0d0fa647639f196dc4485f9435c"
+- "./.git/objects/70/5da6fdb5474c4f92e3c65ddc7ced93dc618ac0"
+- "./.git/objects/74/6aa425712c6bbd485efaf9f95f2f73fa622702"
+- "./.git/objects/75/38d91a4499798974359a00507060121fedea77"
+- "./.git/objects/76/65fc5b4b82ea4f9aadd9672ce66bd6bbbadb0d"
+- "./.git/objects/76/db52c476f419150d6553de60eec15217526a01"
+- "./.git/objects/77/e9713b5406c687b6d38d0a4dc3ea3c4ad4ec9d"
+- "./.git/objects/78/11ce4804edd24700cde280e882392dd0de56e1"
+- "./.git/objects/80/bf6dbac2ae42557f6393c2e78ba81ad6e9c68b"
+- "./.git/objects/8c/18f1abdd4147f400c02b45e57297449ae4bc43"
+- "./.git/objects/8c/4d95b0d995eed356bfe4577315132d4bf8e882"
+- "./.git/objects/8c/713c7faea64564db38b030985b4486a5a71adf"
+- "./.git/objects/8e/b3b06b354e66d48de9d2f18fdc515ed52bafce"
+- "./.git/objects/8e/ec8161a1fb58c6c0dfa12e05cab2817f463efa"
+- "./.git/objects/8e/eec52ca9a4f714fa81b7c628914003717f2f4f"
+- "./.git/objects/8f/17d73858eb80d739567ea846611b293f04bc1f"
+- "./.git/objects/93/cb94306b63a2b3c9ffc36bc84ffc9fbc721777"
+- "./.git/objects/94/e461e0c8ecc3cec222e844b598992de14950f8"
+- "./.git/objects/96/04be05406705b325f60f22904fd7013f8d2bea"
+- "./.git/objects/97/84e27f7a46f6a67c93b4d795a427adf7fc1ccb"
+- "./.git/objects/97/990865fa520687678012c560585a4559eb01a0"
+- "./.git/objects/97/b4449d01443f80ea573c642012412b1913df93"
+- "./.git/objects/98/ba30921fd065c5054fc96966a1b2a2bf36d0f4"
+- "./.git/objects/9a/ca5762ffc9c77aa0ee9142e340cbf7882578be"
+- "./.git/objects/9b/191d4a8975dbc5b07dc9de0ee871d9a2ca4b3e"
+- "./.git/objects/a1/4155143d8e4d6cd0ffae8152fb0cd55f3a2a87"
+- "./.git/objects/a1/e0131ff3206a21075e8bff68bcc80d8d99e3a0"
+- "./.git/objects/ae/a7249864acee43f29aefa916f2c3df3d2b0741"
+- "./.git/objects/b2/385f78fb3d40feb046f478e6925363a14a7bde"
+- "./.git/objects/b3/dfa6ec25306eb895532ae999ba23dea16f90c9"
+- "./.git/objects/b4/cf7f5c04e871793943e616ae6aeeba54c12916"
+- "./.git/objects/b5/b44ab814dc2eef359699d58ee68a8fab4c72b1"
+- "./.git/objects/b5/ea8d2da4d4c15fbd0bdce8ee4c3ec031de086d"
+- "./.git/objects/b7/e9ed549be37412bd2f7e90f0b33165e3052ceb"
+- "./.git/objects/c1/ac00052cd7698f0bf2510530f87da6afebf278"
+- "./.git/objects/ca/15d31acd0eccf4137161c972e6af3beab7fefb"
+- "./.git/objects/cb/8bfa539de51e7585a3ce7f2069ada57eb66d9e"
+- "./.git/objects/ce/a706e0e64d2acc15d653ecc842fd072a15ffed"
+- "./.git/objects/cf/f50a5c7af66c069e935ab1d332e5420ffe7063"
+- "./.git/objects/d0/74fd4b00b5378d84885571802d1c214cab055e"
+- "./.git/objects/d1/85dd79a60fb633167a5f08c4c6d5839a30843f"
+- "./.git/objects/d2/a8d603e12dd80c1eea052ebf3173666f4ba1cb"
+- "./.git/objects/d8/991243827c6ce1382530580ee2271c293600aa"
+- "./.git/objects/d9/40caa2b379fbf0f1a6fced2993a284307b62ad"
+- "./.git/objects/db/d2a1661624b308c54897676bbfa26f407ee5a7"
+- "./.git/objects/dc/e67d860af47a4eb630117ce03624bae45dcf26"
+- "./.git/objects/de/0a2da315f811f866cc6008a0b71d0c9d94b33e"
+- "./.git/objects/df/ca81670614d58b337f023d669498db94ef9cf6"
+- "./.git/objects/e3/f666b411c3df9b90e4a27e0afefd0be386242f"
+- "./.git/objects/e5/0246cfa4aba2947e8ca440c796454ac0954e0c"
+- "./.git/objects/e9/46a801d541b4e1c734a0eff14928e1883df492"
+- "./.git/objects/ea/8caf1050a591b2965d5d06b8b95c877818cfcd"
+- "./.git/objects/eb/8f6c322b3786b1ce6063c74511c76d27115e0a"
+- "./.git/objects/eb/e52995c614ee2d781562bcfcf43983baf0a43a"
+- "./.git/objects/ef/b27738131cce265ab7aab4ec8670ba7c5d470a"
+- "./.git/objects/f4/2cc898a6b4d371cb01f695eb9bd56ca0f419e9"
+- "./.git/objects/f5/9ff20aeecbf2cae832e25b6506d20d54b55578"
+- "./.git/objects/f6/4c0df54b94bf55a1aa56abff0067728e2eb052"
+- "./.git/objects/f8/e8e644d5ad60ba7652767266ff73e60a7f0a6b"
+- "./.git/objects/fa/b7b630518fda529ee1f72a1800ac18e71a823c"
+- "./.git/objects/fc/c211363a3bbdd9bb2e059f52268820421084a2"
+- "./.git/refs/heads/master"
+- "./.git/refs/heads/minor-name-changes"
+- "./.git/refs/remotes/origin/basic-stats-basic-structs"
+- "./.git/refs/remotes/origin/init"
+- "./.git/refs/remotes/origin/master"
+- "./.git/refs/remotes/origin/minor-name-changes"
+- "./.git/refs/stash"
+- "./.gitignore"
+- "./.rspec"
+- "./.rspec_status"
+- "./.travis.yml"
+- "./Dockerfile"
+- "./Gemfile"
+- "./Gemfile.lock"
+- "./README.md"
+- "./Rakefile"
+- "./bin/console"
+- "./bin/setup"
+- "./docker-compose.yml"
+- "./exe/ssstats"
+- "./lib/ssstats.rb"
+- "./lib/ssstats/version.rb"
+- "./spec/spec_helper.rb"
+- "./spec/ssstats_spec.rb"
+- "./ssstats.gemspec"
+homepage: https://github.com/bandmanage/ssstats
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.8
+signing_key: 
+specification_version: 4
+summary: Stupid Simple Statistics (for unstructured data)
+test_files: []