trials 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 303d61afdd0937269e514488c1462361d6bf24a7ade281e65d3aae0270e9ae31
-  data.tar.gz: e0e8e292b51e7166af634fdf2e6117ded4aa700567aa197280dfc35f8de96f27
+  metadata.gz: 8f8af3df6ab16bb8ba4b08d0ea7b1dd11507aeb2c6113a0598a5e9965d0264c9
+  data.tar.gz: c1e9cc7305f5cee913a7091d321811153d33e9e67d383bad0f1518034a195556
 SHA512:
-  metadata.gz: 83503c1320739905b44c54e3bf31b469435562a508cbae1670749e0eeba6cc439a7c3f53b4a15389eeebbf8c3218473c778befbb05a2b75499b7904a0d07ddad
-  data.tar.gz: ae364df5de6484dc9c461838b809d24644202e6979a65c90e206a95e60dabe755a08d695f1a7aa042fb490c52193dbfbf928404f83f75a6bf70499cf93390a54
+  metadata.gz: d4bdc989741b4f86a3b80f91979a3882c2666da4749ee7864f0b45596311cbdebd297e17950d27bb84d39efc6d30b16fa18fc1a1bd4f65b98cd213ec4e0cbba7
+  data.tar.gz: d3fa929a7157792cd5fce04b64f0d0312ce240f59de755d323dc1ee910a21f9a966e74342a5e4c0d05d732291ab28a7b7a324bad6a5e1afb3c0293ea0c682c1c

data/bin/trial

@@ -1,12 +1,13 @@
 #!/usr/bin/env ruby
 
-require_relative '../lib/trials'
-
 # env
 START_TIME = Time.now
 SCRIPT_NAME = ARGV[0].gsub('.rb', '')
 RUN = "#{START_TIME.strftime("%Y%m%dT%H%M%S")}_#{SCRIPT_NAME}"
 ROOT = Dir.pwd
+
+require_relative '../lib/trials'
+
 SECRETS = begin
   if File.exists?(File.expand_path('~/.trials/secrets.yml'))
     JSON.parse(
@@ -22,6 +23,10 @@ end
 FileUtils.mkdir_p("seeds")
 FileUtils.mkdir_p("results/#{RUN}")
 FileUtils.mkdir_p("tmp")
+FileUtils.mkdir_p("cache")
+
+# allow to init the current folder
+return if SCRIPT_NAME == 'init'
 
 # log start
 log "starting #{SCRIPT_NAME} at #{START_TIME.iso8601}"

data/lib/trials.rb

@@ -10,18 +10,18 @@ require 'nokogiri'
 require 'ostruct'
 require 'pp'
 require 'pry'
+require 'rack'
 require 'rest-client'
+require 'securerandom'
 require 'set'
 require 'smalltext'
 require 'sqlite3'
 require 'street_address'
 require 'tty-table'
 require 'yaml'
-require 'rack'
 
 require_relative 'trials/utils/various'
 require_relative 'trials/utils/csvs'
-require_relative 'trials/utils/hashes'
 require_relative 'trials/utils/logging'
 require_relative 'trials/utils/strings'
 require_relative 'trials/utils/files'
@@ -34,7 +34,5 @@ require_relative 'trials/utils/rollbar'
 require_relative 'trials/utils/aws'
 require_relative 'trials/utils/google_drive'
 require_relative 'trials/utils/h_array'
-require_relative 'trials/data_handling/addresses'
-require_relative 'trials/data_handling/names'
 require_relative 'trials/extensions/hash'
 require_relative 'trials/extensions/array'

data/lib/trials/utils/aws.rb

@@ -6,7 +6,7 @@ def ddb_connection
   )
 end
 
-def cached_ddb_scan(query)
+def ddb_scan_with_cache(query)
   json_cache(query.dig(:table_name)) { ddb_scan(query) }
 end
 
@@ -31,7 +31,6 @@ end
 
 def ddb_scan_without_segmentation(query)
   result = nil
-  requests = 0
   items = []
 
   loop do

data/lib/trials/utils/csvs.rb

@@ -1,15 +1,15 @@
 def read_csv(filename)
   CSV
-    .foreach(seeds_path(filename), headers: true)
+    .foreach(seed_path(filename), headers: true)
     .map(&:to_h)
     .map(&:symbolize_keys)
     .select { |i| i.values.any?(&:present?) }
 end
 
 def write_csv_from_hashes(file, hash_set, attrs: nil)
-  attrs ||= uniq_hash_keys(hash_set)
+  attrs ||= hash_set.to_harray.uniq_keys
 
-  CSV.open(results_path(file), 'w') do |csv|
+  CSV.open(result_path(file), 'w') do |csv|
     csv << attrs
 
     hash_set.each do |c|

data/lib/trials/utils/files.rb

@@ -1,110 +1,72 @@
-# paths
-
-def seeds_path(name)
-  "#{ROOT}/seeds/#{name}"
-end
-
-alias seed_path seeds_path
-
-def seed_exists?(name)
-  File.exist?(seed_path(name))
-end
-
-alias seeds_exist? seed_exists?
-
-def results_path(name)
-  "#{ROOT}/results/#{RUN}/#{name}"
-end
-
-alias result_path results_path
-
-def result_exists?(name)
-  File.exist?(result_path(name))
-end
-
-alias results_exist? result_exists?
-
-def tmp_path(name)
-  "#{ROOT}/tmp/#{name}"
-end
-
-def tmp_exists?(name)
-  File.exist?(tmp_path(name))
-end
-
-def list_dir(dir)
-  Dir["#{seeds_path(dir)}/**/*"]
-end
-
-# reading
-
-def read(file)
-  return unless seed_exists?(file)
-  File.read(seeds_path(file))
-end
-
-alias read_seed read
-alias read_seeds read
-
-def read_tmp(file)
-  return unless tmp_exists?(file)
-  File.read(tmp_path(file))
-end
-
-def readlines(file)
-  File.read(seeds_path(file)).split("\n")
-end
-
-# writing
-
-def write(file, content)
-  FileUtils.mkdir_p(File.dirname(results_path(file)))
-  File.open(results_path(file), 'w') { |f| f << content }
-end
-
-alias write_result write
-alias write_results write
-
-def write_tmp(file, content)
-  FileUtils.mkdir_p(File.dirname(tmp_path(file)))
-  File.open(tmp_path(file), 'w') { |f| f << content }
-end
-
-def append(file, content)
-  File.open(results_path(file), 'a') { |f| f << content }
-end
-
-alias append_result append
-alias append_results append
-
-# deleting
-
-def delete(file)
-  return unless result_exists?(file)
-  FileUtils.rm_r(results_path(file))
-end
-
-alias delete_result delete
-alias delete_results delete
-
-def delete_tmp(file)
-  return unless tmp_exists?(file)
-  FileUtils.rm_r(tmp_path(file))
-end
-
-def delete_seeds(file)
-  return unless seed_exists?(file)
-  FileUtils.rm_r(seeds_path(file))
-end
-
-alias delete_seed delete_seeds
-
-# other
-
-def make_seed(file)
-  FileUtils.cp(results_path(file), seeds_path(file))
-end
-
-def make_tmp(file)
-  FileUtils.cp(results_path(file), tmp_path(file))
+dirs = {
+  seed: { run: false, pluralized: ActiveSupport::Inflector.pluralize('seed') },
+  result: { run: true, pluralized: ActiveSupport::Inflector.pluralize('result') },
+  tmp: { run: false, pluralized: 'tmp' },
+  cache: { run: false, pluralized: 'cache' },
+}
+
+dirs.each do |dir, opts|
+  eval <<~RUBY
+    def #{opts.dig(:pluralized)}_root
+      File.join(
+        ROOT,
+        '#{opts.dig(:pluralized)}',
+        '#{opts.dig(:run) ? RUN : nil}',
+      ).to_s
+    end
+
+    def #{dir}_path(name)
+      File.join(
+        #{opts.dig(:pluralized)}_root,
+        name,
+      ).to_s
+    end
+
+    def #{dir}_exists?(name)
+      File.exist?(#{dir}_path(name))
+    end
+
+    def list_#{opts.dig(:pluralized)}(name = nil)
+      Dir[
+        File.join(
+          *[
+            #{opts.dig(:pluralized)}_root,
+            name,
+            '**',
+            '*',
+          ].compact
+        )
+      ].reject { |d| File.directory?(d) }
+    end
+
+    def read_#{dir}(name)
+      File.read(#{dir}_path(name)) if #{dir}_exists?(name)
+    end
+
+    def readlines_#{dir}(name)
+      read_#{dir}(name).split("\\n")
+    end
+
+    def write_#{dir}(name, content)
+      FileUtils.mkdir_p(File.dirname(#{dir}_path(name)))
+      File.open(#{dir}_path(name), 'w') { |f| f << content }
+    end
+
+    def append_to_#{dir}(name, content)
+      File.open(#{dir}_path(name), 'a') { |f| f << content }
+    end
+
+    def delete_#{dir}(name)
+      FileUtils.rm_r(#{dir}_path(name)) if #{dir}_exists?(name)
+    end
+  RUBY
+
+  dirs.each do |o_dir, o_opts|
+    next if o_dir == dir
+    eval <<~RUBY
+      def cp_#{o_dir}_to_#{opts.dig(:pluralized)}(name)
+        FileUtils.cp(#{o_dir}_path(name), #{dir}_path(name))
+      end
+    RUBY
+  end
 end

data/lib/trials/utils/google_drive.rb

@@ -1,7 +1,7 @@
 def gd_session
   @gd_session ||= begin
-    write('config.json', secrets.google.drive_config_json)
-    session = GoogleDrive::Session.from_config(results_path("config.json"))
+    write_tmp('config.json', secrets.google.drive_config_json)
+    session = GoogleDrive::Session.from_config(tmp_path("config.json"))
     delete('config.json')
     session
   end

data/lib/trials/utils/h_array.rb

@@ -20,6 +20,14 @@ class HArray < Array
     end
   end
 
+  def normalize_keys
+    hashes.map do |h|
+      h
+        .map { |k, v| [normalize_key(k), v] }
+        .to_h
+    end
+  end
+
   def uniq_keys
     hashes.flat_map(&:keys).uniq.compact
   end
@@ -49,4 +57,10 @@ class HArray < Array
         .reduce(&:merge)
     end
   end
+
+  private
+
+  def normalize_key(k)
+    k.downcase.to_sym
+  end
 end

data/lib/trials/utils/jsons.rb

@@ -1,5 +1,5 @@
 def read_json(filename)
-  result = JSON.parse(read(filename))
+  result = JSON.parse(read_seed(filename))
 
   aggressive_deep_symbolize_keys(result)
 end
@@ -11,12 +11,12 @@ end
 def json_cache(key)
   name = "json/#{key}.json"
 
-  return aggressive_deep_symbolize_keys(JSON.parse(read_tmp(name))) if tmp_exists?(name)
+  return aggressive_deep_symbolize_keys(JSON.parse(read_cache(name))) if cache_exists?(name)
 
-  write_tmp(name, yield.to_json)
+  write_cache(name, yield.to_json)
   json_cache(key)
 end
 
 def invalidate_json_cache
-  delete_tmp("json")
+  delete_cache("json")
 end

data/lib/trials/utils/logging.rb

@@ -1,7 +1,7 @@
 def render_table_from_hashes(hash_set, sort: true, headers: nil)
   return 'no data' if hash_set.blank?
 
-  headers = headers || uniq_hash_keys(hash_set)
+  headers = headers || hash_set.to_harray.uniq_keys
 
   headers.sort! if sort
 
@@ -21,7 +21,7 @@ def log(item, nl: true, quiet: false, each: true)
     return
   end
 
-  File.open(results_path('log.txt'), 'a') do |f|
+  File.open(result_path('log.txt'), 'a') do |f|
     f << begin
       if item.is_a?(String) || item.is_a?(Numeric)
         item.to_s

data/lib/trials/utils/pdfs.rb

@@ -1,4 +1,4 @@
 def pdf_to_text(path)
-  `pdftotext "#{seeds_path(path)}" #{tmp_path('tmp_pdf.txt')}; \
+  `pdftotext "#{seed_path(path)}" #{tmp_path('tmp_pdf.txt')}; \
   cat #{tmp_path('tmp_pdf.txt')}`
 end

data/lib/trials/utils/sqls.rb

@@ -1,6 +1,6 @@
 def db(db_name = nil)
   db_name ||= 'data.db'
-  db = SQLite3::Database.new(tmp_path(db_name))
+  db = SQLite3::Database.new(cache_path(db_name))
   db.results_as_hash = true
   db
 end
@@ -14,7 +14,7 @@ def get_db(db_name = nil)
 end
 
 def import_csv_into_db(db_name = 'data.db', table, csv)
-  system("sqlite3 -csv #{tmp_path(db_name)} '.import #{seed_path(csv)} #{table}'")
+  system("sqlite3 -csv #{cache_path(db_name)} '.import #{seed_path(csv)} #{table}'")
 end
 
 def query_db(db_name = nil, query)

data/lib/trials/utils/various.rb

@@ -2,6 +2,14 @@ def secrets
   SECRETS
 end
 
+def run
+  RUN
+end
+
+def root_path
+  ROOT
+end
+
 def or_nil
   val = yield
   raise if val.blank?
@@ -25,3 +33,13 @@ def aggressive_deep_symbolize_keys(maybe)
 
   maybe
 end
+
+def marshal_fetch(key)
+  return Marshal.load(read_cache(key)) if cache_exists?(key)
+
+  File.open(cache_path(key), 'wb') do |f|
+    f.write(Marshal.dump(yield))
+  end
+
+  marshal_fetch(key)
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: trials
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - grahamotte
@@ -299,8 +299,6 @@ extra_rdoc_files: []
 files:
 - bin/trial
 - lib/trials.rb
-- lib/trials/data_handling/addresses.rb
-- lib/trials/data_handling/names.rb
 - lib/trials/extensions/array.rb
 - lib/trials/extensions/hash.rb
 - lib/trials/utils/aws.rb
@@ -309,7 +307,6 @@ files:
 - lib/trials/utils/files.rb
 - lib/trials/utils/google_drive.rb
 - lib/trials/utils/h_array.rb
-- lib/trials/utils/hashes.rb
 - lib/trials/utils/jsons.rb
 - lib/trials/utils/logging.rb
 - lib/trials/utils/pdfs.rb

data/lib/trials/data_handling/addresses.rb

@@ -1,37 +0,0 @@
-# normal format <number> <STREET NAME>, <CITY>, <STATE> <postal>
-
-def normalize_address(address)
-  return if address.blank?
-
-  cleaner_string = if address.respond_to?(:address1)
-    "#{address.address1}, #{address.city}, #{address.state} #{address.postal_code}"
-  else
-    address
-  end
-
-  cleaner_string = cleaner_string.gsub('#', ' #').squish
-  parsed = StreetAddress::US.parse(cleaner_string)
-
-  return if parsed.blank?
-  return if parsed.number.blank?
-  return if parsed.street.blank?
-  return if parsed.city.blank?
-  return if parsed.state.blank?
-  return if parsed.postal_code.blank?
-
-  parsed.prefix = nil
-  parsed.suffix = nil
-  parsed.unit_prefix = nil
-  parsed.unit = nil
-  parsed.postal_code_ext = nil
-
-  parsed.to_s.upcase
-end
-
-def parse_address(address_string)
-  StreetAddress::US.parse(address_string)
-end
-
-def normalize_and_parse_address(address_string)
-  StreetAddress::US.parse(normalize_address(address_string))
-end

data/lib/trials/data_handling/names.rb

@@ -1,84 +0,0 @@
-# normal <FIRST> <LAST>
-
-Name = Struct.new(:first_name, :middle_name, :last_name)
-
-class NamePartsParser
-  def initialize(name_string)
-    @namae = Namae.parse((name_string || '').upcase).first
-  end
-
-  def first
-    normalize_name(split_first_and_middle(given).first)
-  end
-
-  def middle
-    normalize_name(split_first_and_middle(given).last)
-  end
-
-  def last
-    normalize_name(family)
-  end
-
-  private
-
-  def given
-    @namae&.given || ''
-  end
-
-  def family
-    @namae&.family || ''
-  end
-
-  def split_first_and_middle(first_and_middle)
-    names = first_and_middle.split(' ', 2)
-    names.length == 1 ? names + [''] : names
-  end
-end
-
-def normalize_name(name)
-  return if name.blank?
-
-  name = name_from_parts(name) if name.respond_to?(:first_name)
-
-  name.strip.upcase.delete('^A-Z\ \-').squeeze(" ")
-end
-
-def normalize_full_names(names)
-  Array.wrap(names)
-    .map { |n| normalize_full_name(n) }
-    .map(&:to_s)
-    .map(&:presence)
-    .compact
-    .uniq
-end
-
-def normalize_full_name(name)
-  return if name.blank?
-
-  name.strip.upcase.delete('^A-Z\ \-').squeeze(" ")
-end
-
-def parse_name(name)
-  NamePartsParser.new(name)
-end
-
-def normalize_and_parse_name(name)
-  NamePartsParser.new(normalize_name(name))
-end
-
-# private
-
-def name_from_parts(name)
-  first = normalize_name_part(name.first_name)
-  last = normalize_name_part(name.last_name)
-
-  [first, last].join(' ')
-end
-
-def full_name_from_parts(name)
-  first = normalize_name_part(name.first_name)
-  middle = normalize_name_part(name.middle_name)
-  last = normalize_name_part(name.last_name)
-
-  [first, middle, last].join(' ')
-end

data/lib/trials/utils/hashes.rb

@@ -1,90 +0,0 @@
-def uniq_hash_keys(hashes)
-  hashes.flat_map(&:keys).uniq.compact
-end
-
-def sanitize_hash_value(hash, key:, type:, date_format: '%Y-%m-%d')
-  hash.merge(
-    key => or_nil do
-      case type
-      when :date
-        Date.strptime(hash.dig(key), date_format)
-      when :datetime
-        DateTime.parse(hash.dig(key))
-      when :integer, :int
-        hash.dig(key).to_i
-      when :float
-        hash.dig(key).to_f
-      when :string
-        hash.dig(key).to_s
-      when :alphanum
-        string_to_alphanum(hash.dig(key))
-      when :present?
-        hash.dig(key).present?
-      end
-    end
-  )
-end
-
-def sanitize_hash_values(hash, scheme = {})
-  scheme.each do |k, v|
-    hash = sanitize_hash_value(hash, key: k, type: v)
-  end
-
-  hash
-end
-
-def rename_hash_key(hash, from:, to:)
-  hash[to] = hash.delete(from)
-  hash
-end
-
-def rename_hash_keys(hash, scheme = {})
-  scheme.each do |k, v|
-    hash = rename_hash_key(hash, from: k, to: v)
-  end
-
-  hash
-end
-
-def merge_hash_groups(*groups, key:, join_type: :inner)
-  groups = groups.map { |group| group.map { |g| [g.dig(key), g] }.to_h }
-
-  keys = begin
-    case join_type
-    when :inner
-      groups.map(&:keys).reduce(&:&)
-    when :all
-      groups.flat_map(&:keys).uniq
-    when :first
-      groups.first.keys
-    end
-  end
-
-  keys.map { |key| groups.map { |g| g.dig(key) }.compact.reduce(&:merge) }
-end
-
-def count_for_group_by(batch, &block)
-  batch
-    .group_by(&block)
-    .map { |k, v| [k, v.length] }
-    .to_h
-end
-
-def array_to_count_hash(list)
-  list.uniq.reduce({}) do |h, i|
-    h[i] = list.count(i)
-    h
-  end
-end
-
-def update_counts_hash(counts, update)
-  update.each do |k, v|
-    if counts.key?(k)
-      counts[k] += v
-    else
-      counts[k] = v
-    end
-  end
-
-  counts
-end