ix-cli 0.0.10 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9356248fcc4f5fe5c83fedeb01e13166a9764bed70c39fe06132963141f394d9
-  data.tar.gz: c7676df3c6b0e8427888882bc02fa609e6aa987816f76ef90e656ca6ce196ed7
+  metadata.gz: d054eabc68581f7a85447e44ce7471bd3bfd361fec428cd6f52b07bc23d99f61
+  data.tar.gz: ef9b8158e1a5282d68956f9188f47d2e0753a7e42606b8fd48a8e684f2379d1a
 SHA512:
-  metadata.gz: 21233826e4b716857231aa89c0abc3263702cf0347b71a2bfc40e3502b79946b98a7b3ae5496bf1975121ef1f3c44952bedd0415d817ac7eee34246ec91007b6
-  data.tar.gz: fbee1010c9fda5caf7144b8855c6ff8f502413ba3dbee2b81c630340d6c429299878b9b20cc34bc6d9fe2912e3b5e4cee44159f0b31fc947d0467957f7cf6802
+  metadata.gz: 312c40f4c85f2c8919ddaa1ea33a9306dfc2be0eecf51782d18abaaabf4a5d1cc8fd00507674081ecf5598e694d9c28f4c31783b7416afd11a6d86024b096904
+  data.tar.gz: 4f7d68947f74467984e028a8d66460d662b8a32d57cc2e0072bb63454b325b3ce7abaa7600ea02f7a8bf64b4e6c8355bb1a51d8b5be8c0b5bf3d095e11716cee

data/bin/ix-json-stats

@@ -13,7 +13,12 @@ OptionParser.new do |opts|
   opts.banner = "Usage: #{$0} [OPTIONS]"
 
   opts.on('-k', '--keys [KEYS]', 'List of keys to parse from json separated by :.') do |value|
-    options[:keys] = value
+    unless value
+      $stderr.puts "Please provide valid --keys got: #{value.inspect}"
+      STDIN.each_line {}
+      exit 1
+    end
+    options[:keys] = value.split(':')
   end
 
   opts.on('-c', '--count [NUMBER]', 'Number of top N items to show.') do |value|
@@ -26,7 +31,7 @@ OptionParser.new do |opts|
 
 end.parse!
 
-required_options = [:keys]
+required_options = []
 required_options.each do |option|
   unless options[option]
     $stderr.puts "Can not run #{option.to_s} was not given."
@@ -35,13 +40,17 @@ required_options.each do |option|
 end
 
 map = {}
-keys = options[:keys].split(':')
 
 STDIN.each_line do |line|
   begin
     object = JSON.parse(line)
+    unless options[:keys]
+      options[:keys] == object.keys
+    end
     object.each do |k, v|
-      next unless keys.include?(k)
+      if options[:keys]
+        next unless options[:keys].include?(k)
+      end
       submap = map[k] ||= {}
       submap[v] ||= 0
       submap[v] += 1
@@ -53,36 +62,56 @@ STDIN.each_line do |line|
   end
 end
 
-def print_map(map, sample)
-  map.each do |category, stats|
-    values = []
-    total = 0
-
-    counter = 0
-    stats.values.sort.reverse.each do |value_1|
-      stats.each do |key, value_2|
-        if value_1 == value_2
-          counter += 1
-          break if counter > sample
-          total += value_1
-          values.push({ :value => value_1, :key => key })
-        end
-      end
+def print_value(result, total)
+  percentage = "%2.2f" % (result[:incidents] / total.to_f * 100)
+  h_percentage = (percentage.to_s + '%').rjust(6, ' ').to_ansi.yellow.to_s
+  value = result[:incidents].to_s.rjust(10, ' ').to_ansi.green.to_s
+  key = result[:label]
+  puts "%s %s %s" % [value, h_percentage, key]
+end
+
+class GroupObject
+  attr_accessor :category
+  attr_accessor :results
+  def initialize(category)
+    @category = category
+    @results = []
+  end
+  def add_result(label, incidents)
+    @results.push({ :label => label, :incidents => incidents })
+  end
+  def total
+    results.sum do |result|
+      result[:incidents]
     end
+  end
+end
 
-    puts ""
-    puts "#{category.to_s.to_ansi.cyan.to_s} (#{total})"
+def get_group_objects(map)
+  group_objects = []
 
-    values.each do |object|
-      percentage = "%2.2f" % (object[:value] / total.to_f * 100)
-      h_percentage = (percentage.to_s + '%').rjust(6, ' ').to_ansi.yellow.to_s
-      value = object[:value].to_s.rjust(10, ' ').to_ansi.green.to_s
-      key = object[:key]
-      puts "%s %s %s" % [value, h_percentage, key]
+  map.each do |category, hash|
+    group_object = GroupObject.new(category)
+    hash.each do |k, v|
+      group_object.add_result(k, v)
     end
+    group_objects.push(group_object)
+  end
 
+  group_objects.sort do |a, b|
+    a.category <=> b.category
   end
 end
 
-print_map(map, options[:count])
+def print_map(map, sample)
+  get_group_objects(map).each do |group_object|
+    puts "\n#{group_object.category.to_s.to_ansi.cyan.to_s} (#{group_object.total})"
+    group_object.results.sort do |a, b|
+      a[:incidents] <=> b[:incidents]
+    end.reverse.first(sample).each do |result|
+      print_value(result, group_object.total)
+    end
+  end
+end
 
+print_map(map, options[:count])

data/bin/ix-json-to-html-table-2

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'json'
+
+def tag(name, content)
+  "<#{name}>#{content}</#{name}>"
+end
+
+def row(name, array)
+  array.map do |value|
+    tag(name, value)
+  end * ' '
+end
+
+table_header = false
+
+puts '
+<style>
+tr:nth-child(even) {
+  background-color: whitesmoke;
+}
+th {
+    background: darkgray;
+    font-color: white;
+    padding: 1mm;
+}
+td {
+    padding: 1mm;
+}
+</style>
+'
+
+STDIN.each_line do |line|
+  begin
+    object           = JSON.parse(line)
+    unless table_header
+      puts '<table>'
+      table_header_row = row('th', object.keys.sort)
+      puts tag('tr', table_header_row)
+      table_header = true
+    end
+    copy = object.keys.sort.map do |key|
+      object[key]
+    end
+    table_body_row = row('td', copy)
+    puts tag('tr', table_body_row)
+  rescue => error
+  end
+end
+
+puts '</table>'

data/bin/ix-string-similarity

@@ -1,152 +1,222 @@
 #!/usr/bin/env ruby
 
-require 'optparse'
-require 'ostruct'
+class Chunk
+  attr_accessor :char_1
+  attr_accessor :char_2
+  attr_accessor :index_1
+  attr_accessor :index_2
+
+  def initialize(c1, c2, i1, i2)
+    @char_1 = c1
+    @char_2 = c2
+    @index_1 = i1
+    @index_2 = i2
+  end
 
-configuration = OpenStruct.new
+  def to_s
+    "#{char_1} #{char_2} #{index_1} #{index_2}"
+  end
+end
 
-OptionParser.new do |opts|
+class Similarity
+  CHAR_REGEX = /./
 
-  opts.banner = "Find similarity in a set of strings."
-  opts.separator ''
-  opts.separator "Usage: #{File.basename($0)} [OPTIONS]"
-  opts.separator ''
+  attr_accessor :string_1
+  attr_accessor :string_2
 
-  configuration.group = false
-  configuration.threshold = 50
-  configuration.summary = false
+  def initialize(string_1, string_2)
+    @string_1 = string_1
+    @string_2 = string_2
+  end
 
-  description = "Group input in batches and process each individually (faster)"
-  opts.on("-g", "--group", description) do |v|
-    configuration.group = v
+  def tokens
+    chunks = []
+    string_1.scan(CHAR_REGEX).each_with_index do |char_1, index_1|
+      string_2.scan(CHAR_REGEX).each_with_index do |char_2, index_2|
+        next if char_1 != char_2
+        chunks.push(Chunk.new(char_1, char_2, index_1, index_2))
+      end
+    end
+    chunks
   end
 
-  description = "Limit the number of results by threshold, default is 50"
-  opts.on("-t", "--threshhold [NUMBER]", OptionParser::DecimalNumeric, description) do |v|
-    configuration.threshold = v
+  def count
+    counter = 0
+    prev = false
+    tokens.each_with_index do |chunk, index|
+
+      unless prev
+        prev = chunk.index_1
+        next
+      end
+
+      if prev == (chunk.index_1 - 1)
+        counter += 1
+      end
+
+      prev = chunk.index_1
+    end
+
+    counter
   end
 
-  description = "Print a Summary of the groups found"
-  opts.on("-s", "--summary", description) do |v|
-    configuration.summary = v
+  def score
+    desired = (string_1.size + string_2.size) / 2
+    size_thresh = ([string_1.size, string_2.size].sort.first.to_f / desired)
+    compatibility_thresh = (count.to_f + 1) / string_1.size
+    (size_thresh + compatibility_thresh).to_f / 2
   end
+end
+
+require 'optparse'
+
+options = {}
+options[:threshold] = 0.8
 
+OptionParser.new do |opts|
 
-  opts.separator ''
+  opts.banner = "Usage: #{$0} [OPTIONS]"
+
+  opts.on('-t', '--threshold [NUMBER]', 'Threshold default value is 0.8.') do |value|
+    options[:threshold] = value.to_f
+  end
+
+  opts.on('-s', '--summary', 'Print Summarized version.') do |value|
+    options[:summary] = value
+  end
 
 end.parse!
 
-# puts configuration.inspect
+required_options = [:threshold]
+required_options.each do |option|
+  unless options[option]
+    $stderr.puts "Can not run #{option.to_s} was not given."
+    exit 1
+  end
+end
+
+# hash = {
+#   'line' => [
+#     { :line => 'line', :score => 1 },
+#   ]
+# }
+
+hash = {}
+lines = 0
 
-class Array
-  def product
-    inject do |cumulative, value|
-      cumulative += value
+STDIN.each_line do |line|
+  line.chomp!
+  next if line == ''
+  lines += 1
+  resolved = false
+  hash.keys.each do |registered_line|
+    score = Similarity.new(line, registered_line).score
+    if score > options[:threshold]
+      hash[registered_line].push({
+        :line => line,
+        :score => score
+      })
+      resolved = true
     end
   end
+  next if resolved
+  hash[line] ||= []
 end
 
-class String
-  def to_a
-    array = []
-    size.times do |n|
-      array << self[n]
+module Template
+
+  class Banner < Struct.new(:lines, :groups, :threshold, :datetime)
+    def to_s
+      format(template, to_h)
+    end
+    def template
+      '
+Total Lines Parsed: %<lines>s
+Total Groups Generated: %<groups>s
+Similarity Theshold at: %<threshold>s
+Generated on: %<datetime>s
+      '
     end
-    array
   end
 
-  def scores(other_string)
-    longest_string = nil
-    if other_string.size > self.size
-      longest_string = other_string
-      shortest_string = self
-    else
-      longest_string = self
-      shortest_string = other_string
-    end
-    scores = longest_string.to_a.map do |char|
-      0
+  class Group < Struct.new(:number, :percent, :items, :line)
+    def to_s
+      format(template, to_h)
     end
-    shortest_string.size.times do |index|
-      if shortest_string[index] == longest_string[index]
-        scores[index] = 1
-      end
+    def template
+      'Group %<number>s represents %<percent>s and has %<items>s items similar to: %<line>s'
     end
-    scores
   end
 
-  def similarity(other_string)
-    scores(other_string).product * 100.0 / size
+  class Item < Struct.new(:count, :total, :score, :line)
+    def to_s
+      format(template, to_h)
+    end
+    def template
+      '  %<count>s/%<total>s %<score>s  %<line>s'
+    end
   end
-end
 
-class TargetString
-  attr_accessor :evaluated
-  attr_accessor :data
-  def to_s
-    data
-  end
 end
 
-# client
+require 'isna'
 
-strings = []
+summary_output = []
+detailed_output = []
 
-STDIN.each_line do |line|
-  next if line.chomp == ''
-  strings << line.chomp
-end
+banner = Template::Banner.new
+banner.lines = lines.to_s.to_ansi.yellow.to_s
+banner.groups = hash.keys.size.to_s.to_ansi.yellow.to_s
+banner.threshold = options[:threshold].to_s.to_ansi.yellow.to_s
+banner.datetime = Time.now.to_s.to_ansi.yellow.to_s
+summary_output.push(banner.to_s)
 
-strings.sort! do |n1, n2|
-  n1.size <=> n2.size
+groups = []
+
+hash.each do |category_name, records|
+  groups.push([category_name, records.size])
 end
-  
-strings.reverse!
-
-strings.map! do |string|
-  target_string = TargetString.new
-  target_string.evaluated = false
-  target_string.data = string
-  target_string
+
+sorted_groups_by_n_records_asc = groups.sort do |array_a, array_b|
+  number_of_records_in_a = array_a[1]
+  number_of_records_in_b = array_b[1]
+  number_of_records_in_a <=> number_of_records_in_b
 end
 
-if configuration.group
-  groups = strings.group_by do |string|
-    string.data.size
+sorted_groups_by_n_records_asc.reverse.each_with_index do |key, index|
+  line, records = key[0], hash[key[0]]
+
+  detailed_output.push('')
+
+  group = Template::Group.new
+  group.percent = ('%2.2f%%' % ((records.size.to_f / lines) * 100)).to_s.to_ansi.red.to_s
+  group.number = (index + 1).to_s.to_ansi.red.to_s
+  group.items = records.size.to_s.to_ansi.cyan.to_s
+  group.line = line.chomp.to_ansi.green.to_s
+  summary_output.push(group.to_s)
+  detailed_output.push(group.to_s)
+
+  sorted_items_in_group = records.sort do |a, b|
+    a[:score] <=> b[:score]
   end
-else
-  groups = { 0 => strings }
-end
 
-counter = 0
-groups.each do |key, group|
-  group.each do |string_1|
-    counter = 0
-    unless string_1.evaluated
-      if configuration.summary
-        summary_string = string_1.to_s
-      else
-        puts "****>>" + string_1.to_s
-      end
-    end
-    string_1.evaluated = true
-    group.each do |string_2|
-      next if string_2.evaluated
-      similarity = string_1.to_s.similarity(string_2.to_s)
-      scores = string_1.to_s.scores(string_2.to_s).inspect
-      template = "%5.f %s"
-      bindings = [similarity, string_2, scores]
-      if similarity >= configuration.threshold
-        string_2.evaluated = true
-        counter += 1
-        unless configuration.summary
-          puts template % bindings
-        end
-      end
-    end
-    if counter > 0
-      puts "#{counter} #{summary_string}"
-    end
+  sorted_items_in_group.reverse.each_with_index do |record, index|
+    item = Template::Item.new
+    item.count = (index + 1).to_s.rjust(4, ' ').to_ansi.cyan.to_s
+    item.total = records.size.to_s.ljust(4, ' ').to_ansi.cyan.to_s
+    item.score = ('%4.2f%%' % (record[:score] * 100)).rjust(7, ' ').to_ansi.green.to_s
+    item.line = record[:line]
+    detailed_output.push(item.to_s)
   end
+
 end
 
+summary_output.each do |output_line|
+  puts output_line
+end
+
+unless options[:summary]
+  detailed_output.each do |output_line|
+    puts output_line
+  end
+end

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: ix-cli
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.14
 platform: ruby
 authors:
 - Kazuyoshi Tlacaelel
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
 date: 2020-02-05 00:00:00.000000000 Z
@@ -24,8 +24,8 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.0.4
-description:
-email:
+description: 
+email: 
 executables:
 - ix
 - ix-acronym
@@ -151,6 +151,7 @@ executables:
 - ix-json-to-csv
 - ix-json-to-dot
 - ix-json-to-html-table
+- ix-json-to-html-table-2
 - ix-json-to-ruby-hash
 - ix-json-to-table
 - ix-json-to-table-2
@@ -431,6 +432,7 @@ files:
 - bin/ix-json-to-csv
 - bin/ix-json-to-dot
 - bin/ix-json-to-html-table
+- bin/ix-json-to-html-table-2
 - bin/ix-json-to-ruby-hash
 - bin/ix-json-to-table
 - bin/ix-json-to-table-2
@@ -582,10 +584,10 @@ files:
 - bin/ix-wrap
 - bin/ix-xy
 - bin/ix-zebra
-homepage:
+homepage: 
 licenses: []
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -600,8 +602,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: ix - string manipulation tools
 test_files: []