aac-metrics 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b00b87081818fb1fe8ff225b6b7120cc8bb26ba162d84c5a8c74f2e2571f6e84
-  data.tar.gz: 7115ff9b2081671fa60e3538e706b61deb3a8bb35bbf1b822c74cc90e4cd57eb
+  metadata.gz: 5cfa37f885dec40c12ac7cbb86c44ac49ceec1138866b3de2e52b84facdddb87
+  data.tar.gz: e29f98ec3c9aa3840a513787003a5db196dd09b81699c3a81b252439929e4dcd
 SHA512:
-  metadata.gz: 93eeae92b189182a457ca487a75327cce7aff50912f08a7152aa3b9715335bba80940687916182cde9421b249425ecfcf2f086954cf8f51db7dbef31013849d5
-  data.tar.gz: 99eab7e060552fd6d11b80016aa9f9f687c59d5a445f4eddb039e61cb39aa35b4dcb0e9307efff51c13dd249ff6996300213b4d1aec4f509a3150e708122eac2
+  metadata.gz: 3e54f361e67a70f8c68f85469ffcc0c9da6c8269928b707b2e6e4b5c353e5bc19f64de3d93d50c7c37304822924bf8ca862e35e3f6a4ed692ad2781c444403bb
+  data.tar.gz: c12274971262bf327f9d2e6c29286bd1147fed6e5cf3d7a022a869d9810669e796fff20bfabcf13b3e973984a480a936e33468c4d4b2b7ad8f82737bf4b1b33d

data/lib/aac-metrics/loader.rb

@@ -78,6 +78,7 @@ module AACMetrics::Loader
     words_path = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'sets', "base_words"))
     words = nil
     do_ingest = true
+    relations_hash = {}
     
     while paths.length > 0
       path = paths.shift
@@ -130,17 +131,7 @@ module AACMetrics::Loader
                 "id" => "btn#{btn_idx}",
                 "label" => (btn['vocalization'] || '').length > 0 ? btn['vocalization'] : btn['label']
               }
-              if do_ingest && new_btn['label']
-                str = new_btn['label'].downcase.sub(/^\s+/, '').sub(/\s+$/, '')
-                if str.scan(/\s+/).length < 2
-                  word_hash = Digest::MD5.hexdigest(str)[0, 10]
-                  raise "collision!" if words[word_hash] && words[word_hash] != str
-                  if add_words || words[word_hash]
-                    words[word_hash] = str
-                    new_btn['label'] = "$#{word_hash}"
-                  end
-                end
-              end
+              # record load_board reference
               btn_idx += 1
               if btn['load_board']
                 if btn['load_board']['path']
@@ -167,6 +158,36 @@ module AACMetrics::Loader
                 # treat action buttons for metrics
                 new_btn = nil
               end
+              # temporarily save semantic_id and possible clone_id for later use
+              # 1. Buttons in the same location with the same
+              # semantic_id should be marked in the obfset as having
+              # the same semantic_id
+              # 2. Buttons in the same location with the same label & voc
+              # and same load_board setting
+              # should be marked in the obfset as having the same clone_id
+              ref = "#{new_json['grid']['rows']}x#{new_json['grid']['columns']}-#{row_ids}.#{col_id}"
+              if btn['semantic_id']
+                relations_hash["s#{ref}-#{btn['semantic_id']}"] ||= []
+                relations_hash["s#{ref}-#{btn['semantic_id']}"] << [new_json['id'], new_btn['id']]
+              end
+              if new_btn['label']
+                # TODO: currently doesn't enforce same-location on links, just whether it's a linked button or not
+                pre = new_btn['load_board'] ? 'cl' : 'c'
+                relations_hash["#{pre}#{ref}-#{new_btn['label']}"] ||= []
+                relations_hash["#{pre}#{ref}-#{new_btn['label']}"] ||= [new_json['id'], new_btn['id']]
+              end
+              if do_ingest && new_btn['label']
+                str = new_btn['label'].downcase.sub(/^\s+/, '').sub(/\s+$/, '')
+                if str.scan(/\s+/).length < 2
+                  word_hash = Digest::MD5.hexdigest(str)[0, 10]
+                  raise "collision!" if words[word_hash] && words[word_hash] != str
+                  if add_words || words[word_hash]
+                    words[word_hash] = str
+                    new_btn['label'] = "$#{word_hash}"
+                  end
+                end
+              end
+
             end
             new_row.push(new_btn ? new_btn['id'] : nil)
             new_json['buttons'].push(new_btn) if new_btn
@@ -176,6 +197,29 @@ module AACMetrics::Loader
         boards << new_json
       end
     end
+    # any semantic_id or clone_id repeats must be recorded
+    relations_hash.each do |id, btns|
+      if btns && btns.length > 0
+        btns.each do |brd_id, btn_id|
+          brd = boards.detect{|b| b['id'] == brd_id }
+          if brd && brd['buttons']
+            btn = brd['buttons'].detect{|b| b['id'] == btn_id }
+            if btn
+              if id.match(/^s/)
+                btn['semantic_id'] = id
+                brd['semantic_ids'] ||= []
+                brd['semantic_ids'] << id
+              elsif id.match(/^c/)
+                btn['clone_id'] = id
+                brd['clone_ids'] ||= []
+                brd['clone_ids'] << id
+              end
+            end
+          end
+        end
+        # 
+      end
+    end
     boards.each do |brd|
       brd['buttons'].each do |btn|
         if btn['load_board'] && btn['load_board']['tmp_path']
@@ -184,9 +228,21 @@ module AACMetrics::Loader
         end
       end
     end
+    # TODO: record whether the board set is expected to have auto-home
     {boards: boards, words: words, words_path: words_path}
   end
 
+  # TODO: Qualitative assessments of common vocabularies,
+  # gather perspectives on what makes a "good" vocabulary
+  # and collect reviews from field experts, also free
+  # response sections.
+  # Some criteria:
+  # - works well for age group X, Y, Z
+  # - works well for a beginning communicator
+  # - allows long-term growth as-is
+  # - comprehensive core
+  # - 
+
   def self.ingest(fn, token=nil)
     output = nil
     boards = nil
@@ -292,6 +348,14 @@ module AACMetrics::Loader
     @@synonyms[locale] = res
   end
 
+  def self.sentences(locale)
+    @@sentences ||= {}
+    return @@sentences[locale] if @@sentences[locale]
+    locale = locale.split(/-|_/)[0]
+    path = File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'sets', "sentences.#{locale}.json"))
+    res = JSON.parse(File.read(path))
+    @@sentences[locale] = res
+  end
 
   def self.base_words(locale)
     @@base_words ||= {}

data/lib/aac-metrics/metrics.rb

@@ -2,31 +2,68 @@
 # Scores for average effort level for word sets (spelling if that's th only way)
 # Effort scores for sentence corpus
 # Effort algorithms for scanning/eyes
-
 module AACMetrics::Metrics
-  def self.analyze(obfset, output=true)
+    # TODO: 
+    # 1. When navigating from one board to the next, grid locations
+    # with the same clone_id or semantic_id should result in a
+    # discount to overall search based more on the number of
+    # uncloned/unsemantic buttons than the number of total buttons
+    # (perhaps also factoring in the percent of board with that
+    # id present in the full board set)
+    # 2. When selecting a button with a semantic_id or clone_id,
+    # a discount to both search and selection should
+    # be applied based on the percent of boards that
+    # contain the same id at that grid location
+    # 3.5 When selecting a button with a semantic_id or clone_id,
+    # if the same id was present on the previous board,
+    # an additional discount to search and selection should be applied
+    def self.analyze(obfset, output=true)
     locale = nil
     buttons = []
+    refs = {}
     total_boards = 1
-    
+
     if obfset.is_a?(Hash) && obfset['buttons']
       locale = obfset['locale'] || 'en'
+      refs = obfset['reference_counts']
       buttons = []
       obfset['buttons'].each do |btn|
         buttons << {
           id: btn['id'],
           label: btn['label'],
           level: btn['level'],
-          effort: btn['effort']
+          effort: btn['effort'],
+          semantic_id: btn['semantic_id'],
+          clone_id: btn['clone_id']
         }
       end
       total_boards = obfset['total_boards']
     else
       visited_board_ids = {}
       to_visit = [{board: obfset[0], level: 0, entry_x: 1.0, entry_y: 1.0}]
+      refs = {}
+      obfset.each do |board|
+        # determine frequency within the board set
+        # for each semantic_id and clone_id
+        if board['clone_ids']
+          boards['clone_ids'].each do |id|
+            refs[id] ||= 0
+            refs[id] += 1
+          end
+        end
+        if board['semantic_ids']
+          boards['semantic_ids'].each do |id|
+            refs[id] ||= 0
+            refs[id] += 1
+          end
+        end
+      end
+      pcts = {}
+      refs.each do |id, cnt|
+        pcts[id] = cnt.to_f / obfset.length.to_f
+      end
       locale = obfset[0]['locale']
       known_buttons = {}
-      sqrt2 = Math.sqrt(2)
       while to_visit.length > 0
         board = to_visit.shift
         visited_board_ids[board[:board]['id']] = board[:level]
@@ -35,32 +72,63 @@ module AACMetrics::Metrics
         btn_width = 1.0  / board[:board]['grid']['columns'].to_f
         board_effort = 0
         # add effort for level of complexity when new board is rendered
-        board_effort += 0.003 * board[:board]['grid']['rows'] * board[:board]['grid']['columns']
+        button_size = button_size_effort(board[:board]['grid']['rows'], board[:board]['grid']['columns'])
+        board_effort += button_size
         # add effort for number of visible buttons
-        board_effort += 0.007 * board[:board]['grid']['order'].flatten.length
+        field_size = field_size_effort(board[:board]['grid']['order'].flatten.length)
+        board_effort += field_size
+        # decrease effort here for every button on the board
+        # whose semantic_id or clone_id is repeated in the board set
+        #       -0.0025 (* pct of matching boards) for semantic_id
+        #       -0.005 (* pct of matching boards) for clone_id
+        board[:board]['grid']['rows'].times do |row_idx|
+          board[:board]['grid']['columns'].times do |col_idx|
+            button_id = (board[:board]['grid']['order'][row_idx] || [])[col_idx]
+            button = board[:board]['buttons'].detect{|b| b['id'] == button_id }
+            if button && button['clone_id'] && pcts[button['clone_id']]
+              board_effort -= 0.005 * pcts[button['clone_id']]
+            elsif button && button['semantic_id'] && pcts[button['semantic_id']]
+              board_effort -= 0.0025 * pcts[button['semantic_id']]
+            end
+          end
+        end
+
         prior_buttons = 0
 
         board[:board]['grid']['rows'].times do |row_idx|
           board[:board]['grid']['columns'].times do |col_idx|
             button_id = (board[:board]['grid']['order'][row_idx] || [])[col_idx]
             button = board[:board]['buttons'].detect{|b| b['id'] == button_id }
-            prior_buttons += 0.1 if !button
+            # prior_buttons += 0.1 if !button
             next unless button
             x = (btn_width / 2)  + (btn_width * col_idx)
             y = (btn_height / 2) + (btn_height * row_idx)
             # prior_buttons = (row_idx * board[:board]['grid']['columns']) + col_idx
             effort = 0
+            # TODO: additional discount on board search effort
+            #       if this button's semantic_id or clone_id
+            #       was also present on the prior board
+            #       board_effort * 0.5 for semantic_id
+            #       board_effort * 0.33 for clone_id
             effort += board_effort
             # add effort for percent distance from entry point
-            distance = Math.sqrt((x - board[:entry_x]) ** 2 + (y - board[:entry_y]) ** 2) / sqrt2
+            distance = distance_effort(x, y, board[:entry_x], board[:entry_y])
+            # TODO: decrease effective distance if the semantic_id or clone_id:
+            #       - are used on other boards in the set (semi)
+            #         distance * 0.5 (* pct of matching boards) for semantic_id
+            #         distance * 0.33 (* pct of matching boards) for clone_id
+            #       - was also present on the prior board (total)
+            #         distance * 0.5 for semantic_id
+            #         distance * 0.33 for clone_id
             effort += distance
-            if distance > 0.1 || (board[:entry_x] == 1.0 && board[:entry_y] == 1.0)
+            if distance > DISTANCE_THRESHOLD_TO_SKIP_VISUAL_SCAN || (board[:entry_x] == 1.0 && board[:entry_y] == 1.0)
               # add small effort for every prior (visible) button when visually scanning
-              effort += prior_buttons * 0.001
+              visual_scan = visual_scan_effort(prior_buttons)
+              effort += visual_scan
             else
               # ..unless it's right by the previous button, then
               # add tiny effort for local scan
-              effort += distance * 0.5
+              effort += distance * SKIPPED_VISUAL_SCAN_DISTANCE_MULTIPLIER
             end
             # add cumulative effort from previous sequence
             effort += board[:prior_effort] || 0
@@ -81,11 +149,12 @@ module AACMetrics::Metrics
               end
               if try_visit
                 next_board = obfset.detect{|brd| brd['id'] == button['load_board']['id'] }
+                puts "LIKE[] #{effort}" if button['label'] == 'like'
                 if next_board
                   to_visit.push({
                     board: next_board,
                     level: board[:level] + 1,
-                    prior_effort: effort + 1.0,
+                    prior_effort: effort + BOARD_CHANGE_PROCESSING_EFFORT,
                     entry_x: x,
                     entry_y: y
                   })
@@ -94,7 +163,8 @@ module AACMetrics::Metrics
             else
               word = button['label']
               existing = known_buttons[word]
-              if !existing || board[:level] < existing[:level]
+              if !existing || existing[:effort] < effort #board[:level] < existing[:level]
+                puts "LIKE #{effort}" if button['label'] == 'like'
                 known_buttons[word] = {
                   id: "#{button['id']}::#{board[:board]['id']}",
                   label: word,
@@ -120,50 +190,94 @@ module AACMetrics::Metrics
       locale: locale,
       total_boards: total_boards,
       total_buttons: buttons.length,
+      reference_counts: refs,
       buttons: buttons,
       levels: clusters
     }
   end
 
+  SQRT2 = Math.sqrt(2)
+  BUTTON_SIZE_MULTIPLIER = 0.09
+  FIELD_SIZE_MULTIPLIER = 0.017
+  VISUAL_SCAN_MULTIPLIER = 0.02
+  BOARD_CHANGE_PROCESSING_EFFORT = 1.0
+  DISTANCE_MULTIPLIER = 0.5
+  DISTANCE_THRESHOLD_TO_SKIP_VISUAL_SCAN = 0.1
+  SKIPPED_VISUAL_SCAN_DISTANCE_MULTIPLIER = 0.5
+
+  def self.button_size_effort(rows, cols)
+    BUTTON_SIZE_MULTIPLIER * (rows + cols) / 2
+  end
+
+  def self.field_size_effort(button_count)
+    FIELD_SIZE_MULTIPLIER * button_count
+  end
+
+  def self.visual_scan_effort(prior_buttons)
+    prior_buttons * VISUAL_SCAN_MULTIPLIER
+  end
+  
+  def self.distance_effort(x, y, entry_x, entry_y)
+    Math.sqrt((x - entry_x) ** 2 + (y - entry_y) ** 2) / SQRT2 * DISTANCE_MULTIPLIER
+  end
+
+  def self.spelling_effort(word)
+    10 + (word.length * 2.5)
+  end
+
   def self.analyze_and_compare(obfset, compset)
     target = AACMetrics::Metrics.analyze(obfset, false)
     res = {}.merge(target)
 
     compare = AACMetrics::Metrics.analyze(compset, false)
+    res[:comp_boards] = compare[:total_boards]
+    res[:comp_buttons] = compare[:total_buttons]
     
     compare_words = []
     compare_buttons = {}
+    comp_efforts = {}
     compare[:buttons].each do |btn|
       compare_words << btn[:label]
       compare_buttons[btn[:label]] = btn
+      comp_efforts[btn[:label]] = btn[:effort]
     end
 
-    efforts = {}
+    sortable_efforts = {}
+    target_efforts = {}
     target_words = []
+    # Track effort scores for each button in the set,
+    # used to sort and for assessing priority
+    # TODO: keep a list of expected effort scores for
+    # very frequent core words and use that when available
     res[:buttons].each{|b| 
       target_words << b[:label]
-      efforts[b[:label]] = b[:effort] 
+      target_efforts[b[:label]] = b[:effort]
+      sortable_efforts[b[:label]] = b[:effort] 
       comp = compare_buttons[b[:label]]
       if comp
         b[:comp_level] = comp[:level]
         b[:comp_effort] = comp[:effort]
       end
     }
+    # Effort scores are the mean of thw scores from the
+    # two sets, or just a singular value if in only one set
     compare[:buttons].each{|b| 
-      if efforts[b[:label]]
-        efforts[b[:label]] += b[:effort] 
-        efforts[b[:label]] /= 2
+      if sortable_efforts[b[:label]]
+        sortable_efforts[b[:label]] += b[:effort] 
+        sortable_efforts[b[:label]] /= 2
       else
-        efforts[b[:label]] ||= b[:effort] 
+        sortable_efforts[b[:label]] ||= b[:effort] 
       end
     }
     
     core_lists = AACMetrics::Loader.core_lists(target[:locale])
     common_words_obj = AACMetrics::Loader.common_words(target[:locale])
     synonyms = AACMetrics::Loader.synonyms(target[:locale])
-    common_words_obj['efforts'].each{|w, e| efforts[w] ||= e }
+    sentences = AACMetrics::Loader.sentences(target[:locale])
+    common_words_obj['efforts'].each{|w, e| sortable_efforts[w] ||= e }
     common_words = common_words_obj['words']
     
+    # Track which words are significantly harder or easier than expected
     too_easy = []
     too_hard = []
     target[:buttons].each do |btn|
@@ -177,11 +291,11 @@ module AACMetrics::Metrics
     end
 
     
-    missing = (compare_words - target_words).sort_by{|w| efforts[w] }
+    missing = (compare_words - target_words).sort_by{|w| sortable_efforts[w] }
     missing = missing.select do |word|
       !synonyms[word] || (synonyms[word] & target_words).length == 0
     end
-    extras = (target_words - compare_words).sort_by{|w| efforts[w] }
+    extras = (target_words - compare_words).sort_by{|w| sortable_efforts[w] }
     extras = extras.select do |word|
       !synonyms[word] || (synonyms[word] & compare_words).length == 0
     end
@@ -199,27 +313,131 @@ module AACMetrics::Metrics
     missing = missing.select do |word|
       !synonyms[word] || (synonyms[word] & target_words).length == 0
     end
+    common_effort = 0
+    comp_effort = 0
+    common_words.each do |word|
+      effort = target_efforts[word]
+      if !effort && synonyms[word]
+        synonyms[word].each do |syn|
+          effort ||= target_efforts[syn]
+        end
+      end
+      effort ||= 2 + (word.length * 2.5)
+      common_effort += effort
+
+      effort = comp_efforts[word]
+      if !effort && synonyms[word]
+        synonyms[word].each do |syn|
+          effort ||= comp_efforts[syn]
+        end
+      end
+      effort ||= 2 + (word.length * 2.5)
+      comp_effort += effort
+    end
+    common_effort = common_effort.to_f / common_words.length.to_f
+    comp_effort = comp_effort.to_f / common_words.length.to_f
     # puts "MISSING FROM COMMON (#{missing.length})"
     res[:missing] = {
       :common => {name: "Common Word List", list: missing}
     }
+    res[:cores] = {
+      :common => {name: "Common Word List", list: common_words, average_effort: common_effort, comp_effort: comp_effort}
+    }
+    target_effort_tally = 0.0
+    comp_effort_tally = 0.0
+    # For each core list, find any missing words, and compute
+    # the average level of effort for all words in the set,
+    # using a fallback effort metric if the word isn't in the
+    # board set
     # puts missing.join('  ')
     core_lists.each do |list|
-      puts list['id']
       missing = []
+      comp_missing = []
+      list_effort = 0
+      comp_effort = 0
       list['words'].each do |word|
         words = [word] + (synonyms[word] || [])
+        # Check if any words from the core list are missing in the set
         if (target_words & words).length == 0
           missing << word
-          raise "orng" if word == 'orange'
         end
+        if (compare_words & words).length == 0
+          comp_missing << word
+        end
+
+        # Calculate the effort for the target and comp sets
+        effort = target_efforts[word]
+        if !effort
+          words.each{|w| effort ||= target_efforts[w] }
+        end
+        # Fallback penalty for missing word
+        effort ||= spelling_effort(word)
+        list_effort += effort
+
+        effort = comp_efforts[word]
+        if !effort
+          words.each{|w| effort ||= comp_efforts[w] }
+        end
+        effort ||= spelling_effort(word)
+        comp_effort += effort
       end
       if missing.length > 0
         # puts "MISSING FROM #{list['id']} (#{missing.length}):"
-        res[:missing][list['id']] = {name: list['name'], list: missing}
+        res[:missing][list['id']] = {name: list['name'], list: missing, average_effort: list_effort}
         # puts missing.join('  ')
       end
+      list_effort = list_effort.to_f / list['words'].length.to_f
+      comp_effort = comp_effort.to_f / list['words'].length.to_f
+      target_effort_tally += list_effort
+      comp_effort_tally += comp_effort
+      res[:cores][list['id']] = {name: list['name'], list: list['words'], average_effort: list_effort, comp_effort: comp_effort}
+    end
+    target_effort_tally = (target_effort_tally / core_lists.to_a.length) * 5.0
+
+    comp_effort_tally = (comp_effort_tally / core_lists.to_a.length) * 5.0
+
+    # TODO: Assemble or allow a battery of word combinations,
+    # and calculate the level of effort for each sequence,
+    # as well as an average level of effort across combinations.
+    res[:sentences] = []
+    sentences.each do |words|
+      puts "  #{words.join(' ')}"
+      BOARD_CHANGE_PROCESSING_EFFORT
+      target_effort_score = 0.0
+      comp_effort_score = 0.0
+      words.each_with_index do |word, idx|
+        synonym_words = [word] + (synonyms[word] || [])
+        effort = target_efforts[word] || target_efforts[word.downcase]
+        if !effort
+          synonym_words.each{|w| effort ||= target_efforts[w] }
+        end
+        effort ||= spelling_effort(word)
+        effort += (idx == 0) ? 0.0 : BOARD_CHANGE_PROCESSING_EFFORT
+        ee = effort
+        target_effort_score += effort
+
+        effort = comp_efforts[word] || comp_efforts[word.downcase]
+        if !effort
+          synonym_words.each{|w| effort ||= comp_efforts[w] }
+        end
+        effort ||= spelling_effort(word)
+        effort += (idx == 0) ? 0.0 : BOARD_CHANGE_PROCESSING_EFFORT
+        comp_effort_score += effort
+        puts "      #{word} #{ee} #{effort}"
+      end
+      target_effort_score = target_effort_score / words.length
+      comp_effort_score = comp_effort_score / words.length
+      res[:sentences] << {sentence: words.join(' '), words: words, effort: target_effort_score, comp_effort: comp_effort_score}
     end
+    target_effort_tally += res[:sentences].map{|s| s[:effort] }.sum.to_f / res[:sentences].length.to_f * 3.0
+    comp_effort_tally += res[:sentences].map{|s| s[:comp_effort] }.sum.to_f / res[:sentences].length.to_f * 3.0
+    target_effort_tally += 100 # placeholder value for future added calculations
+    comp_effort_tally += 100
+
+
+
+    res[:target_effort_score] = target_effort_tally
+    res[:comp_effort_score] = comp_effort_tally
     # puts "CONSIDER MAKING EASIER"
     res[:high_effort_words] = too_hard
     # puts too_hard.join('  ')

data/lib/aac-metrics.rb

@@ -8,7 +8,7 @@
 # Are grammar inflections available?
 # Does this support auto-home, or other motor planning supports?
 # Are other language options available?
-# How is thee vocabulary organized?
+# How is the vocabulary organized?
 # What platforms are supported?
 # Access to keyboard w/ prediction? Numbers?
 # How easy is it to say these personalized sentences: _____