glim_ai 0.2.0

data/examples/window_check/fruits.rb

@@ -0,0 +1,118 @@
+require_relative '../../lib/globals'
+require_relative 'data'
+
+glim = GlimContext.new(log_name: "ask_all")
+
+# Given item_count items, get a list of indices into data
+# that are equidistant and include 0 and item_count-1
+def select_data_point_indices(data_point_count, item_count)
+    delta = item_count / (data_point_count - 1.0)
+    indices = []
+    for i in 0..(data_point_count - 2)
+        indices << (i * delta).round
+    end
+    indices << (item_count - 1)
+end
+
+def extract_between_markers(str, start_marker, end_marker)
+    str[/#{Regexp.escape(start_marker)}(.*?)#{Regexp.escape(end_marker)}/m, 1]
+end
+
+#models =  ["claude-instant-1",  "gpt-3.5-turbo"]
+#models =  ["claude-instant-1"] # ,  "gpt-3.5-turbo"]
+# models =  ["gpt-4"]
+
+models =  ["gpt-3.5-turbo"]
+
+models = ["meta-llama/Llama-2-70b-chat-hf"] # 7,13,70 all 4k
+
+models = ["codellama/CodeLlama-34b-Instruct-hf"]
+
+# models = ["claude-instant-1"]
+
+
+srand 0
+
+names = raw_names
+fruits = raw_fruits
+
+puts "We have #{names.length} names and #{fruits.length} fruits."
+
+fruit_counts = [] # name_index -> fruit -> count
+sentences = []
+
+index_for_name = {}
+
+names.each_with_index do |name, i|
+    index_for_name[name] = i
+    my_fruits = fruits.sample(10)
+    for mf in my_fruits
+        count = rand(1000000) + 2
+        fruit_counts[i] ||= {}
+        fruit_counts[i][mf] = count
+    end
+    s  = "#{name} has: "
+    s << my_fruits[0..-2].map { |fruit| "#{fruit_counts[i][fruit]} #{fruit}" }.join(", ")
+    s << ", and 42 #{my_fruits[0]}.\n"
+    sentences [i] = s
+end
+
+question1 = "\n\n\n\nHow many of each fruit did "
+question2 = " have? Respond with a json that maps names to a dictionary with number of each fruit they had."
+
+data_point_count = 8
+
+for model in models
+    request = glim.request(model:)
+    
+    request.prompt = question1 + question2
+    question_length = request.prompt_token_count
+    reserve = question_length + 100 * data_point_count
+
+    p = ""
+    sentence_count = 0
+    for sentence in sentences
+    request.prompt = p + sentence
+        if (request.prompt_token_count + reserve) > request.context_length
+            break
+        end
+        p += sentence
+        sentence_count += 1
+    end
+
+    # get relatively evenly spaced data points
+    indices = select_data_point_indices(data_point_count, sentence_count)
+    puts "Data point indices: #{indices.inspect}"
+
+    name_list = indices[0..-2].map { |i| names[i] }.join(", ")
+    name_list += ", and #{names[indices.last]}"
+    request.prompt = p + question1 + name_list + question2
+    
+    request.prepare!
+    puts "Ready to send to API. Cost will be at least $ #{request.min_cost}"
+    puts "Tokens: #{request.prompt_token_count}"
+
+    response = request.response
+    r = response.completion
+    puts "\n\nResponse from #{model}: \n#{r}"
+    # r = extract_between_markers(r, "```json","```")
+    puts "\n\JSON from #{model}: \n#{r}"
+    
+    all_counts = JSON.parse(r)
+    scores = []
+    for name in all_counts.keys
+        # name = my_counts["name"]
+        my_counts = all_counts[name]
+        index = index_for_name[name]
+        ground_truth = fruit_counts[index]
+        score = 0
+        ground_truth.each_pair do |k, v|
+            if my_counts[k] == v
+                score += 1
+            end
+        end
+        scores << score
+    end
+    puts "#{model}: score = #{scores}"
+
+end

data/examples/window_check/tools.rb

@@ -0,0 +1,56 @@
+
+# Given item_count items, get a list of indices into data
+# that are equidistant and include 0 and item_count-1
+def select_data_point_indices(data_point_count, item_count)
+    delta = item_count / (data_point_count - 1.0)
+    indices = []
+    for i in 0..(data_point_count - 2)
+        indices << (i * delta).round
+    end
+    indices << (item_count - 1)
+end
+
+
+
+def make_code(length)
+  chars = ([*('0'..'9'), *('a'..'z')])
+  (Array.new(length) { chars.sample }).join
+end
+
+
+# if false
+#   pos = rand(len-1)
+#   t = s[pos]
+#   s[pos] = s[pos+1]
+#   s[pos+1] = t
+# end
+
+# this generates a string of length len that will be
+# a repeating sequence of an extended alphabet, but with
+# deletions chars missing in random locations. 
+# the length will always be the same, but the levenshtein distance
+# can be up to 2*deletions + 2*flips. Of course it can be less, 
+# if somehow some of these edits simplify or cancel out. 
+# safest to use only deletions or only flips, not both.
+def make_special_code_with_fixed_length(len, deletions: 0, flips: 0)
+  raise ArgumentError.new("len must be > 0") unless len > 0
+  raise ArgumentError.new("deletions must be >= 0") unless deletions >= 0
+  raise ArgumentError.new("flips must be >= 0") unless flips >= 0
+  raise ArgumentError.new("deletions + flips must be <= len") unless deletions + flips <= len
+
+  alphabet = ((0..9).to_a + ("a".."z").to_a + ("A".."Z").to_a).join
+  count = (len+deletions+10)/alphabet.length+1
+  s = alphabet * count
+  deletions.times do
+      pos = rand(len-3)
+      s[pos] = ""
+  end
+  flips.times do
+    pos = 2+rand(len-5)
+    t = s[pos]
+    s[pos] = s[pos+1]
+    s[pos+1] = t
+  end
+  s = s[0..len-1]
+  return s
+end

data/examples/window_check/window_check.rb

@@ -0,0 +1,214 @@
+require_relative '../../lib/globals'
+require_relative 'data'
+require_relative 'tools'
+
+glim = GlimContext.new(log_name: "ask_all")
+
+# llm_name = "gpt-3.5-turbo"
+# llm_name = "claude-instant-1"
+
+llm_scores = []
+
+data_point_count = 20
+row_count = 18
+
+for run in 0..100
+    puts "run = #{run}"
+    llm_scores[run] = {}
+
+    for llm_name in GlimRequest.llama2_llms[1..1]
+        puts "llm_name = #{llm_name}"
+
+        srand run+2000
+
+        names = raw_names
+        sentences = []
+        index_for_name = {}
+        codes = []
+
+        code_length = 100
+
+        names.each_with_index do |name, i|
+            index_for_name[name] = i    
+            if i % 2 == 0
+                code = make_special_code_with_fixed_length(code_length,flips: 2)
+            else
+                code = make_code(code_length)
+            end
+            codes[i] = code
+            sentences[i] = "#{name}'s code: #{code}\n"
+        end
+
+        instructions = "\n\nGenerate a json that maps each name to its code."
+
+        answer_line = "  \"#{names[1]}\": \"#{codes[1]}\",\n"
+
+        request = glim.request(llm_name:, max_tokens: 4000)
+        
+        long_code = make_code(10000)
+        token_length = long_code.length/request.count_tokens(long_code).to_f
+        puts "1 token is on average #{token_length} chars long."
+
+        # estimate how many rows will fit into the context window, including response
+        instructions_length = request.count_tokens(instructions)
+        answer_line_length = request.count_tokens(answer_line)
+        sentence_length = request.count_tokens(sentences[0])
+        max_row_count = ((request.context_length - instructions_length - 80) / (sentence_length + answer_line_length + 10)) - 1
+
+        puts "row_count = #{row_count} < max_row_count = #{max_row_count}"
+
+        data = sentences[0..row_count-1].join
+        request.prompt = data + instructions
+
+        puts "Prompt:\n#{request.prompt}"
+
+        answer_length = answer_line_length * row_count
+        data_length = request.count_tokens(data)
+        puts "Instructions: #{instructions_length} tokens"
+        puts "Each sentence: roughly #{sentence_length} tokens, #{data_length} in total"
+        puts "Each answer line: #{answer_line_length} tokens, #{answer_length} in total"
+        puts "Estimated total: #{instructions_length + answer_length + data_length} tokens"
+
+        puts "Ready to send #{request.prompt_token_count} tokens to API. Cost will be at least $ #{request.min_cost}"
+
+        response = request.response
+        r = response.completion.strip
+        #puts "\n\nResponse from #{model}: \n#{r}"
+
+        begin
+            result = extract_json(r)
+        rescue
+            puts "ERROR "*10
+            puts "Failed to parse JSON:"
+            puts r
+            next
+        end
+        scores = []
+        result.each_pair do |name, v|
+            index = index_for_name[name]
+            distance, explanation = levenshtein_distance(codes[index],v)
+            scores[index] = (code_length - distance) / code_length.to_f
+            puts explanation
+        end
+        puts "#{llm_name}: run = #{run}, scores = #{scores}"
+        llm_scores[run][llm_name] = scores
+    end
+    puts JSON.pretty_generate(llm_scores)
+
+    clean_llm_scores = []
+    for i in 0..llm_scores.length-1
+        if llm_scores[i]["meta-llama/Llama-2-13b-chat-hf"].length == row_count
+            clean_llm_scores << llm_scores[i]
+        else
+            puts "Skipping #{i}"
+        end
+    end
+
+    sums = []
+    for i in 0..row_count-1
+        sums[i] = clean_llm_scores.map { |x| x["meta-llama/Llama-2-13b-chat-hf"][i] }.count { |x| x >= 0.999 }
+    end
+
+    puts sums.inspect
+
+
+    # sums = llm_scores[0]["meta-llama/Llama-2-13b-chat-hf"].map { |x| (x >= 0.999) ? 1 : 0 }
+    
+    # raise "sums.length = #{sums.length}" unless sums.length == row_count
+    # for runscores in llm_scores[1..-1]
+    #     runscores13b = runscores["meta-llama/Llama-2-13b-chat-hf"]
+    #     raise "runscores13b.length = #{runscores13b.length}" unless runscores13b.length == row_count
+    #     sums = sums.zip(runscores13b).map do |a,b|
+    #         (b >= 0.999) ? a : a+1
+    #     end
+    #     puts sums.inspect
+
+    # end
+
+end
+
+
+
+
+# Instructions: 12 tokens
+# Each sentence: roughly 83 tokens, 1646 in total
+# Each answer line: 83 tokens, 1660 in total
+# Estimated total: 3318 tokens
+# 0.237:  Tiktoken doesn't know model meta-llama/Llama-2-70b-chat-hf
+# 0.237:  Tiktoken doesn't know model meta-llama/Llama-2-70b-chat-hf
+# Ready to send 1657 tokens to API. Cost will be at least $ 0.0016569999999999998
+# meta-llama/Llama-2-70b-chat-hf: scores = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
+# {
+#   "meta-llama/Llama-2-7b-chat-hf": [
+#     1.0,
+#     1.0,
+#     0.8083333333333333,
+#     0.9916666666666667,
+#     0.7583333333333333,
+#     0.9916666666666667,
+#     0.5416666666666666,
+#     0.9083333333333333,
+#     0.7666666666666667,
+#     0.30833333333333335,
+#     0.7083333333333334,
+#     0.75,
+#     0.8166666666666667,
+#     0.75,
+#     0.825,
+#     0.75,
+#     0.7916666666666666
+#   ],
+#   "meta-llama/Llama-2-13b-chat-hf": [
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0
+#   ],
+#   "meta-llama/Llama-2-70b-chat-hf": [
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0,
+#     1.0
+#   ]
+# }
+
+# 13b / 20
+# [38.920000000000016, 41.0, 40.82000000000001, 41.0, 40.839999999999996, 40.980000000000004, 40.809999999999995, 41.0, 40.84, 40.989999999999995, 40.72, 41.0, 40.77, 40.99, 40.88, 40.99, 40.55, 40.98, 40.76]
+
+
+# srand 1000+ 
+# [96.21999999999993, 100.99, 100.78, 100.99, 100.65999999999998, 101.0, 100.72, 101.0, 100.64, 100.98, 100.72999999999999, 100.97999999999999, 100.78, 100.99000000000001, 100.74000000000001, 100.99000000000001, 100.6, 100.99000000000001]
+
+# srand 1000+
+# counting pnly
+# [35, 100, 91, 100, 87, 101, 87, 101, 88, 99, 90, 99, 92, 100, 89, 100, 83, 100]

data/glim_generated_tests/make_special_code_with_fixed_length_test.rb

@@ -0,0 +1,44 @@
+# File: make_special_code_with_fixed_length_test.rb
+require 'minitest/autorun'
+require_relative '../lib/globals'
+
+class MakeSpecialCodeWithFixedLengthTest < Minitest::Test
+  def setup
+    @base = "0123456789" + ("a".."z").to_a.join +  ("A".."Z").to_a.join
+    @base_len = @base.length
+    @len = 100
+  end
+  
+  def test_basic_function
+    test_string = make_special_code_with_fixed_length(@len, deletions: 0, flips: 0)
+    # Length should be @len
+    assert_equal @len, test_string.length
+    # There should be no randomness introduced
+    assert_equal @base[0...@len], test_string
+  end
+  
+  def test_deletions
+    test_string = make_special_code_with_fixed_length(@len, deletions: 10, flips: 0)
+    assert_equal @len, test_string.length
+    # Deletion is random, but levenshtein distance should be 10
+    assert_equal 10, levenshtein_distance(@base[0...@len], test_string).first
+  end
+
+  def test_flips
+    test_string = make_special_code_with_fixed_length(@len, deletions: 0, flips: 10)
+    assert_equal @len, test_string.length
+    # Flips is random, but levenshtein distance should be 20
+    assert_equal 20, levenshtein_distance(@base[0...@len], test_string).first
+  end
+  
+  def test_argument_errors
+    # Raises error for non positive length
+    assert_raises(ArgumentError) { make_special_code_with_fixed_length(0, deletions: 0, flips: 0) }
+    assert_raises(ArgumentError) { make_special_code_with_fixed_length(-10, deletions: 0, flips: 0) }
+    # Raises error for negative deletions and flips
+    assert_raises(ArgumentError) { make_special_code_with_fixed_length(@len, deletions: -1, flips: 0) }
+    assert_raises(ArgumentError) { make_special_code_with_fixed_length(@len, deletions: 0, flips: -10) }
+    # Raises error if deletions + flips > len
+    assert_raises(ArgumentError) {  make_special_code_with_fixed_length(@len, deletions: 50, flips: 60) }
+  end
+end

data/glim_generated_tests/old-20230831120513-make_special_code_with_fixed_length_test.rb

@@ -0,0 +1 @@
+# File: make_special_code_with_fixed_length_test.rb

data/glim_generated_tests/old-20230831121222-make_special_code_with_fixed_length_test.rb

@@ -0,0 +1,55 @@
+# File: make_special_code_with_fixed_length_test.rb
+require 'minitest/autorun'
+require_relative "../lib/globals"
+
+class MakeSpecialCodeWithFixedLengthTest < Minitest::Test
+  def setup
+    @alphabet = ((0..9).to_a + ("a".."z").to_a + ("A".."Z").to_a).join
+  end
+
+  def test_normal_case
+    result = make_special_code_with_fixed_length(15, 2)
+    assert_kind_of String, result
+    assert_equal 15, result.length
+    assert_equal 2, @alphabet.length*5 - levenshtein_distance(@alphabet*5, result)[0]
+  end
+
+  def test_zero_deletions
+    result = make_special_code_with_fixed_length(15, 0)
+    assert_kind_of String, result
+    assert_equal 15, result.length
+    assert_equal 0, @alphabet.length*5 - levenshtein_distance(@alphabet*5, result)[0]
+  end
+
+  def test_string_has_more_than_requested_length
+    result = make_special_code_with_fixed_length(50, 2)
+    assert_kind_of String, result
+    assert_equal 50, result.length
+    assert_equal 2, @alphabet.length*2 - levenshtein_distance(@alphabet*2, result)[0]
+  end
+
+  def test_zero_length
+    result = make_special_code_with_fixed_length(0, 2)
+    assert_kind_of String, result
+    assert_equal 0, result.length
+    assert_equal 2, @alphabet.length*5 - levenshtein_distance(@alphabet*5, result)[0]
+  end
+
+  def test_negative_length
+    assert_raises do
+      make_special_code_with_fixed_length(-1, 2)
+    end
+  end
+
+  def test_negative_deletions
+    assert_raises do
+      make_special_code_with_fixed_length(15, -1)
+    end
+  end
+  
+  def test_length_less_than_deletions
+    assert_raises do
+      make_special_code_with_fixed_length(2, 15)
+    end
+  end  
+end

data/glim_generated_tests/old-20230831124501-make_special_code_with_fixed_length_test.rb

@@ -0,0 +1,33 @@
+# File: make_special_code_with_fixed_length_test.rb
+require_relative '../lib/globals'
+require 'minitest/autorun'
+
+class TestMakeSpecialCodeWithFixedLength < Minitest::Test
+  def setup
+    @len = 10
+    @deletions = 2
+    @test_string = make_special_code_with_fixed_length(@len, @deletions)
+  end
+
+  def test_length
+    assert_equal @len, @test_string.length, "Output string length does not match requested length."
+  end
+
+  def test_levenshtein_distance
+    original_string = make_special_code_with_fixed_length(@len, 0)
+    expected_distance, _ = levenshtein_distance(original_string, @test_string)
+    assert_equal @deletions, expected_distance, "Levenshtein distance does not match the number of deletions."
+  end
+
+  def test_negative_length
+    assert_raises { make_special_code_with_fixed_length(-1, @deletions) }
+  end
+
+  def test_negative_deletions
+    assert_raises { make_special_code_with_fixed_length(@len, -1) }
+  end
+    
+  def test_deletion_greater_than_length
+    assert_raises { make_special_code_with_fixed_length(@len, @len + 1) }
+  end
+end

data/glim_generated_tests/test/make_special_code_with_fixed_length_test.rb

@@ -0,0 +1,58 @@
+# File: test/make_special_code_with_fixed_length_test.rb
+require 'minitest/autorun'
+require_relative '../lib/globals'
+
+class TestMakeSpecialCodeWithFixedLength < Minitest::Test
+  def setup
+    @len = 20
+  end
+
+  def test_normal_case
+    code = make_special_code_with_fixed_length(@len, deletions: 2, flips: 3)
+    assert_equal @len, code.length, "Failed on test where code length != len"
+    orig_code = make_special_code_with_fixed_length(@len, deletions: 0, flips: 0)
+    distance, _ = levenshtein_distance(orig_code, code)
+    assert_equal 4, distance, "Failed on test where levenshtein distance != 2*deletions"
+  end
+
+  def test_zero_len
+    code = make_special_code_with_fixed_length(0, deletions: 2, flips: 2)
+    assert_equal "", code, "Failed on test where len = 0"
+  end
+
+  def test_large_len
+    @len = 1_000_000
+    code = make_special_code_with_fixed_length(@len, deletions: 2, flips: 2)
+    assert_equal @len, code.length, "Failed on test where len is very large"
+  end
+
+  def test_negative_len
+    assert_raises(ArgumentError) do
+      make_special_code_with_fixed_length(-1, deletions: 2, flips: 2)
+    end
+  end
+
+  def test_negative_deletions
+    assert_raises(ArgumentError) do
+      make_special_code_with_fixed_length(@len, deletions: -2, flips: 2)
+    end
+  end
+
+  def test_negative_flips
+    assert_raises(ArgumentError) do
+      make_special_code_with_fixed_length(@len, deletions: 2, flips: -2)
+    end
+  end
+
+  def test_deletions_plus_flips_greater_than_len
+    assert_raises(ArgumentError) do
+      make_special_code_with_fixed_length(@len, deletions: @len + 1, flips: 1)
+    end
+  end
+
+  def test_fractional_len
+    assert_raises(ArgumentError) do
+      make_special_code_with_fixed_length(1.5, deletions: 2, flips: 2)
+    end
+  end
+end

data/lib/anthropic_request_details.rb

@@ -0,0 +1,37 @@
+require 'tiktoken_ruby' # TODO only for token counting while anthropic doesn't support it
+
+require 'anthropic'
+#require_relative 'globals'
+
+Anthropic.configure do |config|
+  config.access_token = ENV.fetch('ANTHROPIC_API_KEY')
+  config.request_timeout = 480 # Optional
+end
+
+# GlimRequest delegates to this
+class AnthropicRequestDetails  # only for requests that involve a message array, like OpenAI  
+  
+  def initialize(req)
+    @req = req
+  end
+  
+  attr_accessor :req
+
+  def response_class
+    AnthropicResponse
+  end
+
+  def llm_class_changed
+    update_request_hash
+  end
+
+  def update_request_hash
+    req.request_hash[:max_tokens_to_sample] = req.max_tokens || 2000
+    req.request_hash[:prompt] = req.prompt    
+    req.request_hash[:temperature] = req.temperature 
+    req.request_hash[:model] = req.llm_name 
+    # deeply remove keys for any values that are nil
+    req.request_hash.delete_if { |k, v| v.nil? }
+  end
+
+end