glim_ai 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 439f2dac7e8ac38e125560a5475f94f427b6649c10517bf0ce0ddc9643ffd0f2
+  data.tar.gz: 5eecfa7efb241e85f67368d90b061d227ad9b33c44312984d6dcf385f818aba0
+SHA512:
+  metadata.gz: c4fc1d7361ff5f465298d5e9a32f1447a900ac165b80c40de6a23781735b7d0a92aec42a2f86a254e7da71ce49e489ae5696e61905037292e81b7f73c4230cb1
+  data.tar.gz: a4e2fa686290840c2f8183397bbdc7aad1e1ca4a8466d2fa0336202428a364cb52ed1311b609ee72ff34f0fb523fc08b246dffb6bf8ae23803e1e09077c67d61

data/Gemfile

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in llm.gemspec
+gemspec
+
+gem 'must_be'
+
+gem 'rake', '~> 13.0'
+
+gem 'minitest', '~> 5.0'
+
+gem 'anthropic'
+gem 'ruby-openai'
+
+gem 'json-schema'
+
+# this was a bad idea; purpose was to use this for OpenAI functions, but those use kwargs which
+#   are not supported very well by Sorbet
+# gem 'sorbet', :group => :development
+# gem 'sorbet-runtime'
+# gem 'tapioca', require: false, :group => :development
+
+gem "tiktoken_ruby", "~> 0.0.5"

data/Gemfile.lock

@@ -0,0 +1,49 @@
+PATH
+  remote: .
+  specs:
+    llm (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.5)
+      public_suffix (>= 2.0.2, < 6.0)
+    anthropic (0.1.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
+    dotenv (2.8.1)
+    faraday (2.7.10)
+      faraday-net_http (>= 2.0, < 3.1)
+      ruby2_keywords (>= 0.0.4)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
+    faraday-net_http (3.0.2)
+    json-schema (4.0.0)
+      addressable (>= 2.8)
+    minitest (5.19.0)
+    multipart-post (2.3.0)
+    must_be (1.1.0)
+    public_suffix (5.0.3)
+    rake (13.0.6)
+    ruby-openai (4.2.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
+    ruby2_keywords (0.0.5)
+    tiktoken_ruby (0.0.5-arm64-darwin)
+
+PLATFORMS
+  arm64-darwin-22
+
+DEPENDENCIES
+  anthropic
+  dotenv
+  json-schema
+  llm!
+  minitest (~> 5.0)
+  must_be
+  rake (~> 13.0)
+  ruby-openai
+  tiktoken_ruby (~> 0.0.5)
+
+BUNDLED WITH
+   2.4.6

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Ulrich Gall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,125 @@
+
+## Design goals
+
+It's becoming clear that langchain's abstractions aren't the right thing. Langchain introduced abstractions that allow converting their way of interacting with LLMs to whichever underlying model is used. That's too much complexity overhead. We take a more pragmatic approach: Make it easy to work with current APIs. The problem right now isn't that we need future proof code, it's that we need good code quickly. 
+
+Specifically, goals include:
+- Make it easy to iterate on prompts separately from iterating on the code.
+- Allow changing of the model as easily as possible. Model can be specified by default in code and overridden by prompt, or the other way around. 
+- Make it as easy as possible to send requests asynchronously without having to think about concurrency more than necessary
+- Handle rate limits in a smart way.
+
+## Status
+
+Still lots to do, just asking for some initial feedback on:
+
+- spec template idea in general
+- naming!
+- ruby conventions i'm violating
+
+## Design Details
+
+A GlimRequest represents a request to an LLM to perform a task. GlimRequest itself contains 
+functionality and parameters that are common to all supported LLMs:
+- parameters like temperature, top_p, etc
+- the name of the llm to be used
+- code for handling erb templates
+- token counting and cost estimate code
+
+To support functionality that is specific to some LLM APIs, there is, for each support LLM API, 
+a GlimRequestDetails class. 
+
+Each GlimRequest can have a reference to a GlimRequestDetails object, to which it delegates any
+methods. The GlimRequest, potentially with support from a GlimRequestDetails object, has to meet
+one key responsibility: After it is created, it must at all times be able to provide a request_hash, 
+which is a Hash that contains all of the data that needs to be sent to the LLM's API in order to
+submit the request.
+
+Thus, the GlimRequest and GlimRequestDetails must, whenever the user make a modification to either, 
+update its internal request_hash to stay consistent. 
+
+There is one tricky situation that is a bit annoying, but we decided to be pragmatic about it
+and tolerate some awkwardness: If you change the llm for a GlimRequest to an llm that requires a different
+GlimRequestDetails class, then the GlimRequestDetails will be replaced and any data in it is lost. 
+
+For example, when changing from "gpt-3.5-turbo" (ChatRequestDetails) to "claude-instant-1" (AnthropicRequestDetails),
+then the output_schema or function_object will of course be deleted. This is facilitated by the GlimRequest
+creating a new AnthropicRequestDetails instance; as it is created, it is responsible for making sure that
+the request_hash is accurate. In the other direction, changing from claude to GPT, similarly, a new
+ChatRequestDetails instance would be created. 
+
+Above we have described that (and how) a GlimRequest can always provide a request_hash object. The point of
+this is that 
+- this hash is used for generating the cache key. If the hashes are identical, we don't need
+to contact the LLM API again, which saves time and money. 
+- the corresponding GlimResponse class can call GlimRequest#request_hash to obtain the necessary data, 
+and then it is responsibe for sending the request off to an LLM, as well as interpreting the response
+and making it accessible in a convenient way to the user. 
+
+There is one additional feature that is related: For each GlimRequest, there is a log directory, in which
+at any time there are several files that represent the content of the GlimRequest:
+- generic_request_params: temperature, llm_name, etc 
+- prompt 
+- template_text (if a template was used)
+- request_hash
+
+And for ChatRequestDetails, also:
+- messages: the array of messages, up to and including the message that will be sent
+- output_schema.json
+
+Once a response has been created, it would also contain:
+- raw_response.json: the exact reponse as received when making the LLM API call
+- completion.txt: just the completion that was generated by the LLM for this request
+
+## Running the code
+
+Probably some annoying issues with paths and whatnot. 
+Maybe best to just read the code without running it. 
+Check out:
+test/*
+examples/*
+
+
+## License
+
+The gem will be available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## TODO
+
+# Cleanup before alpha
+
+- write a better README
+- make a proper gem
+
+# Features to add / change
+
+(1) post processing, including a way to *throw* errors and perhaps have an errors/ subdir in the llm_logs
+-- maybe not needed because extract_data? or maybe support it only for json? 
+
+(2) AICallable 
+- more data types: array, boolean?
+- allow changing the ai_name for the function, not just the args
+
+(3)
+- request#response -- rename to make clear that it's async
+- ask for feedback on this one?
+
+(4) support "continue" prompting, especially for claude; 2k tokens is not much
+
+(5) make include_files and extract_files so that they can be model specific... for example, to use Anthropic's XML tag training
+- might need to get rid of the ruby anthropic API since it forces the \n\nHuman: pattern
+
+
+(6) web view on the input and outputs for the llm?
+
+(7) iterate on prompt to measure effectiveness. have LLM develop variations of prompt. 
+try it out on the multi-file-generation.
+
+(8) Token healing? 
+https://github.com/guidance-ai/guidance
+
+
+---
+git pull for code_gen?
+---
+

data/Rakefile

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'fileutils'
+
+def clear(var,what)
+  path = ENV[var] || what
+  FileUtils.mv(path, File.join("/tmp","glim_#{what}_#{Time.now.to_i}"))
+  FileUtils.mkdir(path)
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'test'
+  t.libs << 'lib'
+  t.test_files = FileList['test/**/test_*.rb']
+end
+
+task default: :test
+
+task :clear_cache do
+  clear('GLIM_CACHE_DIRECTORY', 'glim_cache')
+end
+
+task :clear_logs do
+  clear('GLIM_LOG_DIRECTORY', 'glim_logs')
+end
+
+task :clear_generated_code do
+  clear('GLIM_GENERATED_CODE_DIRECTORY', 'glim_generated_code')
+end

data/examples/autocode/autocode.rb

@@ -0,0 +1,166 @@
+require_relative '../../lib/globals'
+require_relative '../../lib/glim_ai_callable'
+
+require 'tempfile'
+require 'open3'
+
+class CodeBase
+    include AICallable
+    def done?
+        return @done != nil
+    end
+
+    def initialize(run_command:, project_root:)
+        @run_command = run_command
+        @project_root = project_root
+        @done = nil
+    end
+
+    attr_reader :run_command
+    attr_accessor :project_root
+
+    def done_confidence
+        return @done
+    end
+
+    ai_callable_as :show_file do
+        describe "Returns the contents of the file with the given path name"
+        string :path_name, "The path name of the file to show", required:true
+    end
+    def show_file(path_name:)
+        puts "Reading file and providing to LLM: project_root = #{project_root}  /   #{path_name}"
+        begin
+            full_path = File.join(project_root, path_name)
+            return File.read(full_path)
+        rescue Errno::ENOENT
+            return "File not found"
+        rescue
+            raise
+        end
+    end
+
+    # ai_callable_as :run_code do
+    #     describe "Run the code to try it out. Returns [stderr, stdout] with the output produced by running the code."
+    #     number :confidence_level, "Your estimate of the likelihood (0..1) that your code changes will work.", required:true
+    # end    
+
+    # plan was to allow LLM to trigger this, but it turns out that's more complicated
+    # because it doesn't reliably call a function AND provide a completion
+    def run_code(confidence_level: 1)
+        Open3.popen3(@run_command) do |stdin, stdout, stderr, thread|
+            output, errors = "", ""
+            out_thread = Thread.new { output = stdout.read }
+            err_thread = Thread.new { errors = stderr.read }
+            out_thread.join
+            err_thread.join
+            puts "\n\n\n>>> $#{@run_command}"
+            return errors,output
+        end
+    end
+    
+    ai_callable_as :code_ran_as_expected do
+        describe "Call this to indicate that the code ran as expected."
+        number :confidence_level, "Your estimate of the likelihood (0..1) that the code ran as expected and accomplished the given task.", required:true
+    end    
+    def code_ran_as_expected(confidence_level:)
+        puts "\n\n\nWe think we are done. Confidence level: #{confidence_level}"
+        @done = confidence_level
+    end
+end
+
+def perform_task(llm_name:, project_root:, task:, run_command:)
+
+    cb = CodeBase.new(run_command:, project_root:)
+    glim = GlimContext.new(log_name:"autocode-#{Time.now.strftime('%Y-%m-%d-%H-%M-%S')}")
+    req = glim.request_from_template("task", task:)
+    req.llm_name = llm_name
+
+    max_iter = 4
+
+    for iter in 0..max_iter
+        req.set_functions_object(cb)
+        response = req.response
+        completion = response.completion
+        if completion
+            puts "Got Completion."
+            if response.function_call_message?
+                puts "WEIRD: ALso got a function call message; LLM wants us to call #{response.function_name_from_message}"
+                puts "IGNORED IT."
+            end
+            text, files_saved = extract_and_save_files(completion, cb.project_root)
+            puts "Extracted #{files_saved} files from completion."
+            if files_saved.length > 0
+                stderr, stdout = cb.run_code
+                # construct new request with the output of the code
+                req = response.create_request_for_chat
+                req.process_template("changed_files_now_evaluate_output", stdout:, stderr:, files_saved:)
+                # function object will be set in net iter
+                next
+            else
+                puts "No files saved, no point in running the code."
+            end
+        end
+        # either there was a completion and we handed it, or there was no completion.
+        # regardless, we need to now see if there is a function to call.
+        if response.function_call_message?
+            if completion
+                puts "Weird, we have a completion and also a function call that is not run_code."
+                puts "Completion:"
+                puts completion
+                puts "\n\nFunction call requested: "
+                puts response.function_name_from_message
+                puts "IGNORING COMPLETION. CALLING FUNCTION."
+            end
+            new_req = response.create_request_with_function_result
+            # that invoked the function requested, so now we can check:
+            break if cb.done?
+            req = new_req
+        else
+            if completion
+                puts "TODO -- ONLY got a completion."
+                puts completion
+                break
+            end
+        end
+    end
+end
+
+# This did not work well, mostly because the arg sent by GPT did not have strings escaped properly
+
+# ai_callable_as :apply_patch do
+#     describe "Modify the codebase by appling a patch"
+#     string :patch_string, "A string containing the patch to be applied, in diff -u format", ai_name: "string_with_patch_in_diff_u_format", required: true
+#     string :file_path_name, "The path name of the file to apply the patch to", required: true
+# end
+# def apply_patch(file_path_name:, patch_string:)
+#     puts "PATCH to #{file_path_name}: \n>>\n#{patch_string}\n<<\n"
+#     patch_file = Tempfile.new('patch')
+#     patch_file.write(patch_string)
+#     patch_file.rewind
+#     Open3.popen3("patch #{file_path_name} #{patch_file.path}") do |stdin, stdout, stderr, thread|
+#         err = stderr.read
+#         unless err.empty?
+#             patch_file.close
+#             patch_file.unlink
+#             raise "Failed to apply patch: #{err}"
+#         end
+#     end
+#     patch_file.close
+#     patch_file.unlink
+# end
+
+
+# task = "Expand 'test/test_levenshtein.rb' to improve test coverage. Include edge cases with legit input, as well as incorrect input."
+# #llm_name = "gpt-3.5-turbo"
+# llm_name  = "gpt-4"
+# project_root = "../autocode/ruby-llm"
+# run_command = "cd #{project_root}; ruby test/test_levenshtein.rb"
+
+
+task = "I just wrote examples/improve_prompt/improve_prompt.rb, but it has some syntax errors. Plese try to get it to work."
+#llm_name = "gpt-3.5-turbo"
+llm_name  = "gpt-4"
+project_root = "../autocode/ruby-llm"
+run_command = "cd #{project_root}; ruby examples/improve_prompt/improve_prompt.rb"
+
+perform_task(llm_name:, project_root:, task:, run_command:)

data/examples/autocode/solargraph_test.rb

@@ -0,0 +1,59 @@
+# require 'solargraph'
+# require_relative '../../lib/globals'
+
+
+
+
+#    NOT WORKING, UGH.
+
+
+
+
+# putt :info, "I belong in line 5, which means it's line 4 for solagraph"
+
+
+# require 'solargraph'
+
+
+# def find_definition(filename, lineno, identifier, project_dir)
+#     # Initialize a Solargraph::ApiMap object
+#     api_map = Solargraph::ApiMap.new
+    
+#     # Map all Ruby files in the project directory
+#     source_files = Dir.glob("#{project_dir}/**/*.rb")
+#     puts "Mapping #{source_files.length} Ruby source files from project directory #{project_dir}"
+    
+#     source_files.each do |file|
+#       source = Solargraph::Source.load(file)
+#       puts "Mapping source file: #{file}"
+#       api_map.map source
+#     end
+    
+#     # Use `get_path_pins` to find the pin (definition) for the identifier
+#     pins = api_map.get_path_pins(identifier)
+#     puts "Number of pins found for identifier '#{identifier}': #{pins.length}"
+  
+#     if pins.empty?
+#       return "Definition not found."
+#     else
+#       pin = pins.first
+#       puts "First pin details: #{pin.inspect}"
+#       return pin.location.to_s
+#     end
+#   end
+  
+
+# # Replace these variables with the appropriate values
+# project_path = "."
+# identifier = "GlimRequest"
+# context_file = "examples/autocode/solargraph_test.rb"
+# context_file = "lib/globals.rb"
+
+
+# context_line = 4  # Line numbers are zero-based in Solargraph
+# context_line = 4  # Line numbers are zero-based in Solargraph
+
+
+# source_code = find_definition(context_file, context_line, identifier,".")
+# puts source_code
+

data/examples/autocode/templates/changed_files_now_evaluate_output.erb

@@ -0,0 +1,29 @@
+<% req.replace_initial_system_message <<~SYSTEM 
+
+You are an experienced ruby developer. You will be asked to work on a large project; 
+however, you don't have full access to the source code. Therefore, you have to explicit ask for 
+any files you need. When asking for files, remember that you can use the "require" and "require_relative" 
+statements to determine which files might be helpful. To request the contents of a file, use the functions
+available to you. 
+When you want to make changes to the code, simply provide it in your response, as follows:
+
+#{prompt_output_files}
+
+SYSTEM
+%>
+
+You have just made some changes to the code. Specifically, the following files have changed:
+<%= files_saved.join ','%>.
+
+Running the code again, we got the following output: 
+
+stderr:
+<%= stderr %>
+---
+stdout:
+<%= stdout %>
+---
+
+If it looks like all your changes worked and the output is as expected, call the function "code_ran_as_expected".
+Otherwise, you can provide updated code to address the issues, or call the function show_file to see more of the source
+code.

data/examples/autocode/templates/task.erb

@@ -0,0 +1,16 @@
+<% req.replace_initial_system_message <<~SYSTEM 
+
+You are an experienced ruby developer. You will be asked to work on a large project; 
+however, you don't have full access to the source code. Therefore, you have to explicit ask for 
+any files you need. When asking for files, remember that you can use the "require" and "require_relative" 
+statements to determine which files might be helpful. To request the contents of a file, use the functions
+available to you. 
+
+When you want to make changes to the code, simply provide the whole file (not just your additions!) in your response, as follows:
+
+#{prompt_output_files}
+
+SYSTEM
+%>
+
+<%= task %>

data/examples/calc/calc.rb

@@ -0,0 +1,50 @@
+
+require_relative '../../lib/globals'
+require_relative '../../lib/glim_ai_callable'
+
+class CalculatorService
+include AICallable
+  ai_callable_as :evaluate_expression do
+    describe "Evaluates the given ruby expression and returns the result."
+    string :exp, "The expression, as a string, in correct ruby syntax", ai_name: :expression_to_evaluate, required: true
+  end
+  def evaluate_expression(exp:)
+    # Add validation logic here
+    return eval(exp).to_s
+  end
+
+end
+
+# puts CalculatorService.ai_method_signatures
+
+glim = GlimContext.new
+
+llm_name = "gpt-3.5-turbo"
+
+calc = CalculatorService.new
+raise "Calculator Service must be AICallable" unless calc.is_a?(AICallable)
+puts "Let's test the calculator locally: "
+exp = "   1+2"
+puts exp + "=" + calc.send(:evaluate_expression, exp:).to_s
+
+puts("And now let's get GPT to use it:")
+req = glim.request(llm_name:)
+req.set_functions_object(calc)
+
+req.prompt = "What is the resistance of a 100m long copper cable that has a 6mm^2 cross section?"
+puts "First question to GPT: #{req.prompt}"
+response = req.response
+puts "   response.completion:"
+puts response.completion
+
+# this will also invoke the function
+new_req = response.create_request_with_function_result
+
+# and now we send the request with the result of the function
+# evaluation, so that the LLM can use that
+new_response = new_req.response
+
+puts("new_response.messages:")
+puts(JSON.pretty_generate(new_response.messages))
+puts "new_response.completion:"
+puts new_response.completion

data/examples/code_competition/code_competition.rb

@@ -0,0 +1,78 @@
+require_relative '../../lib/globals'
+
+llm_names =  ["claude-instant-1", "claude-2", "gpt-3.5-turbo-16k", "gpt-4" ]
+
+glim = GlimContext.new
+
+code = {} # lang,llm_name -> code
+for lang in ["ruby", "python"] #, "javascript"]
+    for llm_name in llm_names
+        req = glim.request_from_template("write_code", language: lang)
+        req.llm_name = llm_name
+        code[[lang,llm_name]] = req.response 
+        template_text = req.template_text
+    end
+end
+
+for lang, llm_name in code.keys
+    code[[lang,llm_name]] = code[[lang,llm_name]].completion
+    File.write("examples/output/#{lang}_#{llm_name}.rb", code[[lang,llm_name]])
+end
+
+code[["ruby","human"]] = File.read("examples/output/ruby_human.rb")
+ratings = {}
+
+for llm_name in llm_names
+    puts "analyze_code: #{llm_name}"
+    request = glim.request_from_template("analyze_code", code:, text: template_text)
+    request.llm_name = llm_name
+    ratings[llm_name] = request.response
+end
+
+# TODO - would need to support arrays as args, first. 
+
+# class CodeRatings
+# include AICallable
+
+# def initialize(developer_name)
+#     @developer_name = developer_name
+# end
+
+# ai_callable_as :add_ratings do
+#     describe "Report ratings extracted from the text"
+#     string :lang, "The language of the code", required: true
+#     number :elegance, "The elegance of the code", required: true
+#     number :parallelism, "The parallelism of the code", required: true
+#     number :correctness, "The correctness of the code", required: true
+#     number :instructions_conformity, "The instructions conformity of the code", required: true
+# end
+# def add_ratings(**args)
+
+
+response_by_llm_name = {}
+for llm_name in llm_names
+    req = glim.request(llm_name: "gpt-3.5-turbo")
+    req.set_output_schema({
+        type: "object",
+        properties: {
+            lang: { type: "string" },
+            llm_name:  { type: "string" },
+            elegance: {type: "number"},
+            parallelism: {type: "number"},
+            correctness: {type: "number"},
+            instructions_conformity: {type: "number"}
+        }   
+    }, :list)
+    text = ratings[llm_name].completion
+    req.prompt = "Extract all of the json data from the following text:\n\n#{text}" 
+    response_by_llm_name[llm_name] = req.response
+end
+
+ratings_by_llm_name = {}
+for llm_name in llm_names
+    ratings_by_llm_name[llm_name] = response_by_llm_name[llm_name].extracted_data
+end
+
+# puts JSON.pretty_generate(ratings_by_llm_name)
+
+# TODO -- analyze results