glim_ai 0.2.0

data/examples/code_competition/output/python_claude-2.rb

@@ -0,0 +1,33 @@
+ ### library (python)
+# no lib needed
+
+### problem specific code
+import asyncio
+
+async def Q(msg, fut):
+    await asyncio.sleep(0.1) # pretend this is an API call
+    fut.set_result(msg)
+    return fut
+
+async def f(m1, m2, m3):
+    return [await m1, await m2, await m3]
+
+async def g(m1, m2):
+    return m1[0] + m2[1]
+
+prompt1 = asyncio.Future()
+prompt1.set_result('code')
+
+m1 = Q('analyzing code 1', asyncio.Future())
+m2 = Q('analyzing code 2', asyncio.Future()) 
+m3 = Q('analyzing code 3', asyncio.Future())
+
+prompt2 = asyncio.gather(f(m1, m2, m3))
+
+m1 = Q('summary 1', asyncio.Future())
+m2 = Q('summary 2', asyncio.Future())
+
+prompt3 = g(m1, m2)
+
+result = asyncio.gather(prompt3)
+print(asyncio.run(result))

data/examples/code_competition/output/python_claude-instant-1.rb

@@ -0,0 +1,18 @@
+ '''
+### library (python) 
+# no lib needed
+
+### problem specific code
+import asyncio
+
+async def prompt2_analyze_code():
+    tasks = [Q(m1,"prompt1_write_code"), Q(m2,"prompt1_write_code"), Q(m3,"prompt1_write_code")]
+    await asyncio.gather(*tasks)
+
+async def prompt3_summarize():
+    tasks = [Q(m1, await prompt2_analyze_code()), Q(m2, await prompt2_analyze_code())]
+    await asyncio.gather(*tasks)
+
+asyncio.run(prompt3_summarize())
+result = Q(m1, await prompt3_summarize())
+'''

data/examples/code_competition/output/python_gpt-3.5-turbo-16k.rb

@@ -0,0 +1,69 @@
+### library ( python )
+
+```python
+import threading
+
+class APICall:
+    def __init__(self, func, args):
+        self.func = func
+        self.args = args
+        self.result = None
+        self.finished = False
+        self.lock = threading.Lock()
+
+    def finish(self, result):
+        with self.lock:
+            self.result = result
+            self.finished = True
+            self.lock.notify_all()
+
+class Q:
+    def __init__(self, module, prompt):
+        self.module = module
+        self.prompt = prompt
+
+    def __call__(self, *args):
+        api_call = APICall(self.module.Q, (self.prompt,) + args)
+        threading.Thread(target=self.module.Q, args=((self.prompt,) + args, api_call.finish)).start()
+        return api_call
+
+### problem specific code
+
+def f(*args):
+    print("API call f with args:", args)
+    # Perform the API call using external library or code
+    return "Some result from f"
+
+def g(*args):
+    print("API call g with args:", args)
+    # Perform the API call using external library or code
+    return "Some result from g"
+
+def prompt1_write_code(arg):
+    print("Prompt 1:", arg)
+    # Perform some operation using external library or code
+
+def prompt2_analyze_code(arg):
+    print("Prompt 2:", arg)
+    # Perform some operation using external library or code
+
+def prompt3_summarize(arg):
+    print("Prompt 3:", arg)
+    # Perform some operation using external library or code
+
+m1 = None  # Placeholder for module 1, replace with actual module
+m2 = None  # Placeholder for module 2, replace with actual module
+m3 = None  # Placeholder for module 3, replace with actual module
+
+def main():
+    prompt2_analyze_code = f(Q(m1, prompt1_write_code), Q(m2, prompt1_write_code), Q(m3, prompt1_write_code))
+    prompt3_summarize = g(Q(m1, prompt2_analyze_code), Q(m2, prompt2_analyze_code))
+    result = Q(m1, prompt3_summarize)
+    
+    print("Result:", result.result)  # Wait for the result, if needed
+
+if __name__ == '__main__':
+    main()
+```
+
+Note: Replace the `print` statements and function bodies with actual implementation according to your requirements.

data/examples/code_competition/output/python_gpt-3.5-turbo.rb

@@ -0,0 +1,43 @@
+### library ( python )
+```python
+import threading
+
+class Q:
+    def __init__(self, caller, func):
+        self.caller = caller
+        self.func = func
+        self.result = None
+        self.thread = None
+        self.lock = threading.Lock()
+
+    def __call__(self, *args, **kwargs):
+        if self.result is None:
+            with self.lock:
+                if self.result is None:
+                    self.thread = threading.Thread(target=self.execute, args=args, kwargs=kwargs)
+                    self.thread.start()
+        self.thread.join()
+        return self.result
+
+    def execute(self, *args, **kwargs):
+        self.result = self.func(*args, **kwargs)
+
+def f(*args, **kwargs):
+    return sum(args)
+
+def g(*args, **kwargs):
+    return sum(args)
+```
+
+### problem specific code
+```python
+m1 = "John"
+m2 = "Jane"
+m3 = "David"
+
+prompt1_write_code = "Write code"
+
+prompt2_analyze_code = f(Q(m1, prompt1_write_code), Q(m2, prompt1_write_code), Q(m3, prompt1_write_code))
+prompt3_summarize = g(Q(m1, prompt2_analyze_code), Q(m2, prompt2_analyze_code))
+result = Q(m1, prompt3_summarize)
+```

data/examples/code_competition/output/python_gpt-4.rb

@@ -0,0 +1,34 @@
+'''
+### library ( python )
+
+import asyncio
+import concurrent.futures
+
+class AsyncAPI:
+    def __init__(self):
+        self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=5)
+
+    def Queue(self, function, *args):
+        loop = asyncio.get_event_loop()
+        return loop.run_in_executor(self.executor, function, *args)
+
+asyncApi = AsyncAPI()
+
+### problem specific code
+
+def Q(model, task):
+    ## write the actual code for api call
+    pass
+
+def f(*results):
+    ## write the actual code for analyze code
+    pass
+
+def g(*results):
+    ## write the actual code for summarize 
+    pass
+
+prompt2_analyze_code = f( asyncApi.Queue(Q,m1,prompt1_write_code), asyncApi.Queue(Q,m2,prompt1_write_code), asyncApi.Queue(Q,m3,prompt1_write_code) ) 
+prompt3_summarize = g( asyncApi.Queue(Q,m1,prompt2_analyze_code), asyncApi.Queue(Q,m2,prompt2_analyze_code) )
+result = asyncApi.Queue(Q, m1, prompt3_summarize)
+'''

data/examples/code_competition/output/ruby_claude-2.rb

@@ -0,0 +1,22 @@
+ ### library (ruby)
+# no lib needed
+
+### problem specific code
+
+m1 = Q.async(:prompt1_write_code)
+m2 = Q.async(:prompt1_write_code) 
+m3 = Q.async(:prompt1_write_code)
+
+m1.wait
+m2.wait
+m3.wait
+
+prompt2_analyze_code = f.async(m1, m2, m3)
+
+m1.wait
+m2.wait 
+m3.wait
+
+prompt3_summarize = g.async(m1, m2)
+
+result = prompt3_summarize.wait

data/examples/code_competition/output/ruby_claude-instant-1.rb

@@ -0,0 +1,20 @@
+ ### library ( ruby )
+# no lib needed
+
+### problem specific code
+require 'concurrent'
+
+executor = Concurrent::CachedThreadPool.new
+
+prompt2_analyze_code = -> do
+  f(executor.post { Q(m1, prompt1_write_code) }, 
+    executor.post { Q(m2, prompt1_write_code) },
+    executor.post { Q(m3, prompt1_write_code) })
+end
+
+prompt3_summarize = -> do
+  g(executor.post { prompt2_analyze_code.call }, 
+    executor.post { prompt2_analyze_code.call })  
+end
+
+result = executor.post { prompt3_summarize.call }.await

data/examples/code_competition/output/ruby_gpt-3.5-turbo-16k.rb

@@ -0,0 +1,27 @@
+### library (ruby)
+
+No library is needed for this problem.
+
+### problem specific code
+
+```ruby
+def Q(module_name, statement)
+  # Code to make API call and return the result
+  # (Assuming implementation for making API call is already done)
+end
+
+def f(*statements)
+  # Code to process multiple statements asynchronously
+end
+
+def g(*statements)
+  # Code to process multiple statements asynchronously
+end
+
+# Example usage
+prompt2_analyze_code = f(Q(:m1, Q(:prompt1_write_code)), Q(:m2, Q(:prompt1_write_code)), Q(:m3, Q(:prompt1_write_code)))
+prompt3_summarize = g(Q(:m1, prompt2_analyze_code), Q(:m2, prompt2_analyze_code))
+result = Q(:m1, prompt3_summarize)
+```
+
+Note: The implementation of making API calls and processing statements asynchronously is not provided as it is specific to the API being used and the requirements of processing the statements.

data/examples/code_competition/output/ruby_gpt-3.5-turbo.rb

@@ -0,0 +1,30 @@
+### library (ruby)
+```ruby
+module Q
+  def self.call(api, prompt)
+    # logic for making an API call
+  end
+end
+```
+
+### problem specific code
+```ruby
+m1 = "m1"
+m2 = "m2"
+m3 = "m3"
+
+prompt1_write_code = "prompt1_write_code"
+
+prompt2_analyze_code = Q.call(m1, prompt1_write_code)
+Q.call(m2, prompt1_write_code)
+Q.call(m3, prompt1_write_code)
+
+prompt3_summarize = Q.call(m1, prompt2_analyze_code)
+Q.call(m2, prompt2_analyze_code)
+
+result = Q.call(m1, prompt3_summarize)
+```
+
+In the above code, I have created a module `Q` which defines a `call` method to make the API call. The `call` method takes two parameters - the API name and the prompt. This allows us to easily make API calls without waiting for the answer, unless the answer is needed to proceed.
+
+The `problem specific code` section shows how the `Q` module can be used to solve the problem mentioned in the example. The API calls are made in the desired sequence, and the result is obtained by making the necessary API calls in the desired order.

data/examples/code_competition/output/ruby_gpt-4.rb

@@ -0,0 +1,31 @@
+```ruby
+### library ( ruby )
+
+require 'concurrent'
+
+class Q
+  def initialize(m, action)
+    @future = Concurrent::Future.execute do
+      m.send(action)
+    end
+  end
+
+  def result
+    @future.value
+  end
+end
+
+### problem specific code
+
+def f(*actions)
+  actions.map(&:result)
+end
+
+def g(*actions)
+  actions.map(&:result)
+end
+
+prompt2_analyze_code = f( Q.new(m1, :prompt1_write_code), Q.new(m2, :prompt1_write_code), Q.new(m3, :prompt1_write_code)) 
+prompt3_summarize = g( Q.new(m1, :prompt2_analyze_code), Q.new(m2, :prompt2_analyze_code)) 
+result = Q.new(m1, :prompt3_summarize).result
+```

data/examples/code_competition/output/ruby_human.rb

@@ -0,0 +1,41 @@
+
+# library code - can be reused for other applications
+class Future
+  def initialize(&block)
+    @thread = Thread.new(&block)
+  end
+
+  def value
+    @thread.value
+  end
+end
+
+####################
+# code specific to problem
+
+def Q(m, p)
+  Future.new do
+    # Define the asynchronous operation here
+  end
+end
+
+p1 = # define p1
+m1 = # define m1
+m2 = # define m2
+
+# Create futures for Q(m1, p1) and Q(m2, p1), will be evaluated in parallel
+q1_p1 = Q(m1, p1)
+q2_p1 = Q(m2, p1)
+
+# Retrieve values and calculate p2; this blocks until both values are there
+p2 = f(q1_p1.value, q2_p1.value)
+
+# Create futures for Q(m1, p2) and Q(m2, p2), will be evaluated in parallel
+q1_p2 = Q(m1, p2)
+q2_p2 = Q(m2, p2)
+
+# Retrieve values and calculate p3, blocks until both values are there
+p3 = g(q1_p2.value, q2_p2.value)
+
+# Calculate the final result
+result = Q(m1, p3).value # blocks until value is there

data/examples/code_competition/templates/analyze_code.erb

@@ -0,0 +1,33 @@
+I gave the following programming task to some LLMs and a human, in different programming languages:
+'''
+<%= text %>
+'''
+Below are the answers:
+<% for l, m in code.keys %>
+---l
+Language: <%= l %>
+Author: <%= m %>
+'''
+<%= code[[l,m]] %>
+'''
+<% end %>
+---
+Write a brief critique of each answer. Carefully check if the provided code will work and if it maximizes parallelism.
+Rate the elegance of the problem-specific code; disregard the library code for rating elegance.
+Then, analyze which language works better for this, and which LLMs did best. 
+Finally: Can you draw any conclusions about connections between the LLMs and the languages?
+For example, does it seem like a particular LLM prioritizes patterns that are more common in one language than another?
+At the end of your answer, after a "---" as a separator, write a json array with the various ratings for each one, on a scale of 0..1, like this:
+---
+[
+<% for l, m in code.keys %>
+  {
+    lang: <%=l%>, 
+    model: <%=m%>, 
+    elegance: 0.42, # reason for elegance score
+    parallelism: 0.42, # reason for parallelism score
+    correctness: 0.42, # reason for correctness score
+    instructions_conformity: 0.42,}, # reason this score; it should measure how well the answer conforms to the instructions
+<% end %>
+]
+(Replace 0.42 with your rating for each answer)

data/examples/code_competition/templates/write_code.erb

@@ -0,0 +1,26 @@
+
+I am working on a  <%= language %> library for developing software that frequently requires use of external APIs. 
+It can take a while for the external API calls to finish, so we want to make it easy for the developer 
+to send API calls without waiting for the answer, unless the answer is needed to proceed. 
+
+For example:
+'''
+prompt2_analyze_code = f( Q(m1,prompt1_write_code), Q(m2,prompt1_write_code), Q(m3,prompt1_write_code)) 
+prompt3_summarize = g( Q(m1,prompt2_analyze_code), Q(m2,prompt2_analyze_code)) 
+result = Q(m1,prompt3_summarize)
+'''
+The API calls are in function Q. Q,f,g do not have side effects we need to worry about.
+
+Write a <%= language %> library for this, and then show how it can be used for the example above. 
+If no library is needed, or one already exists, then no need to write one - just write "# no lib needed"
+
+Structure your code like this:
+'''
+### library ( <%= language %> )
+(library code if needed, or # no lib needed)
+
+### problem specific code
+(elegant code that solves the problem above)
+'''
+Your goal is to make the (non-library) code as short, readable, and elegant as possible.
+Do not explain your code -- only respond with the code itself. 

data/examples/glim_demo/ask_all.rb

@@ -0,0 +1,35 @@
+require_relative '../../lib/globals'
+glim = GlimContext.new(log_name: "ask_all")
+# these first two lines above are the only ones that you need to add to your code
+
+# in this example, we will want to compare the answers of these different models
+models =  ["claude-instant-1",  "gpt-3.5-turbo"]
+
+# we will ask this question to each model
+question = "If, in some cataclysm, all of scientific knowledge were to be destroyed, and only one sentence passed on to the next generation of creatures, what statement would contain the most information in the fewest words?"
+
+responses = {}
+for model in models
+    # construct a request that will be sent to the LLM
+    request = glim.request(llm_name: model)
+    request.prompt = question
+    # LLMResponse.compute will send the request to the model specified in the request
+    responses[model] = request.response
+end
+
+# now we can rate and summarize  the answers
+
+# construct a request using an erb template. The template is in the specs directory
+# and is called "rate_all.erb". We will pass the question and the hash with all of the
+# answers to the template.
+request = glim.request_from_template("rate_all", question:, answers: responses)
+
+# the request now contains a prompt that is based on the template and the
+# arguments that we passed to the template (question and answers)
+puts request.inspect
+
+# send the request and print the completion it generated
+response = request.response
+puts response.completion
+
+

data/examples/glim_demo/templates/rate_all.erb

@@ -0,0 +1,24 @@
+<%
+# This is the template for rendering the prompt for rating all answers to a question.
+req.llm_name = 'gpt-3.5-turbo'
+%>
+<%= prompt_output_files %>
+
+---
+
+Below are different answers to the following question:
+"<%= question # this will insert the question passed in to the erb template
+%>"
+
+<% 
+# iterate over the answers we want to rate so that they will all be in the prompt
+answers.each_pair do |model, answer| 
+%>
+<%= model %>'s response: <%= answer.completion %>
+<% end %>
+
+<% 
+# now we will instruct the LLM to rate each of the answers: 
+%>
+Write a brief critique of each answer. 
+Then, generate a string in json format providing, for each respondent, a rating for that respondent's response on a scale of 0 to 1.

data/examples/improve_prompt/improve_prompt.rb

@@ -0,0 +1,62 @@
+require_relative '../../lib/globals'
+
+glim = GlimContext.new
+
+testcases = [
+    # [ "gen_two_files", "Generate two files named f1 and f2, which each contain the world `hello` and nothing else" ],
+    # [ "gen_word_list", "Generate a file named `word_list` which contains the first 5 words from NATO phonetic alphabet, each in its own line." ],
+    # [ "write_code",  "Write a program in ruby called 'count_lines.rb' which reads a file and prints the number of lines in it. Also generate a file for testing your code."],
+    [ "gen_word_list_subdir", "Generate a file named `word_list` which contains the first 5 words from NATO phonetic alphabet, each in its own line. It should go into a subdirectory called 'fun_words'" ]
+]
+
+llm_names = ["gpt-3.5-turbo", "claude-instant-1"]
+
+responses = {}
+
+Dir.glob(File.join(__dir__, "templates/try_*.erb")) do |try_path|
+    try = File.basename(try_path,'.erb')    
+    responses[try] = {} 
+    for test_name, test_prompt in testcases
+        responses[try][test_name] = {}
+        for llm_name in llm_names        
+            #puts "LLM = #{llm_name}, testing #{try} with prompt #{test_prompt}"
+            req = glim.request_from_template(try, test_prompt: test_prompt)
+            req.llm_name = llm_name
+            req.temperature = 0.0
+            responses[try][test_name][llm_name] = req.response
+        end
+    end
+end
+
+extracted_info = {}
+for try in responses.keys
+    extracted_info[try] = {}
+    for test_name in responses[try].keys
+        extracted_info[try][test_name] = {}
+        baseline_extracted_info = nil
+        baseline_completion = nil
+        for llm_name in llm_names # we want them in this order because first one is the gold standard
+            completion = responses[try][test_name][llm_name].completion
+            extracted_by_llm = extract_and_save_files(completion)
+            if !baseline_extracted_info
+                baseline_extracted_info = extracted_by_llm
+                baseline_completion = completion
+                next
+            end
+            info = ""
+            if baseline_extracted_info[0] != extracted_by_llm[0]
+                info += "\n\nExtracted info_text differs:\n#{extracted_info[0]}."
+                info += "\nBaseline was:\n#{baseline_extracted_info[0]}."
+            end
+            # if baseline_extracted_info[1] != extracted_by_llm[1]
+            #     info += "\n\nExtracted files differ: #{JSON.pretty_generate(extracted_by_llm[1])}."
+            #     info += "\nBaseline was: #{JSON.pretty_generate(baseline_extracted_info[1])}."
+            # end
+            if !info.empty?
+                puts "\n\n#{try} on test case #{test_name} with #{llm_name}:"
+                puts info
+                puts "\nCompletion was: \n#{completion}"
+            end
+        end
+    end
+end

data/examples/improve_prompt/templates/stashed/prompt_attempt_explicit_steps.erb

@@ -0,0 +1,15 @@
+<% req.replace_initial_system_message <<~SYSTEM 
+
+ALWAYS, when asked to generate one or more files, include such files in your response as follows:
+1. Write the following file separator line:
+```
+2. Write a line containing # followed by the relative pathname of the file, for example:
+# File: relative_path_to_file/filename.suffix
+3. Write out the content of the file, line by line
+4. Write the following file separator line again:
+```
+
+SYSTEM
+%>
+
+<%= test_prompt %>

data/examples/improve_prompt/templates/stashed/prompt_attempt_explicit_steps_user_message.erb

@@ -0,0 +1,15 @@
+
+
+SYSTEM MESSAGE: ALWAYS, when asked to generate one or more files, include such files in your response as follows:
+1. Write the following file separator line:
+```
+2. Write a line containing # followed by the relative pathname of the file, for example:
+# File: relative_path_to_file/filename.suffix
+3. Write out the content of the file, line by line
+4. Write the following file separator line again:
+```
+
+---
+
+
+<%= test_prompt %>

data/examples/improve_prompt/templates/stashed/prompt_attempt_initial.erb

@@ -0,0 +1,8 @@
+<% req.replace_initial_system_message <<~SYSTEM 
+
+#{prompt_output_files}
+
+SYSTEM
+%>
+
+<%= test_prompt %>

data/examples/improve_prompt/templates/stashed/prompt_attempt_nothing.erb

@@ -0,0 +1,19 @@
+
+
+SYSTEM MESSAGE: ALWAYS, when asked to generate one or more files, include such files in your response as follows:
+1. Write the following file separator line:
+```
+2. Write a line containing # followed by the relative pathname of the file, for example:
+# File: relative_path_to_file/filename.suffix
+3. Write out the content of the file, line by line
+4. Write the following file separator line again:
+```
+
+---
+
+
+<%= test_prompt %>
+
+
+
+<%= test_prompt %>

data/examples/improve_prompt/templates/try_code_first.erb

@@ -0,0 +1,13 @@
+
+SYSTEM MESSAGE: ALWAYS, when asked to generate source code or other text files, use the following format:
+<file pathname="path_to_file/hello.rb">
+puts "Hello from Line 1"
+puts "hello from Line 2"
+</file>
+So, the example above shows how you would include a file called "hello.rb" that belongs in the subdirectory "path_to_file" of the current directory. 
+The file would contain two "puts" statements. 
+Use this for all text files you generate, not just source code.
+
+---
+
+<%= test_prompt %>

data/examples/improve_prompt/templates/try_code_first_system.erb

@@ -0,0 +1,22 @@
+<% req.replace_initial_system_message <<~SYSTEM 
+
+When asked to generate source code or other text files, use the following format:
+<file pathname="path_to_file/hello.rb">
+puts "Hello from Line 1"
+puts "hello from Line 2"
+</file>
+So, the example above shows how you would include a file called "hello.rb" that belongs in the subdirectory "path_to_file" of the current directory. 
+The file would contain two "puts" statements. 
+Use this for all text files you generate, not just source code.
+
+SYSTEM
+%>
+
+
+<%= test_prompt %>
+
+
+
+---
+
+<%= test_prompt %>