promptcraft 0.1.0

data/examples/maths/start/already_answered_multiple_providers.yml

@@ -0,0 +1,30 @@
+---
+system_prompt: I like to solve maths problems.
+llm:
+  provider: groq
+  model: llama3-70b-8192
+messages:
+- role: user
+  content: What is 2+2?
+- role: assistant
+  content: That's an easy one! The answer is... 4!
+---
+system_prompt: I like to solve maths problems.
+llm:
+  provider: groq
+  model: llama3-8b-8192
+messages:
+- role: user
+  content: What is 2+2?
+- role: assistant
+  content: The answer to 2+2 is 4.
+---
+system_prompt: I like to solve maths problems.
+llm:
+  provider: openai
+  model: gpt-4-turbo
+messages:
+- role: user
+  content: What is 6 divided by 2?
+- role: assistant
+  content: 6 divided by 2 is 3.

data/examples/maths/start/another_question.yml

@@ -0,0 +1,11 @@
+---
+system_prompt: |-
+  I like to solve maths problems.
+
+messages:
+- role: "user"
+  content: "What is 2+2?"
+- role: assistant
+  content: 2 + 2 = 4
+- role: user
+  content: "What is 3+3?"

data/examples/maths/start/basic.yml

@@ -0,0 +1,9 @@
+---
+system_prompt: |-
+  I like to solve maths problems.
+
+messages:
+- role: "user"
+  content: "What is 2+2?"
+
+# Missing assistant final response so next iteration will add it.

data/examples/maths/start/many_limericks.yml

@@ -0,0 +1,51 @@
+---
+system_prompt: I am excellent at limericks.
+llm:
+  provider: groq
+  model: llama3-70b-8192
+messages:
+- role: user
+  content: A limerick about a llama.
+- role: assistant
+  content: |-
+    Here is a limerick about a llama:
+
+    There once was a llama named Pete,
+    Whose fur was so soft, it was neat.
+    He'd spit with a flair,
+    When his space was not fair,
+    And his hums were so loud, they'd repeat!
+---
+system_prompt: I am excellent at limericks.
+llm:
+  provider: groq
+  model: llama3-70b-8192
+messages:
+- role: user
+  content: A limerick about cats.
+- role: assistant
+  content: |-
+    Here is a limerick about cats:
+
+    There once was a feline so fine,
+    Whose whiskers were long and divine.
+    She'd purr with delight,
+    As she hunted at night,
+    And her claws were as sharp as a vine.
+---
+system_prompt: I am excellent at limericks.
+llm:
+  provider: groq
+  model: llama3-70b-8192
+messages:
+- role: user
+  content: A limerick about a dog.
+- role: assistant
+  content: |-
+    Here is a limerick about a dog:
+
+    There once was a pup named Pete,
+    Whose wagging tail couldn't be beat.
+    He'd chase after balls,
+    And fetch them through halls,
+    And always bring back a treat to eat.

data/examples/maths/start/prompt.yml

@@ -0,0 +1,3 @@
+---
+system_prompt: |-
+  I like to solve maths problems.

data/examples/maths/start/system_only.yml

@@ -0,0 +1,5 @@
+---
+system_prompt: |-
+  I like to solve maths problems.
+
+messages: []

data/exe/promptcraft

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "promptcraft"
+
+cmd = Promptcraft::Cli::RunCommand.new
+
+cmd.parse.run(stdin: $stdin)

data/lib/promptcraft/cli/run_command.rb

@@ -0,0 +1,177 @@
+require "concurrent"
+require "langchain"
+require "tty-option"
+
+# Pick an LLM provider + model:
+#   promptcraft --provider groq
+#   promptcraft --provider openai --model gpt-3.5-turbo
+# Pass in a prompt via CLI (-p,--prompt expects a string, or filename)
+#   promptcraft -c tmp/maths/start/basic.yml -p "I'm terrible at maths. If I'm asked a maths question, I reply with a question."
+#   promptcraft -c tmp/maths/start/basic.yml -p <(echo "I'm terrible at maths. If I'm asked a maths question, I reply with a question.")
+# The prompt file can also be YAML with system_prompt: key.
+class Promptcraft::Cli::RunCommand
+  include TTY::Option
+
+  usage do
+    program "promptcraft"
+
+    command "run"
+
+    desc "Re-run conversation against new system prompt"
+  end
+
+  option :conversation do
+    arity zero_or_more
+    short "-c"
+    long "--conversation filename"
+    desc "Filename of conversation (or use STDIN)"
+  end
+
+  option :prompt do
+    short "-p"
+    long "--prompt prompt"
+    desc "String or filename containing system prompt"
+  end
+
+  flag :help do
+    short "-h"
+    long "--help"
+    desc "Print usage"
+  end
+
+  option :model do
+    short "-m"
+    long "--model model_name"
+    desc "Model name to use for chat completion"
+  end
+
+  option :provider do
+    long "--provider provider_name"
+    desc "Provider name to use for chat completion"
+  end
+
+  option :format do
+    short "-f"
+    long "--format format"
+    desc "Output format (yaml, json)"
+    default "yaml"
+  end
+
+  option :threads do
+    long "--threads threads"
+    desc "Number of threads to use for concurrent processing"
+    convert :int
+    default 5
+  end
+
+  # TODO: --debug
+  # * faraday debugging
+  # * Promptcraft::Llm.new(debug: true)
+  flag :debug do
+    long "--debug"
+    desc "Enable debug mode"
+  end
+
+  def run(stdin: nil)
+    if params[:help]
+      warn help
+    elsif params.errors.any?
+      warn params.errors.summary
+    else
+      # Load files in threads
+      pool = Concurrent::FixedThreadPool.new(params[:threads])
+      conversations = Concurrent::Array.new
+      # TODO: load in thread pool
+      (params[:conversation] || []).each do |filename|
+        pool.post do
+          # check if --conversation=filename is an actual file, else store it in StringIO and pass to load_from_io
+          if File.exist?(filename)
+            conversations.push(*Promptcraft::Conversation.load_from_file(filename))
+          else
+            conversations.push(*Promptcraft::Conversation.load_from_io(StringIO.new(filename)))
+          end
+        end
+      end
+      pool.shutdown
+      pool.wait_for_termination
+
+      # if STDIN piped into the command, read stream of YAML conversations from STDIN
+      if io_ready?(stdin)
+        conversations.push(*Promptcraft::Conversation.load_from_io(stdin))
+      end
+
+      if conversations.empty?
+        conversations << Promptcraft::Conversation.new(system_prompt: "You are helpful. If you're first, then ask a question. You like brevity.")
+      end
+
+      if (prompt = params[:prompt])
+        # if prompt is a file, load it; else set the prompt to the value
+        new_system_prompt = if File.exist?(prompt)
+          File.read(prompt)
+        else
+          prompt
+        end
+
+        # If new_system_prompt is YAML and a Hash, use "system_prompt" key
+        begin
+          obj = YAML.load(new_system_prompt, symbolize_keys: true)
+          if obj.is_a?(Hash) && obj[:system_prompt]
+            new_system_prompt = obj[:system_prompt]
+          end
+        rescue
+        end
+      end
+
+      # Process each conversation in a concurrent thread via a thread pool
+      pool = Concurrent::FixedThreadPool.new(params[:threads])
+      mutex = Mutex.new
+
+      updated_conversations = Concurrent::Array.new
+      conversations.each do |conversation|
+        pool.post do
+          # warn "Post processing conversation for #{conversation.messages.inspect}"
+          llm = if params[:provider]
+            Promptcraft::Llm.new(provider: params[:provider], model: params[:model])
+          elsif conversation.llm
+            conversation.llm
+          else
+            Promptcraft::Llm.new
+          end
+          llm.model = params[:model] if params[:model]
+
+          system_prompt = new_system_prompt || conversation.system_prompt
+
+          cmd = Promptcraft::Command::RechatConversationCommand.new(system_prompt:, conversation:, llm:)
+          cmd.execute
+          updated_conversations << cmd.updated_conversation
+
+          mutex.synchronize do
+            dump_conversation(cmd.updated_conversation, format: params[:format])
+          end
+        rescue => e
+          mutex.synchronize do
+            warn "Error: #{e.message}"
+            warn "for conversation: #{conversation.inspect}"
+          end
+        end
+      end
+      pool.shutdown
+      pool.wait_for_termination
+    end
+  end
+
+  # Currently we support only streaming YAML and JSON objects so can immediately
+  # dump them to STDOUT
+  def dump_conversation(conversation, format:)
+    if format == "json"
+      puts conversation.to_json
+    else
+      puts conversation.to_yaml
+    end
+  end
+
+  def io_ready?(io)
+    return false unless io
+    IO.select([io], nil, nil, 5)
+  end
+end

data/lib/promptcraft/cli.rb

@@ -0,0 +1,3 @@
+module Promptcraft::Cli
+  autoload :RunCommand, "promptcraft/cli/run_command"
+end

data/lib/promptcraft/command/llm_chat_command.rb

@@ -0,0 +1,22 @@
+require "langchain"
+
+class Promptcraft::Command::LlmChatCommand
+  attr_reader :messages, :llm
+
+  def initialize(messages:, llm:)
+    @messages = messages
+    @llm = llm
+  end
+
+  def execute
+    # cleanse messages of missing content, role, etc
+    messages = @messages.reject { |m| m[:content].nil? || m[:content].empty? || m[:role].nil? || m[:role].empty? }
+    response = @llm.chat(messages:)
+
+    response_text = response.chat_completion
+    {role: "assistant", content: response_text}
+  rescue => e
+    puts e.message
+    raise
+  end
+end

data/lib/promptcraft/command/rechat_conversation_command.rb

@@ -0,0 +1,37 @@
+class Promptcraft::Command::RechatConversationCommand
+  include Promptcraft::Helpers
+
+  def initialize(system_prompt:, conversation:, llm:)
+    @system_prompt = system_prompt
+    @conversation = conversation
+    @llm = llm
+  end
+
+  attr_accessor :system_prompt, :conversation, :llm
+  attr_reader :updated_conversation
+
+  # At each point in @conversation messages where the assistant has replied, or not yet replied,
+  # then ask the LLM to re-chat the preceding messages and generate a new response.
+  def execute
+    @updated_conversation = Promptcraft::Conversation.new(system_prompt:, llm:)
+
+    conversation.messages.each do |message|
+      message = deep_symbolize_keys(message)
+      role = message[:role]
+      if role == "assistant"
+        messages = @updated_conversation.to_messages
+        response_message = Promptcraft::Command::LlmChatCommand.new(messages: messages, llm: @llm).execute
+        @updated_conversation.messages << response_message
+      else
+        @updated_conversation.messages << message
+      end
+    end
+
+    # if last message is from user, then ask the LLM to generate a response
+    unless @updated_conversation.messages.last&.dig(:role) == "assistant"
+      messages = @updated_conversation.to_messages
+      response_message = Promptcraft::Command::LlmChatCommand.new(messages: messages, llm: @llm).execute
+      @updated_conversation.messages << response_message
+    end
+  end
+end

data/lib/promptcraft/command.rb

@@ -0,0 +1,4 @@
+module Promptcraft::Command
+  autoload :LlmChatCommand, "promptcraft/command/llm_chat_command"
+  autoload :RechatConversationCommand, "promptcraft/command/rechat_conversation_command"
+end

data/lib/promptcraft/conversation.rb

@@ -0,0 +1,105 @@
+require "yaml"
+
+class Promptcraft::Conversation
+  include Promptcraft::Helpers
+  extend Promptcraft::Helpers
+
+  attr_accessor :system_prompt, :messages
+  attr_accessor :llm
+
+  def initialize(system_prompt:, messages: [], llm: nil)
+    @system_prompt = system_prompt
+    @messages = messages
+    @llm = llm
+  end
+
+  def add_message(role:, content:)
+    @messages << {role:, content:}
+  end
+
+  class << self
+    def load_from_io(io = $stdin)
+      conversations = []
+      begin
+        YAML.load_stream(io) do |doc|
+          next unless doc
+          conversations << build_from(doc)
+        end
+      rescue Psych::SyntaxError => e
+        warn "Error: #{e.message}"
+        warn "Contents:\n#{io.read}"
+      end
+      conversations
+    end
+
+    def load_from_file(filename)
+      conversations = []
+      File.open(filename, "r") do |file|
+        YAML.parse_stream(file) do |doc|
+          next unless doc
+          conversations << build_from(doc.to_ruby)
+        end
+      end
+      conversations
+    end
+
+    def build_from(doc)
+      if doc.is_a?(Hash)
+        doc = deep_symbolize_keys(doc)
+      elsif doc.is_a?(String)
+        doc = {messages: [{role: "user", content: doc}]}
+      else
+        raise ArgumentError, "Invalid document type: #{doc.class}"
+      end
+
+      system_prompt = doc[:system_prompt]
+      messages = doc[:messages] || []
+      convo = new(system_prompt: system_prompt, messages: messages)
+      if (llm = doc[:llm])
+        convo.llm = Promptcraft::Llm.from_h(llm)
+      end
+      convo
+    end
+
+    # Class method to create a Conversation from an array of messages
+    def from_messages(messages)
+      if messages.empty? || messages.first[:role] != "system"
+        raise ArgumentError, "First message must be from 'system' with the prompt"
+      end
+
+      system_prompt = messages.first[:content]
+      remaining_messages = messages[1..]  # all messages after the first
+      new(system_prompt:, messages: remaining_messages)
+    end
+  end
+
+  def save_to_file(filename)
+    File.write(filename, to_yaml)
+  end
+
+  # system_prompt: 'I like to solve maths problems.'
+  # messages:
+  # - role: "user"
+  #   content: "What is 2+2?"
+  # - role: assistant
+  #   content: 2 + 2 = 4
+  def to_yaml
+    YAML.dump(deep_stringify_keys({
+      system_prompt: @system_prompt&.strip,
+      llm: @llm&.to_h,
+      messages: @messages
+    }.compact))
+  end
+
+  def to_json
+    deep_stringify_keys({
+      system_prompt: @system_prompt&.strip,
+      llm: @llm&.to_h,
+      messages: @messages
+    }.compact).to_json
+  end
+
+  def to_messages
+    [{role: "system", content: @system_prompt}] + @messages
+  end
+end

data/lib/promptcraft/helpers.rb

@@ -0,0 +1,25 @@
+module Promptcraft::Helpers
+  def deep_symbolize_keys(value)
+    case value
+    when Hash
+      value.each_with_object({}) do |(key, v), result|
+        result[key.to_sym] = deep_symbolize_keys(v)  # Convert keys to symbols and recursively handle values
+      end
+    when Array
+      value.map { |v| deep_symbolize_keys(v) }  # Apply symbolization to each element in the array
+    else
+      value  # Return the value as is if it is neither a hash nor an array
+    end
+  end
+
+  def deep_stringify_keys(value)
+    case value
+    when Hash
+      value.map { |k, v| [k.to_s, deep_stringify_keys(v)] }.to_h
+    when Array
+      value.map { |v| deep_stringify_keys(v) }
+    else
+      value
+    end
+  end
+end

data/lib/promptcraft/llm.rb

@@ -0,0 +1,55 @@
+require "active_support/core_ext/module/delegation"
+
+class Promptcraft::Llm
+  DEFAULT_PROVIDER = "groq"
+
+  attr_reader :langchain
+  attr_accessor :provider, :model
+
+  delegate_missing_to :langchain
+
+  def initialize(provider: DEFAULT_PROVIDER, model: nil, api_key: nil)
+    @provider = provider
+    @langchain = case provider
+    when "groq"
+      @model = model || "llama3-70b-8192"
+      require "openai"
+      Langchain::LLM::OpenAI.new(
+        api_key: api_key || ENV.fetch("GROQ_API_KEY"),
+        llm_options: {uri_base: "https://api.groq.com/openai/"},
+        default_options: {chat_completion_model_name: @model}
+      )
+    when "openai"
+      @model = model || "gpt-3.5-turbo"
+      require "openai"
+      Langchain::LLM::OpenAI.new(
+        api_key: api_key || ENV.fetch("OPENAI_API_KEY"),
+        default_options: {chat_completion_model_name: @model}
+      )
+    when "openrouter"
+      @model = model || "meta-llama/llama-3-8b-instruct:free"
+      require "openai"
+      Langchain::LLM::OpenAI.new(
+        api_key: api_key || ENV.fetch("OPENROUTER_API_KEY"),
+        llm_options: {uri_base: "https://openrouter.ai/api/"},
+        default_options: {chat_completion_model_name: @model}
+      )
+    when "ollama"
+      @model = model || "llama3"
+      Langchain::LLM::Ollama.new(
+        default_options: {
+          completion_model_name: @model,
+          chat_completion_model_name: @model
+        }
+      )
+    end
+  end
+
+  def to_h
+    {provider: provider, model: model}
+  end
+
+  def self.from_h(hash)
+    new(provider: hash[:provider], model: hash[:model])
+  end
+end

data/lib/promptcraft/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Promptcraft
+  VERSION = "0.1.0"
+end

data/lib/promptcraft.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "promptcraft/version"
+
+module Promptcraft
+  autoload :Cli, "promptcraft/cli"
+  autoload :Command, "promptcraft/command"
+  autoload :Conversation, "promptcraft/conversation"
+  autoload :Helpers, "promptcraft/helpers"
+  autoload :Llm, "promptcraft/llm"
+end

data/sig/promptcraft.rbs

@@ -0,0 +1,4 @@
+module Promptcraft
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end