roseflow 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5332eb49fa51fee8d237a7201b36694c5a09851e2bfab8cbc2f2f53a941f6ff1
-  data.tar.gz: 44b0b895bf1000c3cecf77c5e09f00001a03bd9600232bc52bf75508f858d0ff
+  metadata.gz: b8b7f5c7688debc9bb3b2e804dfcc45671ca670cc3114fabeb8ceff1169b5098
+  data.tar.gz: cb9d12161f611bf90738c2459e0700698e3443a8fe7b0d3f98c8d6268b315c0f
 SHA512:
-  metadata.gz: f5ee95217a4ea9e7af38942d003299354cf47ddd77af2aaca7d0fc3bc8a9047d76dd95a8c7b390305858285ad40cc9c0ac3d985ce6ff1533a759a09a9a2ed98e
-  data.tar.gz: '08c4e29bb46765709f0698a9b1212cfb9e0d1a636bf2bcf9feac03d7da4d683c99e83bea3c23eb988cd6a02fb2e7e153bc88a17d2774c344b75520b1fd85dbea'
+  metadata.gz: dcb00909e24bf283c2f901f680e86995bbe0fe5060eb9fb243e1e6435c1b1c39735ae918609eb89f81a40c48196861e4a2de8881f5c1c71b86e4fb996adff2af
+  data.tar.gz: 89415e4d06ef6b421da5be3a9a214752dfaa959258082ad09c9321d9f9d47703ea8d17ac7eaf26da9bb18ca4b291e1ca982c5f0a98b4c86b324f697b25446f20

data/.standard.yml

@@ -1,3 +1,5 @@
 # For available configuration options, see:
 #   https://github.com/testdouble/standard
-ruby_version: 2.6
+ruby_version: 3.2
+ignore:
+  - "Gemfile"

data/CHANGELOG.md

@@ -1,5 +1,5 @@
-## [Unreleased]
+## [0.1.0] - 2023-05-10
 
-## [0.1.0] - 2023-04-22
+"La petite fille de la mer"
 
 - Initial release

data/Gemfile

@@ -8,3 +8,5 @@ gemspec
 gem "rake", "~> 13.0"
 gem "rspec", "~> 3.0"
 gem "standard", "~> 1.3"
+
+eval_gemfile "Gemfile.local" if File.exist?("Gemfile.local")

data/examples/github-repo-chat/lib/actions/clone_and_load_repository.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# This class represents an action that clones and loads a repository from a
+# given URL. The action expects a repository URL as input and returns a
+# Repository object containing the repository's data.
+#
+# Example usage:
+#   repository = Actions::CloneAndLoadRepository.call(
+#     repository_url: "https://github.com/ljuti/roseflow.git"
+#   )
+
+require "roseflow/action"
+require "rugged"
+require "tmpdir"
+
+require_relative "../repository"
+
+class Actions::CloneAndLoadRepository
+  extend Roseflow::Action
+
+  # Define the inputs and outputs of the action
+  expects :repository_url # Input: The URL of the repository to clone
+  promises :repository # Output: a Repository object
+
+  executed do |ctx|
+    ctx[:repository] = clone_repository(ctx.repository_url)
+  end
+
+  # Clone a repository from a given URL, load its data, and create a Repository object.
+  #
+  # @param repository_url [String] The URL of the repository to clone
+  # @return [Repository] A Repository object containing the repository's data
+  def self.clone_repository(repository_url)
+    files = {}
+    Dir.mktmpdir do |dir|
+      # Clone the repository into a temporary directory
+      repository = Rugged::Repository.clone_at(repository_url, dir)
+      head_commit = repository.head.target
+      tree = repository.lookup(head_commit.tree.oid)
+
+      # Collect the content of each file in the repository
+      tree.walk_blobs(:preorder) do |root, entry|
+        file_path = "#{root}#{entry[:name]}"
+        files[file_path] = repository.lookup(entry[:oid]).content
+      end
+
+      # Extract the repository's name from the URL and create a Repository object
+      name = repository_url.split("/").last.gsub(".git", "")
+      return Repository.new(name, repository_url, files)
+    end
+  end
+end

data/examples/github-repo-chat/lib/actions/create_prompt.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "roseflow/action"
+require_relative "../github_chat_prompt"
+
+class Actions::CreatePrompt
+  extend Roseflow::Action
+
+  expects :input
+  promises :prompt
+
+  executed do |ctx|
+    ctx[:prompt] = GithubChatPrompt.new(input)
+  end
+end

data/examples/github-repo-chat/lib/actions/embed_repository.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "roseflow/action"
+require "roseflow/vector_stores/base"
+require_relative "../repository_file"
+
+class Actions::EmbedRepository
+  extend Roseflow::Action
+
+  expects :repository
+  expects :vector_store
+
+  executed do |ctx|
+    # Check if the vector store already has embeddings for given repository
+    # If so, we can skip this action
+    next if ctx.vector_store.has_embeddings?(ctx.repository)
+
+    # Otherwise, we need to embed the repository
+    embed_repository(ctx.repository, ctx.vector_store)
+  end
+
+  def self.embed_repository(repository, vector_store)
+    repository.files.each do |file_path, content|
+      unless file_is_embedded?(vector_store, file_path, repository.name)
+        embedding = RepositoryFile.new(file_path, content)
+        vector = vector_store.build_vector(file_path.to_s, embedding.embedding.vector)
+        vector_store.create_vector("github", vector, namespace: repository.name)
+      end
+    end
+  end
+
+  def self.file_is_embedded?(vector_store, file_path, repository_name)
+    vector_store.find("github", file_path.to_s, namespace: repository_name)
+  end
+end

data/examples/github-repo-chat/lib/actions/initialize_vector_store.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+# This class represents an action that initializes a vector store based on a
+# given provider. The action expects a provider as input and returns a
+# VectorStore object.
+#
+# Example usage:
+#   vector_store = Actions::InitializeVectorStore.execute(provider: :pinecone)
+
+require "roseflow/action"
+
+class Actions::InitializeVectorStore
+  # Make it a Roseflow action
+  extend Roseflow::Action
+
+  # Define the inputs and outputs of the action
+  expects :provider # Input: The vector store provider to use
+  promises :vector_store # Output: a VectorStore object
+
+  executed do |ctx|
+    ctx[:vector_store] = initialize_vector_store(ctx.provider)
+  end
+
+  # Initialize a vector store based on the given provider.
+  #
+  # @param provider [Symbol] The symbol representing the vector store provider
+  # @return [Object] A VectorStore object corresponding to the given provider
+  # @raise [ArgumentError] If the provider is unknown
+  def self.initialize_vector_store(provider)
+    case provider
+    when :pinecone
+      # Initialize a Pinecone vector store
+      require "roseflow/pinecone/vector_store"
+      Roseflow::Pinecone::VectorStore.new
+    else
+      # Raise an error if the provider is unknown
+      raise ArgumentError, "Unknown provider: #{provider}"
+    end
+  end
+end

data/examples/github-repo-chat/lib/actions/interact_with_model.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# This class represents an action that interacts with a given model using a
+# provided prompt. The action expects both a prompt and a model as input, and
+# returns a response generated by the model.
+#
+# Example usage:
+#   response = Actions::InteractWithModel.execute(prompt: "What is the capital of France?", model: my_model)
+
+require "roseflow/action"
+
+class Actions::InteractWithModel
+  # Make it a Roseflow action
+  extend Roseflow::Action
+
+  # Define the inputs and outputs of the action
+  expects :prompt # Input: The prompt to use
+  expects :model # Input: The model to interact with
+
+  promises :response # Output: The response generated by the model
+
+  executed do |ctx|
+    # Interact with the model using the provided prompt
+    chat = ctx.model.chat(ctx.prompt)
+
+    # Store the response in the context
+    context[:response] = chat.response
+  end
+end

data/examples/github-repo-chat/lib/actions/split_files_to_documents.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# This action splits the content of source code files into smaller chunks
+# and tokenizing them into documents.
+#
+# It takes a repository as an input and returns an array of documents.
+
+require "roseflow/action"
+require "ulid"
+
+class Actions::SplitFilesToDocuments
+  extend Roseflow::Action
+
+  # Define the inputs and outputs of the action
+  expects :repository # Input: a repository object containing file data
+  promises :documents # Output: Array of RepositoryFile objects representing documents
+
+  executed do |ctx|
+    # Initialize splitter and tokenizer instances
+    splitter = initialize_splitter
+    tokenizer = initialize_tokenizer(ctx)
+
+    # Iterate over each file in the repository, split and tokenize its content
+    ctx[:documents] = ctx.repository.files.each do |file_path, content|
+      chunks = splitter.split(content) # Split the content into chunks
+      tokenized = chunks.map { |chunk| tokenizer.encode(chunk) } # Tokenize each chunks
+
+      # Create a RepositoryFile object for each chunk-token pair
+      chunks.zip(tokenized).map do |chunk, tkn|
+        RepositoryFile.new(
+          name: file_path,
+          content: chunk,
+          tokens: tkn,
+          token_count: tkn.length
+        )
+      end
+    end
+  end
+
+  # Initialize a RecursiveCharacterSplitter instance with a chunk_size of 1000
+  # and no overlap between chunks.
+  #
+  # @return [Roseflow::Text::RecursiveCharacterSplitter]
+  def self.initialize_splitter
+    Roseflow::Text::RecursiveCharacterSplitter.new(chunk_size: 1000, chunk_overlap: 0)
+  end
+
+  # Initialize a Tokenizer instance with the provided model from the context.
+  #
+  # @param ctx [Roseflow::Context] The context of the action execution
+  # @return [Roseflow::Tiktoken::Tokenizer]
+  def self.initialize_tokenizer(ctx)
+    Roseflow::Tiktoken::Tokenizer.new(model: ctx.model.name)
+  end
+end

data/examples/github-repo-chat/lib/github_chat_prompt.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "roseflow/prompt"
+
+class GithubChatPrompt < Roseflow::Prompt
+  def initialize(input)
+    @input = input
+  end
+
+  attr_reader :input
+
+  def template
+    plain_raw condensed(prompt_string)
+  end
+
+  private
+
+  def prompt_string
+    <<-PROMPT
+
+    Command: [#{input}]
+    PROMPT
+  end
+end

data/examples/github-repo-chat/lib/github_repository_chat.rb

@@ -0,0 +1,12 @@
+require_relative "actions/clone_and_load_repository"
+require_relative "actions/create_prompt"
+require_relative "actions/embed_repository"
+require_relative "actions/initialize_vector_store"
+require_relative "actions/interact_with_model"
+
+require_relative "interactions/ask_llm"
+require_relative "interactions/github_repository_chat"
+require_relative "interactions/load_repository"
+require_relative "interactions/prepare_vector_store"
+
+require_relative "github_chat_prompt"

data/examples/github-repo-chat/lib/interactions/ask_llm.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+# This class represents an interaction that asks the LLM (Large Language Model)
+# a question based on a given context. It orchestrates the required actions to
+# initialize the LLM, create a prompt, and interact with the model.
+#
+# Example usage:
+#   result = Interactions::AskLLM.call(context)
+
+require "roseflow"
+require "roseflow/interactions/ai/initialize_llm"
+
+require_relative "../actions/create_prompt"
+
+class Interactions::AskLLM
+  extend Roseflow::Interaction
+
+  def self.call(context = {})
+    with(context).reduce(actions)
+  end
+
+  # Define the sequence of actions (or interactions) to be performed
+  # by the interaction.
+  def self.actions
+    [
+      Roseflow::Interactions::AI::InitializeLlm, # Initialize the LLM
+      CreatePrompt, # Create a prompt for the LLM
+      InteractWithModel # Interact with the LLM
+    ]
+  end
+end

data/examples/github-repo-chat/lib/interactions/github_repository_chat.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This class represents an interaction that allows users to chat about
+# a Github repository. It orchestrates the required actions to load
+# the repository, prepare the vector store, and ask the LLM about the repository.
+#
+# Example usage:
+#   question = "What is the repository about?"
+#   response = Interactions::GithubRepositoryChat.call(
+#     question: question,
+#     repository_url: "https://github.com/ljuti/roseflow"
+#   )
+
+require "roseflow/interaction"
+
+require_relative "./ask_llm"
+require_relative "./load_repository"
+require_relative "./prepare_vector_store"
+
+class Interactions::GithubRepositoryChat
+  extend Roseflow::Interaction
+
+  def self.call(context = {})
+    with(context).reduce(actions)
+  end
+
+  # Define the sequence of actions (or interactions) to be performed
+  # by the interaction.
+  def self.actions
+    [
+      LoadRepository, # Load repository from Github and prepare it
+      PrepareVectorStore, # Prepare vector store for querying
+      AskLLM # Ask the LLM about the repository
+    ]
+  end
+end

data/examples/github-repo-chat/lib/interactions/load_files_to_document_database.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "roseflow/interaction"
+
+class Interactions::LoadFilesToDocumentDatabase
+  extend Roseflow::Interaction
+
+  def self.call(ctx)
+    with(ctx).reduce(actions)
+  end
+
+  def self.actions
+    [
+      SplitFilesToDocuments,
+      LoadDocumentsToDatabase
+    ]
+  end
+end

data/examples/github-repo-chat/lib/interactions/load_repository.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "roseflow/interaction"
+
+require_relative "../actions/clone_and_load_repository"
+
+class Interactions::LoadRepository
+  extend Roseflow::Interaction
+
+  def self.call(context = {})
+    with(context).reduce(actions)
+  end
+
+  def self.actions
+    [
+      CloneAndLoadRepository,
+      LoadFilesToDocumentDatabase
+    ]
+  end
+end

data/examples/github-repo-chat/lib/interactions/prepare_vector_store.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "roseflow/interaction"
+
+require_relative "../actions/initialize_vector_store"
+require_relative "../actions/embed_repository"
+
+class Interactions::PrepareVectorStore
+  extend Roseflow::Interaction
+
+  def self.call(context = {})
+    with(context).reduce(actions)
+  end
+
+  def self.actions
+    [
+      InitializeVectorStore,
+      reduce_if(->(ctx) { ctx.vector_store.has_embeddings? }, EmbedRepository)
+    ]
+  end
+end

data/examples/github-repo-chat/lib/repository.rb

@@ -0,0 +1,9 @@
+class Repository
+  def initialize(name, url, files)
+    @name = name
+    @url = url
+    @files = files
+  end
+
+  attr_reader :name, :url, :files
+end

data/examples/github-repo-chat/lib/repository_file.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "ulid"
+
+class RepositoryFile < Roseflow::Embeddings::Base
+  def initialize(name:, content:, **kwargs)
+    @name = name
+    @content = content
+    @id = ULID.generate
+    @token_count = kwargs[:token_count]
+    @tokens = kwargs[:tokens]
+  end
+
+  def embedding
+    @embedding ||= create_embedding
+  end
+
+  private
+
+  def create_embedding
+    provider.embedding(model: embedding_model, input: @content)
+  end
+
+  def provider
+    @provider ||= Roseflow::OpenAI::Provider.new
+  end
+
+  def embedding_model
+    provider.models.embeddable.find { |model| model.name == "text-embedding-ada-002" } || openai.models.embeddable.first
+  end
+end

data/examples/github-repo-chat/spec/actions/clone_and_load_repository_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "../spec_helper"
+
+RSpec.describe Actions::CloneAndLoadRepository do
+  let(:repository_url) { "https://github.com/abachman/ulid-ruby.git" }
+  let(:ctx) do
+    Roseflow::InteractionContext.make(
+      repository_url: repository_url
+    )
+  end
+
+  describe "cloning and loading the repository" do
+    let(:action) { described_class.execute(ctx) }
+
+    it "returns a repository" do
+      expect(action).to be_a_success
+      expect(action.repository).to be_a(Repository)
+    end
+
+    it "sets repository attributes correctly" do
+      expect(action.repository).to be_a(Repository)
+      expect(action.repository.name).to eq "ulid-ruby"
+      expect(action.repository.url).to eq repository_url
+      expect(action.repository.files).not_to be_empty
+    end
+  end
+end

data/examples/github-repo-chat/spec/actions/embed_repository_spec.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "../spec_helper"
+require "roseflow/pinecone"
+require "roseflow/pinecone/vector_store"
+
+RSpec.describe Actions::EmbedRepository do
+  let(:repository) { Repository.new("ulid-ruby", "https://github.com/abachman/ulid-ruby.git", {"ulid-ruby.gemspec": "This is a gemspec for the gem"}) }
+  let(:ctx) do
+    Roseflow::InteractionContext.make(
+      repository: repository,
+      vector_store: Roseflow::Pinecone::VectorStore.new
+    )
+  end
+
+  describe "embedding a repository" do
+    let(:action) { described_class.execute(ctx) }
+
+    it "embeds the files of the repository to the vector store" do
+      expect(action).to be_a_success
+      expect(action.vector_store).to be_a(Roseflow::VectorStores::Base)
+    end
+  end
+end

data/examples/github-repo-chat/spec/actions/initialize_vector_store_spec.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative "../spec_helper"
+
+RSpec.describe Actions::InitializeVectorStore do
+  let(:ctx) do
+    Roseflow::InteractionContext.make(
+      provider: :pinecone
+    )
+  end
+
+  describe "initializing the vector store" do
+    let(:action) { described_class.execute(ctx) }
+
+    it "returns a vector store" do
+      expect(action).to be_a_success
+      expect(action.vector_store).to be_a Roseflow::Pinecone::VectorStore
+    end
+  end
+end

data/examples/github-repo-chat/spec/actions/load_files_to_document_database_spec.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "../spec_helper"
+
+RSpec.describe Actions::LoadFilesToDocumentDatabase do
+  let(:repository_url) { "https://github.com/abachman/ulid-ruby.git" }
+  let(:ctx) do
+    Roseflow::InteractionContext.make(
+      repository_url: repository_url
+    )
+  end
+
+  before do
+    @context = Actions::CloneAndLoadRepository.execute(ctx)
+  end
+
+  describe "loading files to document database" do
+    it "loads the files" do
+      expect(action).to be_a_success
+      expect(action.database.files.count).to be > 5
+    end
+  end
+end

data/examples/github-repo-chat/spec/github_repository_chat_spec.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+RSpec.describe Interactions::GithubRepositoryChat do
+  let(:prompt) { GithubChatPrompt.new("Describe me how to use OpenAI models with Roseflow.") }
+  let(:ctx) do
+    Roseflow::InteractionContext.make(
+      provider: :openai,
+      model: "gpt-3.5-turbo",
+      repository_url: "https://github.com/roseflow-ai/roseflow-openai",
+      prompt: prompt
+    )
+  end
+
+  describe "" do
+  end
+end

data/examples/github-repo-chat/spec/interactions/prepare_vector_store_spec.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+RSpec.describe Interactions::PrepareVectorStore do
+end

data/examples/github-repo-chat/spec/spec_helper.rb

@@ -0,0 +1,12 @@
+require "spec_helper"
+require "roseflow"
+
+Anyway::Settings.use_local_files = true
+
+module Interactions
+end
+
+module Actions
+end
+
+require_relative "../lib/github_repository_chat"

data/lib/roseflow/action.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "light-service"
+
+module Roseflow
+  module Action
+    extend LightService::Action
+
+    def self.extended(base_class)
+      base_class.extend LightService::Action::Macros
+    end
+  end
+end

data/lib/roseflow/actions/ai/resolve_model.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "roseflow/action"
+require "roseflow/ai/model"
+
+module Roseflow
+  module Actions
+    module AI
+      class ResolveModel
+        extend Roseflow::Action
+
+        expects :model
+        promises :llm
+
+        executed do |context|
+          model = context.provider.models.find(context[:model])
+
+          unless model
+            context.fail_and_return!("Model #{context[:model]} not found")
+          end
+
+          context[:llm] = Roseflow::AI::Model.new(name: model.name, provider: context.provider)
+        end
+      end
+    end
+  end
+end