relevant_chunks 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 46f8f4e6500675c68339102a9be2646c6366c8e190a72cb3e4f3eb3b1dbf6f20
+  data.tar.gz: 9ad4948a253d78402bc42ee00c46320c7b714abefcd49ba3ba08f4031acc3bd1
+SHA512:
+  metadata.gz: 603f04bc204e25034d0feb8d523934fa23567e42fbcbf3f78b7e1ef59a7033bb6766ed1eba4ec41b54f46c20b0982d5193ce03bca7f866f9a25dbe92546f06ad
+  data.tar.gz: 79e57c25c813941aea5184ffedec7eb54b6296b700d922bb2b42a3edd35feadf2e406ace9279dcb9aef4bbb21b27968f6ee13c619ce6abc3371b10f64a4033c7

data/CHANGELOG.md

@@ -0,0 +1,73 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [0.3.0] - 2025-02-27
+
+### Changed
+
+-   Renamed gem from `relevantchunks` to `relevant_chunks` to follow Ruby conventions
+-   Updated all file paths and references to use snake_case
+-   Enhanced consistency across codebase
+
+## [0.2.0] - 2025-02-02
+
+### Changed
+
+-   Renamed project from TokenTrim to RelevantChunks
+-   Updated all module references and documentation
+-   Renamed executable from `tokentrim` to `relevant_chunks`
+
+## [0.1.2] - 2025-02-02
+
+### Fixed
+
+-   Updated gem description to remove references to commercial features
+
+## [0.1.1] - 2025-02-02
+
+### Changed
+
+-   Simplified gem by removing parallelization and paid features
+-   Improved code readability with heredoc strings
+-   Updated documentation to reflect MIT-only license
+
+## [0.1.0] - 2025-02-02
+
+### Added
+
+-   Initial release
+-   Smart text chunking with natural boundary detection
+-   Configurable chunk size and overlap
+-   Relevance scoring using Claude/Anthropic API
+-   Advanced configuration options:
+    -   Model selection (claude-3-5-sonnet-latest)
+    -   Temperature control (0.0-1.0)
+    -   Custom system prompts
+    -   Configurable scoring range (default 0-100)
+-   Comprehensive documentation and examples
+-   CLI tool for command-line usage
+
+### Changed
+
+-   N/A (initial release)
+
+### Deprecated
+
+-   N/A (initial release)
+
+### Removed
+
+-   N/A (initial release)
+
+### Fixed
+
+-   N/A (initial release)
+
+### Security
+
+-   Secure API key handling
+-   No key storage in code

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Robert Lucy
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+# RelevantChunks
+
+RelevantChunks is a Ruby gem that provides intelligent text chunking and relevance scoring using Claude/Anthropic's AI models. It features smart boundary detection and configurable overlap.
+
+## Features
+
+-   Smart text chunking with natural boundary detection
+-   Configurable chunk size and overlap
+-   Relevance scoring using Claude/Anthropic
+-   BYOK (Bring Your Own Key) model
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'relevant_chunks'
+```
+
+And then execute:
+
+```bash
+$ bundle install
+```
+
+Or install it yourself as:
+
+```bash
+$ gem install relevant_chunks
+```
+
+## Usage
+
+First, configure RelevantChunks with your Anthropic API key:
+
+```ruby
+RelevantChunks.configure do |config|
+  config.api_key = "your_anthropic_api_key"
+  config.max_tokens = 1000  # optional, default: 1000
+  config.overlap_size = 100 # optional, default: 100
+end
+```
+
+Then use it to process text:
+
+```ruby
+text = "The solar system consists of the Sun and everything that orbits around it. " \
+       "This includes eight planets, numerous moons, asteroids, comets, and other celestial objects. " \
+       "Earth is the third planet from the Sun and the only known planet to harbor life. " \
+       "Mars, often called the Red Planet, has been the subject of numerous exploration missions."
+
+# Query about Mars
+results = RelevantChunks.process(text, "Tell me about Mars")
+results.each do |result|
+  puts "Chunk: #{result[:chunk]}"
+  puts "Score: #{result[:score]}/100"
+  puts "---"
+end
+```
+
+Example output:
+
+```
+Chunk: "Mars, often called the Red Planet, has been the subject of numerous exploration missions."
+Score: 95/100
+---
+Chunk: "Earth is the third planet from the Sun and the only known planet to harbor life."
+Score: 35/100
+---
+Chunk: "The solar system consists of the Sun and everything that orbits around it."
+Score: 10/100
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/robert10997/relevant_chunks.
+
+## License
+
+The gem is available as open source under the terms of the MIT License.

data/exe/relevant_chunks

@@ -0,0 +1,78 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "relevant_chunks"
+require "optparse"
+require "json"
+
+options = {
+  max_tokens: 1000,
+  overlap_size: 100,
+  format: "text"
+}
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: relevant_chunks [options] <text_file> <query>"
+
+  opts.on("-k", "--api-key KEY", "Anthropic API key") do |key|
+    options[:api_key] = key
+  end
+
+  opts.on("-t", "--max-tokens N", Integer, "Maximum tokens per chunk") do |n|
+    options[:max_tokens] = n
+  end
+
+  opts.on("-o", "--overlap N", Integer, "Overlap size between chunks") do |n|
+    options[:overlap_size] = n
+  end
+
+  opts.on("-f", "--format FORMAT", %w[text json], "Output format (text/json)") do |f|
+    options[:format] = f
+  end
+
+  opts.on("-h", "--help", "Show this help message") do
+    puts opts
+    exit
+  end
+end
+
+parser.parse!
+
+if ARGV.length != 2
+  puts parser
+  exit 1
+end
+
+text_file, query = ARGV
+
+begin
+  text = File.read(text_file)
+rescue StandardError => e
+  puts "Error reading file: #{e.message}"
+  exit 1
+end
+
+begin
+  RelevantChunks.configure do |config|
+    config.api_key = options[:api_key] if options[:api_key]
+    config.max_tokens = options[:max_tokens]
+    config.overlap_size = options[:overlap_size]
+  end
+
+  results = RelevantChunks.process(text, query)
+
+  case options[:format]
+  when "json"
+    puts JSON.pretty_generate(results)
+  else
+    results.each.with_index(1) do |result, i|
+      puts "Chunk #{i} (Score: #{result[:score]}):"
+      puts "-" * 40
+      puts result[:chunk]
+      puts "\n"
+    end
+  end
+rescue RelevantChunks::Error => e
+  puts "Error: #{e.message}"
+  exit 1
+end

data/lib/relevant_chunks/chunker.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module RelevantChunks
+  # Handles text chunking with smart boundary detection and configurable overlap
+  #
+  # The Chunker class splits text into chunks while trying to maintain natural
+  # boundaries like sentence endings and paragraphs. It also supports overlapping
+  # chunks to ensure context is maintained across chunk boundaries.
+  #
+  # @example
+  #   chunker = RelevantChunks::Chunker.new(max_tokens: 1000, overlap_size: 100)
+  #   chunks = chunker.chunk_text("Your long text here...")
+  class Chunker
+    class << self
+      attr_accessor :configuration
+    end
+
+    # @return [Integer] Size of overlap between chunks
+    attr_reader :overlap_size
+
+    # @return [Integer] Maximum number of tokens per chunk
+    attr_reader :max_tokens
+
+    # Initialize a new Chunker instance
+    #
+    # @param max_tokens [Integer] Maximum number of tokens per chunk
+    # @param overlap_size [Integer] Number of tokens to overlap between chunks
+    # @return [Chunker]
+    def initialize(max_tokens: 1000, overlap_size: 100)
+      @max_tokens = max_tokens
+      @overlap_size = overlap_size
+      @logger = Logger.new($stdout)
+    end
+
+    # Split text into chunks with smart boundary detection
+    #
+    # @param text [String] The text to split into chunks
+    # @return [Array<String>] Array of text chunks
+    # @example
+    #   chunker = RelevantChunks::Chunker.new
+    #   chunks = chunker.chunk_text("First sentence. Second sentence.")
+    def chunk_text(text)
+      @logger.info "Starting chunk_text with text length: #{text.length}"
+      return [text] if text.length <= max_tokens
+
+      chunks = []
+      current_position = 0
+
+      while current_position < text.length
+        chunk_end = find_chunk_boundary(text, current_position)
+        add_chunk(text, current_position, chunk_end, chunks)
+
+        # If we've reached the end, break
+        break if chunk_end >= text.length - 1
+
+        # Calculate next position with overlap
+        next_position = calculate_next_position(current_position, chunk_end)
+        break if should_stop_chunking?(next_position, current_position, text)
+
+        current_position = next_position
+        @logger.info "Moving to position: #{current_position}"
+      end
+
+      @logger.info "Final chunks: #{chunks.inspect}"
+      chunks
+    end
+
+    private
+
+    def add_chunk(text, start_pos, end_pos, chunks)
+      @logger.info "Found chunk boundary at position #{end_pos}"
+      chunk = text[start_pos..end_pos]
+      @logger.info "Created chunk: #{chunk.inspect}"
+      chunks << chunk
+    end
+
+    def calculate_next_position(current_pos, chunk_end)
+      next_pos = [chunk_end - overlap_size + 1, current_pos + 1].max
+      @logger.info "Next position would be: #{next_pos}"
+      next_pos
+    end
+
+    def should_stop_chunking?(next_pos, current_pos, text)
+      remaining_length = text.length - next_pos
+      if next_pos <= current_pos || remaining_length <= overlap_size
+        add_final_chunk(text, next_pos) if remaining_length.positive?
+        true
+      else
+        false
+      end
+    end
+
+    def add_final_chunk(text, start_pos)
+      final_chunk = text[start_pos..]
+      @logger.info "Adding final chunk: #{final_chunk.inspect}"
+      final_chunk
+    end
+
+    def find_chunk_boundary(text, start_position)
+      target_end = start_position + max_tokens
+      @logger.info "Target end position: #{target_end}"
+
+      return handle_text_end(text) if target_end >= text.length
+
+      # Look for natural boundaries
+      boundary = find_natural_boundary(text, start_position, target_end)
+      return boundary if boundary
+
+      # If no natural boundary found, break at the last space
+      boundary = find_space_boundary(text, start_position, target_end)
+      return boundary if boundary
+
+      target_end
+    end
+
+    def handle_text_end(text)
+      @logger.info "Target end exceeds text length, returning #{text.length - 1}"
+      text.length - 1
+    end
+
+    def find_natural_boundary(text, start_pos, target_end)
+      search_start = [target_end - 30, start_pos].max
+      @logger.info "Looking for natural boundaries between #{search_start} and #{target_end}"
+
+      target_end.downto(search_start) do |i|
+        break if i >= text.length || i + 1 >= text.length
+
+        if natural_boundary?(text, i)
+          @logger.info "Found natural boundary at #{i}"
+          return i
+        end
+      end
+      nil
+    end
+
+    def natural_boundary?(text, pos)
+      return false if pos + 1 >= text.length
+
+      char = text[pos]
+      next_char = text[pos + 1]
+      @logger.info "Checking position #{pos}: char='#{char}', next_char='#{next_char}'"
+
+      (char == "." && next_char == " ") ||
+        (char == "?" && next_char == " ") ||
+        (char == "!" && next_char == " ") ||
+        (char == "\n" && next_char == "\n")
+    end
+
+    def find_space_boundary(text, start_pos, target_end)
+      search_text = text[start_pos..target_end]
+      @logger.info "Looking for last space in: #{search_text.inspect}"
+      last_space = search_text.rindex(" ")
+      return unless last_space
+
+      @logger.info "Found last space at offset #{last_space}"
+      start_pos + last_space
+    end
+  end
+end

data/lib/relevant_chunks/processor.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+require "faraday"
+require "json"
+
+module RelevantChunks
+  # Handles text processing and relevance scoring using Claude/Anthropic
+  #
+  # The Processor class manages the chunking and scoring of text using the Anthropic API.
+  #
+  # @example Basic usage
+  #   processor = RelevantChunks::Processor.new(api_key: "your_key")
+  #   results = processor.process("Long text here", "What is this about?")
+  #
+  # @example Advanced configuration
+  #   processor = RelevantChunks::Processor.new(
+  #     api_key: "your_key",
+  #     model: "claude-3-5-sonnet-latest",  # Use a different model variant
+  #     temperature: 0.1,                 # Add slight variation to scores
+  #     system_prompt: "Custom scoring system prompt...",
+  #     max_score: 10                     # Use 0-10 scoring range
+  #   )
+  #
+  # @example Scoring text relevance with different queries
+  #   processor = RelevantChunks::Processor.new(api_key: "your_key")
+  #   text = "The solar system consists of the Sun and everything that orbits around it. " \
+  #          "This includes eight planets, numerous moons, asteroids, comets, and other celestial objects. " \
+  #          "Earth is the third planet from the Sun and the only known planet to harbor life. " \
+  #          "Mars, often called the Red Planet, has been the subject of numerous exploration missions."
+  #
+  #   # Query about Mars
+  #   results = processor.process(text, "Tell me about Mars")
+  #   # Returns chunks with scores like:
+  #   # - "Mars, often called the Red Planet..." (Score: 60)
+  #   # - "...numerous exploration missions." (Score: 35)
+  #   # - General solar system info (Score: 15)
+  #
+  #   # Query about life on planets
+  #   results = processor.process(text, "What planets are known to have life?")
+  #   # Returns chunks with scores like:
+  #   # - "Earth is the third planet...only known planet to harbor life" (Score: 65)
+  #   # - Chunks mentioning planets (Score: 35)
+  #   # - Other chunks (Score: 5-15)
+  class Processor
+    class << self
+      attr_accessor :configuration
+    end
+
+    # @return [String] Anthropic API key
+    attr_reader :api_key
+
+    # @return [Chunker] Text chunker instance
+    attr_reader :chunker
+
+    # @return [String] Claude model to use
+    attr_reader :model
+
+    # @return [Float] Temperature for scoring (0.0-1.0)
+    attr_reader :temperature
+
+    # @return [String] System prompt for scoring
+    attr_reader :system_prompt
+
+    # @return [Integer] Maximum score in the scoring range
+    attr_reader :max_score
+
+    # Initialize a new Processor instance
+    #
+    # @param api_key [String] Anthropic API key
+    # @param max_tokens [Integer] Maximum tokens per chunk
+    # @param overlap_size [Integer] Overlap size between chunks
+    # @param model [String] Claude model to use (default: "claude-3-5-sonnet-latest")
+    # @param temperature [Float] Temperature for scoring (0.0-1.0, default: 0.0)
+    # @param system_prompt [String, nil] Custom system prompt for scoring (default: nil)
+    # @param max_score [Integer] Maximum score in range (default: 100)
+    # @return [Processor]
+    def initialize(api_key:, max_tokens: 1000, overlap_size: 100,
+                   model: "claude-3-5-sonnet-latest", temperature: 0.0,
+                   system_prompt: nil, max_score: 100)
+      @api_key = api_key
+      @chunker = Chunker.new(max_tokens: max_tokens, overlap_size: overlap_size)
+      @model = model
+      @temperature = temperature
+      @max_score = max_score
+      @system_prompt = system_prompt || default_system_prompt
+      @conn = Faraday.new(url: "https://api.anthropic.com") do |f|
+        f.request :json
+        f.response :json
+        f.adapter :net_http
+        f.headers = {
+          "accept" => "application/json",
+          "anthropic-version" => "2023-06-01",
+          "content-type" => "application/json",
+          "x-api-key" => api_key
+        }
+      end
+    end
+
+    # Process text and score chunks against a query
+    #
+    # @param text [String] The text to process
+    # @param query [String] The query to score chunks against
+    # @return [Array<Hash>] Array of chunks with their scores and API responses. Each hash contains:
+    #   - :chunk [String] The text chunk that was scored
+    #   - :score [Integer] The relevance score (0-100)
+    #   - :response [Hash] The complete raw response from the Anthropic API
+    # @example
+    #   processor = RelevantChunks::Processor.new(api_key: "your_key")
+    #   results = processor.process("Long text here", "What is this about?")
+    #   results.each do |result|
+    #     puts "Chunk: #{result[:chunk]}"
+    #     puts "Score: #{result[:score]}"
+    #     puts "Raw response: #{result[:response].inspect}"
+    #   end
+    def process(text, query)
+      chunks = chunker.chunk_text(text)
+      chunks.map { |chunk| score_chunk(chunk, query) }
+    end
+
+    private
+
+    # Score a single chunk against the query
+    #
+    # @param chunk [String] Text chunk to score
+    # @param query [String] Query to score against
+    # @return [Hash] Chunk with score and response
+    def score_chunk(chunk, query)
+      response = @conn.post("/v1/messages") do |req|
+        req.body = {
+          model: model,
+          temperature: temperature,
+          system: system_prompt,
+          messages: [
+            {
+              role: "user",
+              content: <<~STRING
+                Text chunk to evaluate: #{chunk}
+
+                Query: #{query}
+
+                Please output only a number from 0-#{max_score} indicating how relevant this text chunk is to the query.
+              STRING
+            }
+          ]
+        }
+      end
+
+      # Parse the response body if it's a string
+      body = response.body.is_a?(String) ? JSON.parse(response.body) : response.body
+      score = body.dig("content", 0, "text").to_i
+      { chunk: chunk, score: score, response: body }
+    end
+
+    # Default system prompt for scoring
+    #
+    # @return [String]
+    def default_system_prompt
+      <<~STRING
+        You are a text relevance scoring system. Your task is to evaluate how relevant a given text chunk is to a query.
+        Score on a scale of 0-#{max_score}, where 0 means completely irrelevant and #{max_score} means highly relevant.
+        Consider semantic meaning, not just keyword matching. Output only the numeric score.
+      STRING
+    end
+  end
+end

data/lib/relevant_chunks/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module RelevantChunks
+  # Current version of RelevantChunks
+  VERSION = "0.3.0"
+end

data/lib/relevant_chunks.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require_relative "relevant_chunks/version"
+require_relative "relevant_chunks/chunker"
+require_relative "relevant_chunks/processor"
+require "net/http"
+require "json"
+
+# RelevantChunks provides intelligent text chunking and relevance scoring using Claude/Anthropic.
+#
+# @example Basic usage
+#   RelevantChunks.configure do |config|
+#     config.api_key = "your_anthropic_api_key"
+#   end
+#
+#   results = RelevantChunks.process("Your text here", "Your query")
+module RelevantChunks
+  class Error < StandardError; end
+
+  class << self
+    attr_accessor :configuration
+  end
+
+  # Configures RelevantChunks with the given settings
+  #
+  # @yield [config] Configuration object
+  # @example
+  #   RelevantChunks.configure do |config|
+  #     config.api_key = "your_anthropic_api_key"
+  #     config.max_tokens = 1000
+  #   end
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration) if block_given?
+  end
+
+  # Configuration class for RelevantChunks
+  class Configuration
+    attr_accessor :api_key, :max_tokens, :overlap_size
+
+    # Initialize a new Configuration instance
+    #
+    # @return [Configuration]
+    def initialize
+      @max_tokens = 1000
+      @overlap_size = 100
+    end
+  end
+
+  # Process text with the given query
+  #
+  # @param text [String] The text to process
+  # @param query [String] The query to score chunks against
+  # @return [Array<Hash>] Array of chunks with relevance scores
+  # @raise [Error] If API key is not configured
+  # @example
+  #   results = RelevantChunks.process("Long text here", "What is this about?")
+  #   results.each do |result|
+  #     puts "Chunk: #{result[:chunk]}"
+  #     puts "Score: #{result[:score]}"
+  #   end
+  def self.process(text, query)
+    raise Error, "API key not configured" unless configuration&.api_key
+
+    processor = Processor.new(
+      api_key: configuration.api_key,
+      max_tokens: configuration.max_tokens,
+      overlap_size: configuration.overlap_size
+    )
+
+    processor.process(text, query)
+  end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: relevant_chunks
+version: !ruby/object:Gem::Version
+  version: 0.3.0
+platform: ruby
+authors:
+- Robert Lucy
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2025-02-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.9'
+description: RelevantChunks provides intelligent text chunking with smart boundaries
+  and overlap, plus relevance scoring using Claude/Anthropic's AI models.
+email:
+- robertlucy@gmail.com
+executables:
+- relevant_chunks
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- exe/relevant_chunks
+- lib/relevant_chunks.rb
+- lib/relevant_chunks/chunker.rb
+- lib/relevant_chunks/processor.rb
+- lib/relevant_chunks/version.rb
+homepage: https://github.com/robert10997/relevant_chunks
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/robert10997/relevant_chunks
+  source_code_uri: https://github.com/robert10997/relevant_chunks
+  changelog_uri: https://github.com/robert10997/relevant_chunks/blob/main/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Smart text chunking and relevance scoring using Claude/Anthropic
+test_files: []