semantic_chunker 0.5.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '0782ee5ae0f80488a985b3c12afad2eb95252ecd45849ed98a8446aef4dbfc66'
+  data.tar.gz: 988a17459d404db90f460527d105e00579a44a442deabeba3c3e6462a4e440de
+SHA512:
+  metadata.gz: 29b84d713798dabc248986ad2040da8bffe029c74dcba9e7f35aef783a8ac5b1b944c4ebdba1f1af50c79b78d92a9fb06c66c9f71a901a9c457032edd1508865
+  data.tar.gz: 26a6d3ac345c0a6d88cbffc7877a2fd387293c75d81d2cb85771b2d8f66778645944c46608f9de405eb787a4c48546f6efe859790963c129d2dfdd1ded8dcd32

data/lib/semantic_chunker/adapters/base.rb

@@ -0,0 +1,10 @@
+# lib/semantic_chunker/adapters/base.rb
+module SemanticChunker
+  module Adapters
+    class Base
+      def embed(sentences)
+        raise NotImplementedError, "#{self.class} must implement #embed"
+      end
+    end
+  end
+end

data/lib/semantic_chunker/adapters/hugging_face_adapter.rb

@@ -0,0 +1,56 @@
+# lib/semantic_chunker/adapters/hugging_face_adapter.rb
+module SemanticChunker
+  module Adapters
+    class HuggingFaceAdapter < Base
+      BASE_URL = "https://router.huggingface.co/hf-inference/models/%{model}"
+
+      def initialize(api_key:, model: 'intfloat/multilingual-e5-large')
+        @api_key = api_key
+        @model = model
+        # @model = 'sentence-transformers/all-MiniLM-L6-v2'
+        # @model = 'BAAI/bge-small-en-v1.5'
+      end
+
+      def embed(sentences)
+        response = post_request(sentences)
+  
+        unless response.content_type == "application/json"
+          raise "HuggingFace Error: Expected JSON, got #{response.content_type}. Body: #{response.body}"
+        end
+        
+        parsed = JSON.parse(response.body)
+        
+        if response.is_a?(Net::HTTPSuccess)
+          parsed
+        else
+          if parsed.is_a?(Hash) && parsed["error"]&.include?("loading")
+            puts "Model warming up... retrying in 10s"
+            sleep 10
+            return embed(sentences)
+          end
+          raise "HuggingFace Error: #{parsed['error'] || parsed}"
+        end
+      end
+
+      private
+
+      def post_request(sentences)
+        uri = URI(BASE_URL % { model: @model })
+        request = Net::HTTP::Post.new(uri)
+        
+        request["Authorization"] = "Bearer #{@api_key}"
+        request["Content-Type"] = "application/json"
+        request["X-Wait-For-Model"] = "true"
+        
+        request.body = { 
+          inputs: sentences
+        }.to_json
+        
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.read_timeout = 60
+          http.request(request)
+        end
+      end
+    end
+  end
+end 

data/lib/semantic_chunker/adapters/openai_adapter.rb

@@ -0,0 +1,44 @@
+# lib/semantic_chunker/adapters/openai_adapter.rb
+require "net/http"
+require "json"
+require "uri"
+
+module SemanticChunker
+  module Adapters
+    class OpenAIAdapter < Base
+      ENDPOINT = "https://api.openai.com/v1/embeddings"
+
+      def initialize(api_key:, model: "text-embedding-3-small")
+        @api_key = api_key
+        @model = model
+      end
+
+      def embed(sentences)
+        response = post_request(sentences)
+        parsed = JSON.parse(response.body)
+
+        if response.is_a?(Net::HTTPSuccess)
+          # OpenAI returns data in the same order as input
+          # We extract just the embedding arrays
+          parsed["data"].map { |entry| entry["embedding"] }
+        else
+          raise "OpenAI Error: #{parsed.dig('error', 'message') || response.code}"
+        end
+      end
+
+      private
+
+      def post_request(sentences)
+        uri = URI(ENDPOINT)
+        request = Net::HTTP::Post.new(uri)
+        request["Authorization"] = "Bearer #{@api_key}"
+        request["Content-Type"] = "application/json"
+        request.body = { input: sentences, model: @model }.to_json
+
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
+          http.request(request)
+        end
+      end
+    end
+  end
+end

data/lib/semantic_chunker/adapters/test_adapter.rb

@@ -0,0 +1,17 @@
+# lib/semantic_chunker/adapters/test_adapter.rb
+module SemanticChunker
+  module Adapters
+    class TestAdapter < Base
+      # We can pass specific vectors to simulate "topics"
+      def initialize(predefined_vectors = nil)
+        @predefined_vectors = predefined_vectors
+      end
+
+      def embed(sentences)
+        # If we have specific vectors, use them; 
+        # otherwise, return random vectors for each sentence
+        @predefined_vectors || sentences.map { [rand, rand, rand] }
+      end
+    end
+  end
+end

data/lib/semantic_chunker/chunker.rb

@@ -0,0 +1,107 @@
+# lib/semantic_chunker/chunker.rb
+require 'matrix'
+require 'pragmatic_segmenter'
+
+module SemanticChunker
+  class Chunker
+    DEFAULT_THRESHOLD = 0.82
+    DEFAULT_BUFFER = 1
+    DEFAULT_MAX_SIZE = 1500 # Characters
+
+    def initialize(embedding_provider: nil, threshold: DEFAULT_THRESHOLD, buffer_size: DEFAULT_BUFFER, max_chunk_size: DEFAULT_MAX_SIZE, segmenter_options: {})
+      @provider = embedding_provider || SemanticChunker.configuration&.provider
+      @threshold = threshold
+      @buffer_size = buffer_size
+      @max_chunk_size = max_chunk_size
+      @segmenter_options = segmenter_options # e.g., { language: 'hy', doc_type: 'pdf' }
+
+      raise ArgumentError, "A provider must be configured" if @provider.nil?
+    end
+
+    def chunks_for(text)
+      return [] if text.nil? || text.strip.empty?
+      sentences = split_sentences(text)
+
+      # Step 1: Logic to determine the best buffer window
+      effective_buffer = determine_buffer(sentences)
+      
+      # Step 2: Create overlapping "context groups" for more stable embeddings
+      context_groups = build_context_groups(sentences, effective_buffer)
+      
+      # Step 3: Embed the groups, not the raw sentences
+      group_embeddings = @provider.embed(context_groups)
+      
+      calculate_groups(sentences, group_embeddings)
+    end
+
+    private
+
+    # Selects buffer based on average sentence length if user passes :auto
+    def determine_buffer(sentences)
+      return @buffer_size unless @buffer_size == :auto
+
+      avg_length = sentences.map(&:length).sum / sentences.size.to_f
+      
+      # Strategy: If sentences are very short (< 50 chars), we need more context.
+      # If they are long (> 150 chars), they are likely self-contained.
+      case avg_length
+      when 0..50   then 2 # Look 2 ahead and 2 behind
+      when 51..150 then 1 # Standard
+      else 0              # Long sentences don't need buffers
+      end
+    end
+
+    def build_context_groups(sentences, buffer)
+      sentences.each_with_index.map do |_, i|
+        start_idx = [0, i - buffer].max
+        end_idx   = [sentences.size - 1, i + buffer].min
+        sentences[start_idx..end_idx].join(" ")
+      end
+    end
+
+    def split_sentences(text)
+      options = @segmenter_options.merge(text: text)
+      ps = PragmaticSegmenter::Segmenter.new(**options)
+      ps.segment
+    end
+
+    def calculate_groups(sentences, embeddings)
+      chunks = []
+      current_chunk_text = [sentences[0]]
+      current_chunk_vectors = [Vector[*embeddings[0]]]
+
+      (1...sentences.size).each do |i|
+        new_sentence = sentences[i]
+        new_vec = Vector[*embeddings[i]]
+        
+        # 1. Calculate Centroid
+        centroid = current_chunk_vectors.inject(:+) / current_chunk_vectors.size.to_f
+        sim = cosine_similarity(centroid, new_vec)
+
+        # 2. Check Constraints: Similarity OR Size
+        # We calculate the potential size of the chunk if we added this sentence
+        potential_size = current_chunk_text.join(" ").length + new_sentence.length + 1
+
+        if sim < @threshold || potential_size > @max_chunk_size
+          # Split if the topic changed OR the chunk is getting too fat
+          chunks << current_chunk_text.join(" ")
+          
+          current_chunk_text = [new_sentence]
+          current_chunk_vectors = [new_vec]
+        else
+          # Keep grouping
+          current_chunk_text << new_sentence
+          current_chunk_vectors << new_vec
+        end
+      end
+
+      chunks << current_chunk_text.join(" ")
+      chunks
+    end
+
+    def cosine_similarity(v1, v2)
+      return 0.0 if v1.magnitude.zero? || v2.magnitude.zero?
+      v1.inner_product(v2) / (v1.magnitude * v2.magnitude)
+    end
+  end
+end

data/lib/semantic_chunker/version.rb

@@ -0,0 +1,3 @@
+module SemanticChunker
+  VERSION = "0.5.3"
+end

data/lib/semantic_chunker.rb

@@ -0,0 +1,33 @@
+# lib/semantic_chunker.rb
+# 1. Require dependencies
+require 'matrix'
+require 'json'
+require 'net/http'
+
+# 2. Require the version and base modules
+require_relative 'semantic_chunker/version' if File.exist?('lib/semantic_chunker/version.rb')
+
+# 3. Require the internal logic
+require_relative 'semantic_chunker/adapters/base'
+require_relative 'semantic_chunker/adapters/openai_adapter'
+require_relative 'semantic_chunker/adapters/test_adapter'
+require_relative 'semantic_chunker/chunker'
+require_relative 'semantic_chunker/adapters/hugging_face_adapter'
+module SemanticChunker
+  class << self
+    attr_accessor :configuration
+  end
+
+  def self.configure
+    self.configuration ||= Configuration.new
+    yield(configuration)
+  end
+
+  class Configuration
+    attr_accessor :provider
+
+    def initialize
+      @provider = nil # User must set this
+    end
+  end
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: semantic_chunker
+version: !ruby/object:Gem::Version
+  version: 0.5.3
+platform: ruby
+authors:
+- Daniele Frisanco
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2026-01-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: vcr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pragmatic_segmenter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: matrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.4'
+description: Split long text into chunks based on semantic meaning.
+email:
+- daniele.frisanco@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/semantic_chunker.rb
+- lib/semantic_chunker/adapters/base.rb
+- lib/semantic_chunker/adapters/hugging_face_adapter.rb
+- lib/semantic_chunker/adapters/openai_adapter.rb
+- lib/semantic_chunker/adapters/test_adapter.rb
+- lib/semantic_chunker/chunker.rb
+- lib/semantic_chunker/version.rb
+homepage: https://github.com/danielefrisanco/semantic_chunker
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key: 
+specification_version: 4
+summary: Split long text into chunks based on semantic meaning.
+test_files: []