semantic_text_chunker 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bd5a74c09a20b7a87ae6d2a3c8a4cfda92315de9d4f18ffe54de76ee35b0db4f
+  data.tar.gz: cfe5098ea9fcd78e0fd8297755f2ac1c568d34656decf4b98b31a7c7f2e69c66
+SHA512:
+  metadata.gz: 8cc50f43d68240887b1af0498ac50a320a3c77fa97656cc60ecb8b2baa4c829c381bb2e66763d53b65dbd32def3f54952b33fd9e0278bfa681b752e046aa1d27
+  data.tar.gz: 391aba7c0d6dc181fc29a9b383dd3f07f1196dd4f4e5487e724b0893f80a6ca9a207f9f9e6cb8c6f003a91d800bdd074feb000bed985ba656cab4289f2dbbc53

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Vlad Tigănilă
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,147 @@
+# SemanticTextChunker
+
+Embedding-aware semantic chunking for Ruby RAG pipelines. Splits text into coherent chunks by detecting topic boundaries using embedding similarity, rather than blindly splitting on character count.
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "semantic_text_chunker"
+```
+
+Then run:
+
+```sh
+bundle install
+```
+
+Or install directly:
+
+```sh
+gem install semantic_text_chunker
+```
+
+## Quick Start
+
+```ruby
+require "semantic_text_chunker"
+
+text = "Your long document text here..."
+
+# Using OpenAI embeddings
+chunks = SemanticTextChunker.chunk(text,
+  embedder: SemanticTextChunker::Embedders::OpenAI.new(api_key: ENV["OPENAI_API_KEY"])
+)
+
+chunks.each { |chunk| puts chunk, "---" }
+```
+
+## Embedders
+
+### OpenAI
+
+```ruby
+embedder = SemanticTextChunker::Embedders::OpenAI.new(
+  api_key: ENV["OPENAI_API_KEY"],
+  model: "text-embedding-3-small"  # default
+)
+```
+
+### Cohere
+
+```ruby
+embedder = SemanticTextChunker::Embedders::Cohere.new(
+  api_key: ENV["COHERE_API_KEY"],
+  model: "embed-english-v3.0"  # default
+)
+```
+
+### OpenRouter
+
+```ruby
+embedder = SemanticTextChunker::Embedders::OpenRouter.new(
+  api_key: ENV["OPENROUTER_API_KEY"],
+  model: "openai/text-embedding-3-small"  # default
+)
+```
+
+### Null (no API required)
+
+A hash-based embedder useful for testing and development. No external API calls needed.
+
+```ruby
+embedder = SemanticTextChunker::Embedders::Null.new
+```
+
+## Options
+
+| Option              | Default | Description                                         |
+|---------------------|---------|-----------------------------------------------------|
+| `embedder`          | `Null`  | Embedder instance to use for generating embeddings   |
+| `threshold`         | `0.75`  | Cosine similarity threshold for detecting boundaries |
+| `max_tokens`        | `512`   | Maximum tokens per chunk (estimated at ~4 chars/token) |
+| `overlap_sentences` | `2`     | Number of sentences to overlap between chunks        |
+
+```ruby
+chunks = SemanticTextChunker.chunk(text,
+  embedder: embedder,
+  threshold: 0.8,
+  max_tokens: 1024,
+  overlap_sentences: 3
+)
+```
+
+## Metadata
+
+Prepend metadata to each chunk for better retrieval context:
+
+```ruby
+chunks = SemanticTextChunker.chunk_with_metadata(text,
+  embedder: embedder,
+  title: "The Great Gatsby",
+  author: "F. Scott Fitzgerald",
+  chapter: "Chapter 1",
+  section: "Opening",
+  source: "gutenberg.org"
+)
+```
+
+Each chunk will be prefixed with:
+
+```
+Title: The Great Gatsby
+Author: F. Scott Fitzgerald
+Chapter: Chapter 1
+Section: Opening
+Source: gutenberg.org
+
+<chunk text>
+```
+
+## Custom Embedders
+
+Create your own embedder by subclassing `SemanticTextChunker::Embedders::Base`:
+
+```ruby
+class MyEmbedder < SemanticTextChunker::Embedders::Base
+  def embed(texts)
+    # texts is an array of strings
+    # Return an array of embedding vectors (arrays of floats)
+    texts.map { |t| your_embedding_logic(t) }
+  end
+end
+```
+
+The base class provides a `cosine_similarity` method used for boundary detection.
+
+## How It Works
+
+1. **Sentence splitting** - Text is split into sentences using punctuation-aware rules that handle abbreviations (Mr., Dr., U.S., etc.)
+2. **Embedding** - Each sentence is embedded using the configured embedder
+3. **Boundary detection** - Consecutive sentences are grouped. A new chunk boundary is created when the cosine similarity between the accumulated chunk embedding and the next sentence drops below the threshold, or when the token limit is exceeded
+4. **Chunk building** - Sentences are assembled into chunks with configurable overlap for context continuity
+
+## License
+
+MIT

data/lib/semantic_text_chunker/boundary_detector.rb

@@ -0,0 +1,60 @@
+module SemanticTextChunker
+  class BoundaryDetector
+    def initialize(sentences:, embeddings:, threshold:, max_tokens:, embedder:)
+      @sentences  = sentences
+      @embeddings = embeddings
+      @threshold  = threshold
+      @max_tokens = max_tokens
+      @embedder   = embedder
+    end
+
+    # Returns array of sentence indices where chunks end
+    def boundaries
+      return [] if @sentences.size <= 1
+
+      boundaries    = []
+      chunk_start   = 0
+      current_text  = ""
+
+      @sentences.each_with_index do |sentence, i|
+        next if i == 0
+        current_text = @sentences[chunk_start..i - 1].join(" ")
+        next_text    = current_text + " " + sentence
+
+        # Force boundary if adding this sentence exceeds token limit
+        if tokens(next_text) > @max_tokens
+          boundaries << i - 1
+          chunk_start  = i
+          current_text = ""
+          next
+        end
+
+        # Compute similarity between accumulated chunk and next sentence
+        chunk_embedding    = mean_embedding(@embeddings[chunk_start..i - 1])
+        sentence_embedding = @embeddings[i]
+        similarity         = @embedder.cosine_similarity(chunk_embedding, sentence_embedding)
+
+        if similarity < @threshold
+          boundaries << i - 1
+          chunk_start = i
+        end
+      end
+
+      boundaries
+    end
+
+    private
+
+    def mean_embedding(embeddings)
+      return embeddings.first if embeddings.size == 1
+      dim = embeddings.first.size
+      sum = Array.new(dim, 0.0)
+      embeddings.each { |e| e.each_with_index { |v, i| sum[i] += v } }
+      sum.map { |v| v / embeddings.size }
+    end
+
+    def tokens(text)
+      (text.length / 4.0).ceil
+    end
+  end
+end

data/lib/semantic_text_chunker/chunk_builder.rb

@@ -0,0 +1,33 @@
+module SemanticTextChunker
+  class ChunkBuilder
+    def initialize(sentences:, boundaries:, overlap_sentences:)
+      @sentences         = sentences
+      @boundaries        = boundaries
+      @overlap_sentences = overlap_sentences
+    end
+
+    def build
+      return [@sentences.join(" ")] if @boundaries.empty?
+
+      chunks = []
+      prev_end = -1
+
+      split_points = @boundaries + [@sentences.size - 1]
+
+      split_points.each_with_index do |boundary, idx|
+        start = if idx == 0
+          0
+        else
+          # Overlap: go back N sentences from previous boundary
+          [prev_end - @overlap_sentences + 1, 0].max
+        end
+
+        chunk = @sentences[start..boundary].join(" ").strip
+        chunks << chunk unless chunk.empty?
+        prev_end = boundary
+      end
+
+      chunks
+    end
+  end
+end

data/lib/semantic_text_chunker/chunker.rb

@@ -0,0 +1,49 @@
+require_relative "splitters/sentence_splitter"
+require_relative "embedders/base"
+require_relative "embedders/null"
+require_relative "boundary_detector"
+require_relative "chunk_builder"
+require_relative "metadata"
+
+module SemanticTextChunker
+  class Chunker
+    def initialize(
+      embedder: Embedders::Null.new,
+      threshold: 0.75,
+      max_tokens: 512,
+      overlap_sentences: 2
+    )
+      @embedder          = embedder
+      @threshold         = threshold
+      @max_tokens        = max_tokens
+      @overlap_sentences = overlap_sentences
+      @splitter          = Splitters::SentenceSplitter.new
+    end
+
+    def chunk(text)
+      return [] if text.nil? || text.strip.empty?
+
+      sentences  = @splitter.split(text)
+      embeddings = @embedder.embed(sentences)
+
+      boundaries = BoundaryDetector.new(
+        sentences:  sentences,
+        embeddings: embeddings,
+        threshold:  @threshold,
+        max_tokens: @max_tokens,
+        embedder:   @embedder
+      ).boundaries
+
+      ChunkBuilder.new(
+        sentences:         sentences,
+        boundaries:        boundaries,
+        overlap_sentences: @overlap_sentences
+      ).build
+    end
+
+    def chunk_with_metadata(text, **metadata)
+      prefix = Metadata.prefix(**metadata)
+      chunk(text).map { |c| prefix + c }
+    end
+  end
+end

data/lib/semantic_text_chunker/embedders/base.rb

@@ -0,0 +1,17 @@
+module SemanticTextChunker
+  module Embedders
+    class Base
+      def embed(texts)
+        raise NotImplementedError, "#{self.class} must implement #embed"
+      end
+
+      def cosine_similarity(a, b)
+        dot   = a.zip(b).sum { |x, y| x * y }
+        mag_a = Math.sqrt(a.sum { |x| x**2 })
+        mag_b = Math.sqrt(b.sum { |x| x**2 })
+        return 0.0 if mag_a.zero? || mag_b.zero?
+        dot / (mag_a * mag_b)
+      end
+    end
+  end
+end

data/lib/semantic_text_chunker/embedders/cohere.rb

@@ -0,0 +1,41 @@
+require "net/http"
+require "json"
+require "uri"
+
+module SemanticTextChunker
+  module Embedders
+    class Cohere < Base
+      BATCH_SIZE = 96
+      ENDPOINT   = "https://api.cohere.com/v1/embed"
+
+      def initialize(api_key:, model: "embed-english-v3.0")
+        @api_key = api_key
+        @model   = model
+      end
+
+      def embed(texts)
+        texts.each_slice(BATCH_SIZE).flat_map do |batch|
+          response = post(batch)
+          response["embeddings"] || raise(EmbedderError, "No embeddings in response: #{response}")
+        end
+      end
+
+      private
+
+      def post(texts)
+        uri  = URI(ENDPOINT)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+
+        req = Net::HTTP::Post.new(uri)
+        req["Authorization"]  = "Bearer #{@api_key}"
+        req["Content-Type"]   = "application/json"
+        req.body = { texts: texts, model: @model, input_type: "search_document" }.to_json
+
+        res = http.request(req)
+        raise EmbedderError, "Cohere #{res.code}: #{res.body}" unless res.is_a?(Net::HTTPSuccess)
+        JSON.parse(res.body)
+      end
+    end
+  end
+end

data/lib/semantic_text_chunker/embedders/null.rb

@@ -0,0 +1,15 @@
+module SemanticTextChunker
+  module Embedders
+    class Null < Base
+      def embed(texts)
+        texts.map do |text|
+          words = text.downcase.split.uniq
+          vec   = Array.new(512, 0.0)
+          words.each { |w| vec[w.hash.abs % 512] += 1.0 }
+          norm = Math.sqrt(vec.sum { |x| x**2 })
+          norm > 0 ? vec.map { |x| x / norm } : vec
+        end
+      end
+    end
+  end
+end

data/lib/semantic_text_chunker/embedders/open_router.rb

@@ -0,0 +1,45 @@
+require "net/http"
+require "json"
+require "uri"
+
+module SemanticTextChunker
+  module Embedders
+    class OpenRouter < Base
+      BATCH_SIZE = 100
+      ENDPOINT   = "https://openrouter.ai/api/v1/embeddings"
+
+      def initialize(api_key:, model: "openai/text-embedding-3-small", site_url: nil, site_name: nil)
+        @api_key   = api_key
+        @model     = model
+        @site_url  = site_url  || "https://github.com/VladTZY/semantic_text_chunker"
+        @site_name = site_name || "semantic_text_chunker"
+      end
+
+      def embed(texts)
+        texts.each_slice(BATCH_SIZE).flat_map do |batch|
+          response = post(batch)
+          response["data"].map { |d| d["embedding"] }
+        end
+      end
+
+      private
+
+      def post(texts)
+        uri  = URI(ENDPOINT)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+
+        req = Net::HTTP::Post.new(uri)
+        req["Authorization"] = "Bearer #{@api_key}"
+        req["Content-Type"]  = "application/json"
+        req["HTTP-Referer"]  = @site_url   # Required by OpenRouter
+        req["X-Title"]       = @site_name
+        req.body = { input: texts, model: @model }.to_json
+
+        res = http.request(req)
+        raise EmbedderError, "OpenRouter #{res.code}: #{res.body}" unless res.is_a?(Net::HTTPSuccess)
+        JSON.parse(res.body)
+      end
+    end
+  end
+end

data/lib/semantic_text_chunker/embedders/openai.rb

@@ -0,0 +1,41 @@
+require "net/http"
+require "json"
+require "uri"
+
+module SemanticTextChunker
+  module Embedders
+    class OpenAI < Base
+      BATCH_SIZE = 100
+      ENDPOINT   = "https://api.openai.com/v1/embeddings"
+
+      def initialize(api_key:, model: "text-embedding-3-small")
+        @api_key = api_key
+        @model   = model
+      end
+
+      def embed(texts)
+        texts.each_slice(BATCH_SIZE).flat_map do |batch|
+          response = post(batch)
+          response["data"].map { |d| d["embedding"] }
+        end
+      end
+
+      private
+
+      def post(texts)
+        uri  = URI(ENDPOINT)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+
+        req = Net::HTTP::Post.new(uri)
+        req["Authorization"] = "Bearer #{@api_key}"
+        req["Content-Type"]  = "application/json"
+        req.body = { input: texts, model: @model }.to_json
+
+        res = http.request(req)
+        raise EmbedderError, "OpenAI #{res.code}: #{res.body}" unless res.is_a?(Net::HTTPSuccess)
+        JSON.parse(res.body)
+      end
+    end
+  end
+end

data/lib/semantic_text_chunker/metadata.rb

@@ -0,0 +1,22 @@
+module SemanticTextChunker
+  module Metadata
+    KNOWN_KEYS = %i[title author chapter section source].freeze
+
+    def self.prefix(**kwargs)
+      lines = []
+
+      KNOWN_KEYS.each do |key|
+        val = kwargs[key]
+        lines << "#{key.to_s.capitalize}: #{val}" if val && !val.to_s.empty?
+      end
+
+      # Any extra keys appended at end, titlecased
+      (kwargs.keys - KNOWN_KEYS).each do |key|
+        val = kwargs[key]
+        lines << "#{key.to_s.split('_').map(&:capitalize).join(' ')}: #{val}" if val
+      end
+
+      lines.empty? ? "" : lines.join("\n") + "\n\n"
+    end
+  end
+end

data/lib/semantic_text_chunker/splitters/sentence_splitter.rb

@@ -0,0 +1,20 @@
+module SemanticTextChunker
+  module Splitters
+    class SentenceSplitter
+      ABBREVS = %w[Mr Mrs Dr Prof Sr Jr vs etc e.g i.e U.S U.K U.S.A Fig Vol No].freeze
+      ABBREV_PATTERN = /\b(#{ABBREVS.map { |a| Regexp.escape(a) }.join("|")})\.\s/
+
+      def split(text)
+        # Temporarily replace abbreviation periods
+        protected = text.gsub(ABBREV_PATTERN) { "#{$1}__ABBREV__ " }
+
+        sentences = protected
+          .split(/(?<=[.?!])\s+(?=[A-Z])/)
+          .map { |s| s.gsub("__ABBREV__", ".").strip }
+          .reject(&:empty?)
+
+        sentences
+      end
+    end
+  end
+end

data/lib/semantic_text_chunker/version.rb

@@ -0,0 +1,3 @@
+module SemanticTextChunker
+  VERSION = "0.1.0"
+end

data/lib/semantic_text_chunker.rb

@@ -0,0 +1,22 @@
+require_relative "semantic_text_chunker/version"
+require_relative "semantic_text_chunker/chunker"
+require_relative "semantic_text_chunker/embedders/cohere"
+require_relative "semantic_text_chunker/embedders/openai"
+require_relative "semantic_text_chunker/embedders/open_router"
+require_relative "semantic_text_chunker/embedders/null"
+
+module SemanticTextChunker
+  class EmbedderError < StandardError; end
+
+  def self.chunk(text, **opts)
+    Chunker.new(**opts).chunk(text)
+  end
+
+  def self.chunk_with_metadata(text, **opts)
+    metadata_keys = %i[title author chapter section source]
+    chunker_opts  = opts.reject { |k, _| metadata_keys.include?(k) }
+    metadata      = opts.select { |k, _| metadata_keys.include?(k) }
+
+    Chunker.new(**chunker_opts).chunk_with_metadata(text, **metadata)
+  end
+end

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification
+name: semantic_text_chunker
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Vlad Tigănilă
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2026-05-19 00:00:00.000000000 Z
+dependencies: []
+description: Detects topic boundaries using embedding similarity to produce semantically
+  coherent chunks from books, articles, and documents. Supports Cohere, OpenAI, and
+  OpenRouter embedders.
+email:
+- tiganilavlad@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- lib/semantic_text_chunker.rb
+- lib/semantic_text_chunker/boundary_detector.rb
+- lib/semantic_text_chunker/chunk_builder.rb
+- lib/semantic_text_chunker/chunker.rb
+- lib/semantic_text_chunker/embedders/base.rb
+- lib/semantic_text_chunker/embedders/cohere.rb
+- lib/semantic_text_chunker/embedders/null.rb
+- lib/semantic_text_chunker/embedders/open_router.rb
+- lib/semantic_text_chunker/embedders/openai.rb
+- lib/semantic_text_chunker/metadata.rb
+- lib/semantic_text_chunker/splitters/sentence_splitter.rb
+- lib/semantic_text_chunker/version.rb
+homepage: https://github.com/VladTZY/semantic_text_chunker
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key:
+specification_version: 4
+summary: Embedding-aware semantic chunking for Ruby RAG pipelines
+test_files: []