chunker-ruby 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fb1949806664ba1e447e440f5dff4a2e1c072e2a0ed44248d235772ab23d121d
-  data.tar.gz: 2b4eb4b750714e39ef11f6fa40fc82640797b52ebca069f5993be18fa9a705f9
+  metadata.gz: 2ef1a60bf60351dc527abc378d992bfa05b0de0d9c64af3db4edbb63b9539c61
+  data.tar.gz: 01452e12091762a1dee9e86b2613e525a3dd9536cd51fd5d52da19e4a4f829dd
 SHA512:
-  metadata.gz: 7b2fc37c66650dfe14a035e36bc4dea38895a98f165520be4c077ca3f85ffb8a45f0b89e30b1f484184cf0e963e2e58227f2af26633896de09af2c4d13297f68
-  data.tar.gz: ae966e3dfa33ec187899018192fa80dbe78473e988bbeb91ccd165bc1e4e747d7cc97c12d6b22e37b9e298b5c35de2a8e581bf60f65516e93f6dbe7c656e70de
+  metadata.gz: a4b276bd94c9c0e7c6749223eecf8e78aa578f65866a4ab98f0d53e6245fc29ff622e6d8d262d3937c22b1f9e35e23f875c3ca52d5fc188138090c251ce1de29
+  data.tar.gz: d4c86c9423f92c20526a4f771c61e95f24edc7a11bf6920189d0eb408213c49e6f9828e02b1c6cb25121953a2f352a9b8c9f9e46db42af0ec5eca98b71820a2a

data/README.md

@@ -0,0 +1,235 @@
+# chunker-ruby
+
+Text chunking/splitting library for Ruby, designed for RAG (Retrieval-Augmented Generation) pipelines. Split documents into optimal pieces for embedding and vector search.
+
+Bad chunking = bad retrieval = bad RAG. This gem solves that.
+
+## Installation
+
+```ruby
+gem install chunker-ruby
+```
+
+Or add to your Gemfile:
+
+```ruby
+gem "chunker-ruby"
+```
+
+## Quick Start
+
+```ruby
+require "chunker_ruby"
+
+text = File.read("long_document.md")
+
+# Simple split (uses RecursiveCharacter by default)
+chunks = ChunkerRuby.split(text, chunk_size: 1000, chunk_overlap: 200)
+
+chunks.each do |chunk|
+  chunk.text       # => "The document begins..."
+  chunk.index      # => 0
+  chunk.offset     # => 0 (character offset in original)
+  chunk.length     # => 342
+  chunk.metadata   # => {}
+end
+```
+
+## Strategies
+
+### Character
+
+Fixed character count with overlap. Simplest strategy.
+
+```ruby
+chunker = ChunkerRuby::Character.new(chunk_size: 1000, chunk_overlap: 200)
+chunks = chunker.split(text)
+```
+
+### RecursiveCharacter
+
+Tries splitting by paragraph, then sentence, then word, then character. The most generally useful strategy.
+
+```ruby
+chunker = ChunkerRuby::RecursiveCharacter.new(
+  chunk_size: 1000,
+  chunk_overlap: 200,
+  separators: ["\n\n", "\n", ". ", ", ", " ", ""]  # default
+)
+chunks = chunker.split(text)
+```
+
+### Sentence
+
+Splits on sentence boundaries. Handles abbreviations (Dr., Mr., etc.) and decimal numbers.
+
+```ruby
+chunker = ChunkerRuby::Sentence.new(
+  min_chunk_size: 500,
+  max_chunk_size: 1500
+)
+chunks = chunker.split(text)
+```
+
+### Separator
+
+Split on a specific string or regex.
+
+```ruby
+chunker = ChunkerRuby::Separator.new(
+  separator: "\n\n",        # or a Regexp
+  keep_separator: true,
+  chunk_size: 1000
+)
+chunks = chunker.split(text)
+```
+
+### Markdown
+
+Splits on markdown headers (h1-h6). Respects code blocks. Preserves header hierarchy in metadata.
+
+```ruby
+chunker = ChunkerRuby::Markdown.new(chunk_size: 1000, chunk_overlap: 100)
+chunks = chunker.split(markdown_text)
+
+chunks.first.metadata[:headers]  # => ["# Introduction", "## Background"]
+```
+
+### HTML
+
+Splits on HTML block tags. Optionally strips tags.
+
+```ruby
+chunker = ChunkerRuby::HTML.new(chunk_size: 1000, strip_tags: true)
+chunks = chunker.split(html_text)
+```
+
+### Code
+
+Splits on function/class/method boundaries. Supports Ruby, Python, JavaScript, and TypeScript.
+
+```ruby
+chunker = ChunkerRuby::Code.new(language: :ruby, chunk_size: 1500)
+chunks = chunker.split(source_code)
+
+chunks.first.metadata[:language]  # => :ruby
+```
+
+### JSON
+
+Splits JSON arrays/objects into chunks. Each chunk is valid JSON.
+
+```ruby
+chunker = ChunkerRuby::JSONSplitter.new(chunk_size: 1000, chunk_overlap: 0)
+chunks = chunker.split(json_string)
+```
+
+### Token
+
+Splits by token count. Uses `tokenizer-ruby` if available, falls back to character estimation (~4 chars/token).
+
+```ruby
+chunker = ChunkerRuby::Token.new(
+  chunk_size: 512,        # in tokens
+  chunk_overlap: 50,
+  tokenizer: "gpt2"
+)
+chunks = chunker.split(text)
+```
+
+### Semantic
+
+Splits where embedding similarity drops (topic boundaries). Requires an embedding function.
+
+```ruby
+chunker = ChunkerRuby::Semantic.new(
+  embed: ->(text) { my_embedding_function(text) },
+  threshold: 0.5,
+  min_chunk_size: 100,
+  max_chunk_size: 2000
+)
+chunks = chunker.split(text)
+```
+
+### Sliding Window
+
+Fixed-size sliding window with configurable stride.
+
+```ruby
+chunker = ChunkerRuby::SlidingWindow.new(
+  chunk_size: 500,
+  chunk_overlap: 100,
+  stride: 200            # optional, defaults to chunk_size - chunk_overlap
+)
+chunks = chunker.split(text)
+```
+
+## Chunk Object
+
+Every strategy returns an array of `ChunkerRuby::Chunk` objects:
+
+```ruby
+chunk.text          # chunk content
+chunk.index         # position in sequence (0, 1, 2, ...)
+chunk.offset        # character offset in original document
+chunk.length        # character length
+chunk.metadata      # arbitrary metadata hash
+chunk.token_count   # estimated token count (or exact with tokenizer)
+chunk.to_h          # { text:, index:, offset:, length:, metadata: }
+chunk.to_s          # same as chunk.text
+```
+
+## Splitting Multiple Documents
+
+```ruby
+splitter = ChunkerRuby::RecursiveCharacter.new(chunk_size: 1000)
+chunks = splitter.split_many(["First document...", "Second document..."])
+
+chunks.first.metadata[:doc_index]  # => 0
+```
+
+## Rails Integration
+
+```ruby
+class Document < ApplicationRecord
+  include ChunkerRuby::Rails::Chunkable
+
+  chunkable :content,
+    strategy: :markdown,
+    chunk_size: 1000,
+    chunk_overlap: 200
+end
+
+document = Document.create!(content: long_text)
+document.chunks  # => [#<DocumentChunk text="..." chunk_index=0>, ...]
+```
+
+Requires a `DocumentChunk` model with `text`, `chunk_index`, `offset`, and `metadata` columns.
+
+## Choosing a Strategy
+
+| Use Case | Recommended Strategy |
+|---|---|
+| General text | `RecursiveCharacter` |
+| Markdown docs | `Markdown` |
+| Source code | `Code` |
+| HTML pages | `HTML` |
+| LLM context window management | `Token` |
+| Topic-based splitting | `Semantic` |
+| Simple fixed-size | `Character` or `SlidingWindow` |
+
+## Chunk Size Guidelines
+
+- **256-512 tokens**: Precise, fact-based retrieval (FAQ, definitions)
+- **512-1024 tokens**: Good balance for most use cases (docs, articles)
+- **1024-2048 tokens**: Complex topics needing more context (tutorials, guides)
+- **10-20% overlap**: Prevents context loss at boundaries
+
+## Dependencies
+
+- **Runtime**: None (pure Ruby)
+- **Optional**: `tokenizer-ruby` for token-based chunking
+
+## License
+
+MIT

data/lib/chunker_ruby/base_splitter.rb

@@ -25,6 +25,31 @@ module ChunkerRuby
 
     def build_chunks(pieces, original_text, metadata: {})
       chunks = []
+      current_pos = 0
+
+      merged = merge_pieces(pieces)
+
+      merged.each do |chunk_text|
+        next if chunk_text.strip.empty?
+
+        # Find the actual position starting from current_pos
+        offset = original_text.index(chunk_text, current_pos) || current_pos
+
+        chunks << Chunk.new(
+          text: chunk_text,
+          index: chunks.size,
+          offset: offset,
+          metadata: metadata.dup
+        )
+
+        current_pos = offset + chunk_text.length
+      end
+
+      chunks
+    end
+
+    def merge_pieces(pieces)
+      merged = []
       current_parts = []
       current_length = 0
 
@@ -32,14 +57,7 @@ module ChunkerRuby
         piece_len = piece.length
 
         if current_length + piece_len > @chunk_size && !current_parts.empty?
-          chunk_text = current_parts.join
-          offset = original_text.index(chunk_text) || 0
-          chunks << Chunk.new(
-            text: chunk_text,
-            index: chunks.size,
-            offset: offset,
-            metadata: metadata.dup
-          )
+          merged << current_parts.join
 
           # Handle overlap: keep trailing parts that fit within overlap size
           overlap_parts = []
@@ -61,18 +79,9 @@ module ChunkerRuby
         current_length += piece_len
       end
 
-      unless current_parts.empty?
-        chunk_text = current_parts.join
-        offset = original_text.rindex(chunk_text) || 0
-        chunks << Chunk.new(
-          text: chunk_text,
-          index: chunks.size,
-          offset: offset,
-          metadata: metadata.dup
-        )
-      end
+      merged << current_parts.join unless current_parts.empty?
 
-      chunks
+      merged
     end
   end
 end

data/lib/chunker_ruby/chunk.rb

@@ -29,6 +29,16 @@ module ChunkerRuby
       { text: @text, index: @index, offset: @offset, length: @length, metadata: @metadata }
     end
 
+    def valid?(original_text = nil)
+      return false if text.nil? || text.empty?
+      return false if offset.negative?
+      return false if index.negative?
+      if original_text
+        return false unless original_text[offset, text.length] == text
+      end
+      true
+    end
+
     def ==(other)
       other.is_a?(Chunk) && text == other.text && index == other.index && offset == other.offset
     end

data/lib/chunker_ruby/json_splitter.rb

@@ -10,6 +10,7 @@ module ChunkerRuby
       parsed = ::JSON.parse(text)
       pieces = extract_pieces(parsed)
       chunks = []
+      current_pos = 0
 
       current_parts = []
       current_length = 0
@@ -19,10 +20,13 @@ module ChunkerRuby
 
         if current_length + json_str.length > @chunk_size && !current_parts.empty?
           chunk_text = ::JSON.generate(current_parts.length == 1 ? current_parts.first : current_parts)
+          # Search for a key or value from the first piece to approximate offset
+          offset = find_json_offset(text, current_parts.first, current_pos)
+          current_pos = offset + chunk_text.length
           chunks << Chunk.new(
             text: chunk_text,
             index: chunks.size,
-            offset: 0,
+            offset: offset,
             metadata: metadata.dup
           )
           current_parts = []
@@ -35,10 +39,11 @@ module ChunkerRuby
 
       unless current_parts.empty?
         chunk_text = ::JSON.generate(current_parts.length == 1 ? current_parts.first : current_parts)
+        offset = find_json_offset(text, current_parts.first, current_pos)
         chunks << Chunk.new(
           text: chunk_text,
           index: chunks.size,
-          offset: 0,
+          offset: offset,
           metadata: metadata.dup
         )
       end
@@ -48,6 +53,22 @@ module ChunkerRuby
 
     private
 
+    def find_json_offset(text, first_piece, current_pos)
+      # Try to find a recognizable key or value from the first piece in the original text
+      search_str = case first_piece
+                   when Hash
+                     first_piece.keys.first.to_s
+                   when String
+                     first_piece
+                   else
+                     first_piece.to_s
+                   end
+
+      # Search for the key/value string as it would appear in JSON (quoted)
+      quoted = "\"#{search_str}\""
+      text.index(quoted, current_pos) || text.index(search_str, current_pos) || current_pos
+    end
+
     def extract_pieces(parsed)
       case parsed
       when Array

data/lib/chunker_ruby/rails/chunkable.rb

@@ -62,6 +62,9 @@ module ChunkerRuby
         when :html then ChunkerRuby::HTML
         when :code then ChunkerRuby::Code
         when :token then ChunkerRuby::Token
+        when :semantic then ChunkerRuby::Semantic
+        when :json then ChunkerRuby::JSONSplitter
+        when :sliding_window then ChunkerRuby::SlidingWindow
         else raise ArgumentError, "Unknown chunking strategy: #{strategy}"
         end
       end

data/lib/chunker_ruby/semantic.rb

@@ -48,6 +48,7 @@ module ChunkerRuby
 
     def build_semantic_chunks(sentences, split_points, original_text, metadata)
       chunks = []
+      current_pos = 0
       boundaries = [-1] + split_points + [sentences.length - 1]
 
       (0...boundaries.length - 1).each do |i|
@@ -64,15 +65,18 @@ module ChunkerRuby
           )
           sub_chunks = sub_splitter.split(chunk_text, metadata: metadata)
           sub_chunks.each do |sc|
+            offset = original_text.index(sc.text, current_pos) || current_pos
+            current_pos = offset + sc.text.length
             chunks << Chunk.new(
               text: sc.text,
               index: chunks.size,
-              offset: original_text.index(sc.text) || 0,
+              offset: offset,
               metadata: sc.metadata
             )
           end
         elsif chunk_text.length >= @min_chunk_size
-          offset = original_text.index(chunk_text) || 0
+          offset = original_text.index(chunk_text, current_pos) || current_pos
+          current_pos = offset + chunk_text.length
           chunks << Chunk.new(
             text: chunk_text,
             index: chunks.size,
@@ -89,8 +93,10 @@ module ChunkerRuby
             offset: prev.offset,
             metadata: prev.metadata
           )
+          current_pos = prev.offset + merged.length
         else
-          offset = original_text.index(chunk_text) || 0
+          offset = original_text.index(chunk_text, current_pos) || current_pos
+          current_pos = offset + chunk_text.length
           chunks << Chunk.new(
             text: chunk_text,
             index: chunks.size,

data/lib/chunker_ruby/token.rb

@@ -45,15 +45,19 @@ module ChunkerRuby
       tokens = @tokenizer.encode(text)
       chunks = []
       start = 0
+      current_pos = 0
 
       while start < tokens.length
         end_pos = [start + @chunk_size, tokens.length].min
         chunk_tokens = tokens[start...end_pos]
-        chunk_text = @tokenizer.decode(chunk_tokens)
+        raw_text = @tokenizer.decode(chunk_tokens)
+        stripped = raw_text.strip
+
+        offset = text.index(stripped, current_pos) || current_pos
+        current_pos = offset + stripped.length
 
-        offset = text.index(chunk_text.strip) || 0
         chunks << Chunk.new(
-          text: chunk_text,
+          text: raw_text,
           index: chunks.size,
           offset: offset,
           metadata: metadata.merge(token_count: chunk_tokens.length)

data/lib/chunker_ruby/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module ChunkerRuby
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chunker-ruby
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Johannes Dwi Cahyo
@@ -18,6 +18,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
+- README.md
 - lib/chunker_ruby.rb
 - lib/chunker_ruby/base_splitter.rb
 - lib/chunker_ruby/character.rb