semantic_chunks 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f68092098bba7fb9d5d521e5bda9cc7ccce6e668ecf90a3c872932a3528a7700
+  data.tar.gz: bfc7c91df83ebf5caa23187d35e4d459fed38470369e798089ec4bb0a3e6a5ae
+SHA512:
+  metadata.gz: 71bd25566777edace169e54c3013dddd5452e66fe1643b0350351ff18a955488751b5baad789d6255eeafb1b2ba8169a71c719be7a1bfe796e0019bf27ae4eac
+  data.tar.gz: a24da65e4102fed7f7cec62308a7a247f642cd15581f15e308fdd6ace77c69cf5863ce45e2127f464873d2ba51175309273fdd49622ca3e3d92f423a707f6b3d

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 iamzayn19
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,46 @@
+# SemanticChunks
+
+SemanticChunks is a pure Ruby text chunker for RAG, search indexing, and LLM apps.
+
+It solves a common Ruby AI problem: document preparation often gets pushed to Python just to split text into useful chunks. This gem keeps that step in Ruby.
+
+## Install
+
+```bash
+gem install semantic_chunks
+```
+
+## Usage
+
+```ruby
+require "semantic_chunks"
+
+chunks = SemanticChunks.split(markdown, max_tokens: 300, overlap: 40)
+
+chunks.each do |chunk|
+  puts chunk.text
+  p chunk.metadata
+end
+```
+
+Each chunk includes:
+
+- `text`
+- `index`
+- `token_count`
+- `char_start`
+- `char_end`
+- `headings`
+- `metadata`
+
+## CLI
+
+```bash
+semantic-chunks README.md --max-tokens 300 --overlap 40
+```
+
+Outputs JSON.
+
+## Why
+
+Ruby has excellent web frameworks and backend tooling, but a lot of AI utility work still assumes Python. SemanticChunks gives Ruby apps a native, predictable chunking layer before embeddings, vector search, or LLM calls.

data/exe/semantic-chunks

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "json"
+require "optparse"
+require "semantic_chunks"
+
+options = { max_tokens: 400, overlap: 60 }
+
+OptionParser.new do |parser|
+  parser.banner = "Usage: semantic-chunks FILE [--max-tokens N] [--overlap N]"
+  parser.on("--max-tokens N", Integer) { |value| options[:max_tokens] = value }
+  parser.on("--overlap N", Integer) { |value| options[:overlap] = value }
+end.parse!
+
+path = ARGV.fetch(0) do
+  warn "semantic-chunks: missing file"
+  exit 1
+end
+
+text = File.read(path, encoding: "UTF-8")
+chunks = SemanticChunks.split(text, **options).map(&:to_h)
+puts JSON.pretty_generate(chunks)

data/lib/semantic_chunks/chunk.rb

@@ -0,0 +1,15 @@
+module SemanticChunks
+  Chunk = Struct.new(:text, :index, :token_count, :char_start, :char_end, :headings, :metadata, keyword_init: true) do
+    def to_h
+      {
+        text: text,
+        index: index,
+        token_count: token_count,
+        char_start: char_start,
+        char_end: char_end,
+        headings: headings,
+        metadata: metadata
+      }
+    end
+  end
+end

data/lib/semantic_chunks/splitter.rb

@@ -0,0 +1,143 @@
+module SemanticChunks
+  class Splitter
+    WORD = /[[:alnum:]_]+(?:['-][[:alnum:]_]+)?|[^\s]/.freeze
+    SENTENCE_END = /(?<=[.!?])\s+/.freeze
+    HEADING = /\A(#{'#'}{1,6})\s+(.+?)\s*\z/.freeze
+
+    attr_reader :max_tokens, :overlap, :min_tokens
+
+    def initialize(max_tokens: 400, overlap: 60, min_tokens: 40)
+      raise ArgumentError, "max_tokens must be positive" unless max_tokens.positive?
+      raise ArgumentError, "overlap must be non-negative" if overlap.negative?
+      raise ArgumentError, "overlap must be smaller than max_tokens" if overlap >= max_tokens
+
+      @max_tokens = max_tokens
+      @overlap = overlap
+      @min_tokens = min_tokens
+    end
+
+    def split(text)
+      source = String(text)
+      units = semantic_units(source)
+      chunks = []
+      current = []
+      current_tokens = 0
+      current_start = nil
+      headings = []
+
+      units.each do |unit|
+        headings = unit[:headings] if unit[:heading]
+        if current.any? && current_tokens + unit[:tokens] > max_tokens
+          chunks << build_chunk(chunks.length, current, current_start, headings)
+          current, current_tokens, current_start = carry_overlap(current)
+        end
+
+        current_start ||= unit[:start]
+        current << unit
+        current_tokens += unit[:tokens]
+      end
+
+      chunks << build_chunk(chunks.length, current, current_start, headings) if current.any?
+      merge_small_tail(chunks)
+    end
+
+    private
+
+    def semantic_units(text)
+      headings = []
+      offset = 0
+      units = []
+
+      text.each_line do |line|
+        line_start = offset
+        offset += line.length
+
+        if (match = line.strip.match(HEADING))
+          level = match[1].length
+          headings = headings.take(level - 1) + [match[2]]
+          units << unit(line, line_start, headings, heading: true)
+          next
+        end
+
+        split_sentences(line).each do |sentence, sentence_start|
+          units << unit(sentence, line_start + sentence_start, headings)
+        end
+      end
+
+      units.reject { |entry| entry[:text].strip.empty? }
+    end
+
+    def split_sentences(line)
+      cursor = 0
+      line.split(SENTENCE_END).map do |sentence|
+        start = line.index(sentence, cursor) || cursor
+        cursor = start + sentence.length
+        [sentence, start]
+      end
+    end
+
+    def unit(text, start, headings, heading: false)
+      {
+        text: text,
+        start: start,
+        finish: start + text.length,
+        tokens: token_count(text),
+        headings: headings.dup,
+        heading: heading
+      }
+    end
+
+    def token_count(text)
+      text.scan(WORD).length
+    end
+
+    def carry_overlap(units)
+      return [[], 0, nil] if overlap.zero?
+
+      kept = []
+      count = 0
+      units.reverse_each do |unit|
+        break if count + unit[:tokens] > overlap && kept.any?
+
+        kept.unshift(unit)
+        count += unit[:tokens]
+      end
+
+      [kept, count, kept.first&.fetch(:start)]
+    end
+
+    def build_chunk(index, units, start, headings)
+      text = units.map { |unit| unit[:text].strip }.reject(&:empty?).join("\n\n")
+      chunk_headings = units.reverse.find { |unit| unit[:headings].any? }&.fetch(:headings) || headings
+
+      Chunk.new(
+        text: text,
+        index: index,
+        token_count: token_count(text),
+        char_start: start || 0,
+        char_end: units.last[:finish],
+        headings: chunk_headings,
+        metadata: { strategy: "semantic", overlap: overlap, max_tokens: max_tokens }
+      )
+    end
+
+    def merge_small_tail(chunks)
+      return chunks if chunks.length < 2
+      return chunks if chunks.last.token_count >= min_tokens
+
+      tail = chunks.pop
+      previous = chunks.pop
+      merged_text = [previous.text, tail.text].join("\n\n")
+
+      chunks << Chunk.new(
+        text: merged_text,
+        index: previous.index,
+        token_count: token_count(merged_text),
+        char_start: previous.char_start,
+        char_end: tail.char_end,
+        headings: tail.headings.any? ? tail.headings : previous.headings,
+        metadata: previous.metadata.merge(merged_tail: true)
+      )
+    end
+  end
+end

data/lib/semantic_chunks/version.rb

@@ -0,0 +1,3 @@
+module SemanticChunks
+  VERSION = "0.1.0"
+end

data/lib/semantic_chunks.rb

@@ -0,0 +1,11 @@
+require_relative "semantic_chunks/chunk"
+require_relative "semantic_chunks/splitter"
+require_relative "semantic_chunks/version"
+
+module SemanticChunks
+  class Error < StandardError; end
+
+  def self.split(text, **options)
+    Splitter.new(**options).split(text)
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: semantic_chunks
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- iamzayn19
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2026-09-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: SemanticChunks splits Markdown and plain text into token-budgeted chunks
+  with overlap, headings, offsets, and metadata without Python dependencies.
+email:
+- iamzayn19@gmail.com
+executables:
+- semantic-chunks
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- exe/semantic-chunks
+- lib/semantic_chunks.rb
+- lib/semantic_chunks/chunk.rb
+- lib/semantic_chunks/splitter.rb
+- lib/semantic_chunks/version.rb
+homepage: https://github.com/iamzayn19/semantic_chunks
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://github.com/iamzayn19/semantic_chunks/issues
+  source_code_uri: https://github.com/iamzayn19/semantic_chunks
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.22
+signing_key:
+specification_version: 4
+summary: Pure Ruby semantic text chunking for RAG and LLM apps.
+test_files: []