text_splitters 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7be4926126e20d571d6cdc3b3dd746b40e4831b0f57a618b5af24d9ac3da9ee9
+  data.tar.gz: 9a1e953226c677f60bb9485c5429ac25f359f82b3ff49e6a8ed18de9f3444ceb
+SHA512:
+  metadata.gz: 25d6c946a4cf2e805b78a04b69a8700b5a89ada0be2802b90b876151831466b4241e17b34286d6eb9b5c9d766b7864bc1b81a9836ccdbd6a0bda2ee14e00a542
+  data.tar.gz: e60237fc48cbe8d399c201a99c0208af2ade1430bfb1243e75dcedcea2e6cd9f6d06639e84f90a7dfd368c8fbc315b90f76ca0ffc3face95575494c1308bae8e

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Alex Ghiculescu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+# text_splitters
+
+[![Gem Version](https://badge.fury.io/rb/text_splitters.svg)](https://rubygems.org/gems/text_splitters)
+[![CI](https://github.com/ghiculescu/text_splitters/actions/workflows/ci.yml/badge.svg)](https://github.com/ghiculescu/text_splitters/actions/workflows/ci.yml)
+[![Code Climate](https://codeclimate.com/github/ghiculescu/text_splitters/badges/gpa.svg)](https://codeclimate.com/github/ghiculescu/text_splitters)
+
+Port of [langchain](https://github.com/hwchase17/langchain) text splitters to Ruby.
+
+So far only the `RecursiveCharacterTextSplitter` is implemented. PRs for others are welcome!
+
+---
+
+- [Quick start](#quick-start)
+- [Support](#support)
+- [License](#license)
+- [Code of conduct](#code-of-conduct)
+- [Contribution guide](#contribution-guide)
+
+## Quick start
+
+```
+$ gem install text_splitters
+```
+
+```ruby
+require "text_splitters"
+```
+
+## Usage
+
+### `RecursiveCharacterTextSplitter`
+
+[Learn more about this splitter](https://langchain.readthedocs.io/en/latest/modules/indexes/examples/textsplitter.html#generic-recursive-text-splitting).
+
+```ruby
+text = "Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans."
+splitter = ::TextSplitters::RecursiveCharacterTextSplitter.new(chunk_size: 100, chunk_overlap: 20)
+
+output = splitter.split(text)
+
+output[0] # "Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet."
+output[1] # "and the Cabinet. Justices of the Supreme Court. My fellow Americans."
+```
+
+## Support
+
+If you want to report a bug, or have ideas, feedback or questions about the gem, [let me know via GitHub issues](https://github.com/ghiculescu/text_splitters/issues/new) and I will do my best to provide a helpful answer. Happy hacking!
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](LICENSE.txt).
+
+## Code of conduct
+
+Everyone interacting in this project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](CODE_OF_CONDUCT.md).
+
+## Contribution guide
+
+Pull requests are welcome!

data/lib/text_splitters/recursive_character_text_splitter.rb

@@ -0,0 +1,72 @@
+module TextSplitters
+  # Ruby port of https://github.com/hwchase17/langchain/blob/763f87953686a69897d1f4d2260388b88eb8d670/langchain/text_splitter.py#L221
+  class RecursiveCharacterTextSplitter
+    def initialize(chunk_size:, chunk_overlap:)
+      @chunk_size = chunk_size
+      @chunk_overlap = chunk_overlap
+      @separators = ["\n\n", "\n", " ", ""]
+    end
+
+    def split(text)
+      output = []
+      good_splits = []
+
+      separator = @separators.last
+      @separators.each do |s|
+        if text.include?(s)
+          separator = s
+          break
+        end
+      end
+      splits = text.split(separator)
+
+      splits.each do |s|
+        if s.length < @chunk_size
+          good_splits << s
+        else
+          if good_splits.any?
+            merged_text = merge_splits(good_splits, separator)
+            output.concat(merged_text)
+            good_splits = []
+          end
+
+          other_info = split(s)
+          output.concat(other_info)
+        end
+      end
+
+      if good_splits.any?
+        merged_text = merge_splits(good_splits, separator)
+        output.concat(merged_text)
+      end
+
+      output
+    end
+
+    private
+
+    def merge_splits(splits, separator)
+      output = []
+      current_doc = []
+      total = 0
+
+      splits.each do |split|
+        if total + split.length >= @chunk_size && current_doc.any?
+          doc = current_doc.join(separator).strip
+          output << doc if doc && !doc.empty?
+
+          while total > @chunk_overlap || (total > 0 && (total + split.length > @chunk_size))
+            total -= current_doc.first.length
+            current_doc.shift
+          end
+        end
+        current_doc << split
+        total += split.length
+      end
+      doc = current_doc.join(separator).strip
+      output << doc if doc && !doc.empty?
+
+      output
+    end
+  end
+end

data/lib/text_splitters/version.rb

@@ -0,0 +1,3 @@
+module TextSplitters
+  VERSION = "1.0.0".freeze
+end

data/lib/text_splitters.rb

@@ -0,0 +1,5 @@
+module TextSplitters
+  autoload :VERSION, "text_splitters/version"
+
+  autoload :RecursiveCharacterTextSplitter, "text_splitters/recursive_character_text_splitter"
+end

metadata

@@ -0,0 +1,53 @@
+--- !ruby/object:Gem::Specification
+name: text_splitters
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Alex Ghiculescu
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-03-05 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+- alex@tanda.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- lib/text_splitters.rb
+- lib/text_splitters/recursive_character_text_splitter.rb
+- lib/text_splitters/version.rb
+homepage: https://github.com/ghiculescu/text_splitters
+licenses:
+- MIT
+metadata:
+  bug_tracker_uri: https://github.com/ghiculescu/text_splitters/issues
+  changelog_uri: https://github.com/ghiculescu/text_splitters/releases
+  source_code_uri: https://github.com/ghiculescu/text_splitters
+  homepage_uri: https://github.com/ghiculescu/text_splitters
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: Port of langchain's text splitters
+test_files: []