baran 0.1.11 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 95a0c57558fc237d12ab005d24e444381725f8ddbb1fbbfe1ee730f9e14384ba
-  data.tar.gz: a7eacd8e62b27478df98aaaf534a6b37348696d2baa78e67c5781891c1b74c03
+  metadata.gz: 76dee67df26935789dbd26ee97d603c2f1fa96a49eff4466584016cf03e75d48
+  data.tar.gz: ea8899d24cd2b5177c76623cbfb705b49414a6af01e21c77d1596eb7f8ac845e
 SHA512:
-  metadata.gz: f09dd858f1dee1189ee543b440e8196871129fe33b6558762aedc85a1c9a26aebb1e439ff7c9b9e52bb0d92f6b1ffff37207105a146928c9350b26ed949019cd
-  data.tar.gz: 59ec6d83b1b7ce85e005dee095e8baf087ffffef750851a661957b431ef346521d75661cd266cf81e889eae64966b4876b3b8031ce43987fead5c2678437aace
+  metadata.gz: d1c2abb85d19c69087ec5247eed4769b8e063cef939d653d2d977b318cf838b6991508091105dca6714fb1493d9a73b2859f7a9193a320357683aab2f61ab1df
+  data.tar.gz: 6dd5c438d0d159fa52dec486013fe08cf9037bf9e250fd776af63be8be1f22eb8171f01b45f019c632960f790bdf90965c8c313d119636e0de0fca72b875726d

data/Gemfile

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in baran.gemspec
 gemspec
 
-gem "minitest", "~> 5.21"
+gem "minitest", "~> 5.22"
 
-gem "rake", "~> 13.0"
+gem "rake", "~> 13.1"

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    baran (0.1.11)
+    baran (0.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.21.2)
-    rake (13.0.6)
+    minitest (5.22.3)
+    rake (13.1.0)
 
 PLATFORMS
   arm64-darwin-22
@@ -15,8 +15,8 @@ PLATFORMS
 
 DEPENDENCIES
   baran!
-  minitest (~> 5.21)
-  rake (~> 13.0)
+  minitest (~> 5.22)
+  rake (~> 13.1)
 
 BUNDLED WITH
    2.4.7

data/README.md

@@ -2,7 +2,7 @@
 
 ![v](https://badgen.net/rubygems/v/baran)
 ![dt](https://badgen.net/rubygems/dt/baran)
-![license](https://badgen.net/github/license/moekidev/baran)
+![license](https://badgen.net/github/license/kawakamimoeki/baran)
 
 Text Splitter for Large Language Model datasets.
 
@@ -88,7 +88,7 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/moekidev/baran. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/moekidev/baran/blob/main/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at https://github.com/kawakamimoeki/baran. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/kawakamimoeki/baran/blob/main/CODE_OF_CONDUCT.md).
 
 ## License
 
@@ -96,4 +96,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the Baran project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/moekidev/baran/blob/main/CODE_OF_CONDUCT.md).
+Everyone interacting in the Baran project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/kawakamimoeki/baran/blob/main/CODE_OF_CONDUCT.md).

data/lib/baran/sentence_text_splitter.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Baran
+  class SentenceTextSplitter < TextSplitter
+    def initialize(chunk_size: 1024, chunk_overlap: 64)
+      super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
+    end
+
+    def splitted(text)
+      # Use a regex to split text based on the specified sentence-ending characters followed by whitespace
+      text.scan(/[^.!?]+[.!?]+(?:\s+)/).map(&:strip)
+    end
+  end
+end

data/lib/baran/text_splitter.rb

@@ -22,7 +22,7 @@ module Baran
         chunk = { text: chunk, cursor: cursor }
         chunk[:metadata] = metadata if metadata
         chunks << chunk
-        cursor += chunk.length
+        cursor += chunk[:text].length
       end
 
       chunks

data/lib/baran/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Baran
-  VERSION = "0.1.11"
+  VERSION = "0.2.0"
 end

data/lib/baran.rb

@@ -5,6 +5,7 @@ require_relative "baran/text_splitter"
 require_relative "baran/markdown_splitter"
 require_relative "baran/recursive_character_text_splitter"
 require_relative "baran/character_text_splitter"
+require_relative "baran/sentence_text_splitter"
 
 module Baran
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baran
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.2.0
 platform: ruby
 authors:
 - Moeki Kawakami
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-03-09 00:00:00.000000000 Z
+date: 2024-09-23 00:00:00.000000000 Z
 dependencies: []
 description: Text Splitter for Large Language Model Datasets.
 email:
@@ -30,6 +30,7 @@ files:
 - lib/baran/character_text_splitter.rb
 - lib/baran/markdown_splitter.rb
 - lib/baran/recursive_character_text_splitter.rb
+- lib/baran/sentence_text_splitter.rb
 - lib/baran/text_splitter.rb
 - lib/baran/version.rb
 - sig/baran.rbs