baran 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c113913bc9751c8dccc46fef4cf58b552e2ef629e0cbcbecf4009f1fb0c9665
-  data.tar.gz: b166d1eab9681981435367ec57d24ce0dd12f45086f7654cfee0ffc6b120e2ec
+  metadata.gz: 8342de6adf861cc80d2e3b2b409e607b7b6a4a9eded4ce6218710ba9fa66d164
+  data.tar.gz: 2b50b7070a46aadc2f2a6727e9418134d6123dda81c5f3b9f21787e500142b7e
 SHA512:
-  metadata.gz: 177b7eb08f07236e680316806fbf0f4fd2ef576ec36cf4005660503b594b90b023c1c29db54860b26990e65ea9ffa3b4d6dc94c913007b09cf018dd01d606754
-  data.tar.gz: e7ec13bcadda327bb1ed691fc9bf55e8954f160bd33167549088baf92280a9e2b2c846450b15656034a46f3697f0c2513fee0ddda24528f56920c48606fb3e23
+  metadata.gz: '05859643d5c56270306611402878d0290658c327d086cc0fc3faf3098ada35a9efaa0a2bd027a848ef84157ba82014438e9ceab30139a2d56e1fe1223d8a4e5d'
+  data.tar.gz: b778120886153d0ad29eceb118534ef65ab68de7c6c956d8752d0c78b2aec947831b5a73ba75a202e2b8980a68c5f6d68bda69199b6be6f21657f3b6250b6941

data/CHANGELOG.md

@@ -15,4 +15,13 @@
 
 ## [0.1.3] - 2023-05-28
 
-- Fix README
+- Fix README
+
+## [0.1.4] - 2023-05-28
+
+- Fix test
+- Fix README
+
+## [0.1.5] - 2023-06-02
+
+- Refactor

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    baran (0.1.3)
+    baran (0.1.5)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -41,7 +41,7 @@ Splitting by the specified characters recursively.
 
 ```ruby
 splitter = Baran::RecursiveCharacterTextSplitter.new(
-    separators: ["\nn", "\n", " ", ""]
+    separators: ["\n\n", "\n", " ", ""]
 )
 splitter.chunks(text)
 # => [{ cursor: 0, text: "..." }, ...]

data/lib/baran/character_text_splitter.rb

@@ -2,13 +2,15 @@ require_relative './text_splitter'
 
 module Baran
   class CharacterTextSplitter < TextSplitter
+    attr_accessor :separator
+
     def initialize(chunk_size: 1024, chunk_overlap: 64, separator: nil)
       super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
       @separator = separator || "\n\n"
     end
 
     def splitted(text)
-      splits = @separator.empty? ? text.chars : text.split(@separator)
+      splits = separator.empty? ? text.chars : text.split(separator)
       merged(splits, @separator)
     end
   end

data/lib/baran/recursive_character_text_splitter.rb

@@ -6,46 +6,38 @@ module Baran
 
     def initialize(chunk_size: 1024, chunk_overlap: 64, separators: nil)
       super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
-      @separators = separators || ["\n\n", "\n", " ", ""]
+      @separators = separators || ["\n\n", "\n", " "]
     end
 
     def splitted(text)
-      final_chunks = []
+      results = []
+      good_splits = []
+      separator = ''
 
-      separator = @separators.last
-      @separators.each do |s|
-        if s.empty?
-          separator = s
-          break
-        elsif text.include?(s)
+      separators.each do |s|
+        if text.include?(s)
           separator = s
           break
         end
       end
 
-      splits = separator.empty? ? text.chars : text.split(separator)
-
-      good_splits = []
-      splits.each do |s|
-        if s.length < @chunk_size
+      text.split(separator).each do |s|
+        if s.length < chunk_size
           good_splits << s
         else
-          unless good_splits.empty?
-            merged_text = merged(good_splits, separator)
-            final_chunks.concat(merged_text)
+          if good_splits.length.positive?
+            results += merged(good_splits, separator)
             good_splits.clear
           end
-          other_info = splitted(s)
-          final_chunks.concat(other_info)
+          results += splitted(s)
         end
       end
 
-      unless good_splits.empty?
-        merged_text = merged(good_splits, separator)
-        final_chunks.concat(merged_text)
+      if good_splits.length.positive?
+        results += merged(good_splits, separator)
       end
 
-      final_chunks
+      results
     end
   end
 end

data/lib/baran/text_splitter.rb

@@ -24,39 +24,33 @@ module Baran
       chunks
     end
 
-    def join_docs(docs, separator)
-      text = docs.join(separator).strip
+    def joined(items, separator)
+      text = items.join(separator).strip
       text.empty? ? nil : text
     end
 
     def merged(splits, separator)
-      docs = [] # Array of strings
-      current_doc = [] # Array of strings
+      results = [] # Array of strings
+      current_splits = [] # Array of strings
       total = 0
 
-      splits.each do |d|
-        len = d.length
+      splits.each do |split|
+        if total + split.length >= chunk_size && current_splits.length.positive?
+          results << joined(current_splits, separator)
 
-        if total + len >= @chunk_size
-          unless current_doc.empty?
-            doc = join_docs(current_doc, separator)
-            docs.push(doc) unless doc.nil?
-
-            while total > @chunk_overlap || (total + len > @chunk_size && total.positive?)
-              total -= current_doc.first.length
-              current_doc.shift
-            end
+          while total > chunk_overlap || (total + split.length > chunk_size && total.positive?)
+            total -= current_splits.first.length
+            current_splits.shift
           end
         end
 
-        current_doc.push(d)
-        total += len
+        current_splits << split
+        total += split.length
       end
 
-      doc = join_docs(current_doc, separator)
-      docs.push(doc) unless doc.nil?
+      results << joined(current_splits, separator)
 
-      docs
+      results
     end
   end
 end

data/lib/baran/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Baran
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baran
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Moeki Kawakami
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-28 00:00:00.000000000 Z
+date: 2023-06-02 00:00:00.000000000 Z
 dependencies: []
 description: Text Splitter for Large Language Model Datasets.
 email: