baran 0.1.4 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d64ad8b3b3b0f9d1bd32ce37453a6bf70581f5e229883b2d78b5200c236f01cc
-  data.tar.gz: 703df213c544557522ac36ac16404d04f369fd24c03446928aed7c2cadca1be4
+  metadata.gz: 3c45d696e127a2bdc00606665d6585c74f0a82657bcce81d3e581fb9d5c3e692
+  data.tar.gz: 514c86c899e2a804d26b3bc3f6a24d6d35a8a7108a0450562e3050b3c798a3f0
 SHA512:
-  metadata.gz: 471f2d07989fce4ec393eaff5dc995ae721902f607632f00816d7d8652a5740be13c75d5bae8e2d30028243de05c1eabac259806ff97758c6987efaf619873dc
-  data.tar.gz: f74445571e79a792c9d485f06b943b4a84f8366e80f02036ef7be14b628a1e9124b385a39a543d5964490f52a00b234f29f349e2327744f07b20a19a5242d746
+  metadata.gz: 359d335677a6a7c08f31c4f2602915374a4ed09772d045b2d098713e1ae550e81660f762aa6aec8bb3457ef2498a0832e97b26f19b82b7c5a1548fb45b859250
+  data.tar.gz: 3a010e8aa5547283641f8069a93e98cb2032a191411ec355398e00b4c83ce0953295c8ffea53e54c9240e2ae6edaa83d330f2d2f84dd3066a48438b9b70b223e

data/CHANGELOG.md

@@ -20,4 +20,8 @@
 ## [0.1.4] - 2023-05-28
 
 - Fix test
-- Fix README
+- Fix README
+
+## [0.1.5] - 2023-06-02
+
+- Refactor

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    baran (0.1.4)
+    baran (0.1.6)
 
 GEM
   remote: https://rubygems.org/

data/lib/baran/character_text_splitter.rb

@@ -2,13 +2,15 @@ require_relative './text_splitter'
 
 module Baran
   class CharacterTextSplitter < TextSplitter
+    attr_accessor :separator
+
     def initialize(chunk_size: 1024, chunk_overlap: 64, separator: nil)
       super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
       @separator = separator || "\n\n"
     end
 
     def splitted(text)
-      splits = @separator.empty? ? text.chars : text.split(@separator)
+      splits = separator.empty? ? text.chars : text.split(separator)
       merged(splits, @separator)
     end
   end

data/lib/baran/recursive_character_text_splitter.rb

@@ -6,46 +6,38 @@ module Baran
 
     def initialize(chunk_size: 1024, chunk_overlap: 64, separators: nil)
       super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
-      @separators = separators || ["\n\n", "\n", " ", ""]
+      @separators = separators || ["\n\n", "\n", " "]
     end
 
     def splitted(text)
-      final_chunks = []
+      results = []
+      good_splits = []
+      separator = ''
 
-      separator = @separators.last
-      @separators.each do |s|
-        if s.empty?
-          separator = s
-          break
-        elsif text.include?(s)
+      separators.each do |s|
+        if text.include?(s)
           separator = s
           break
         end
       end
 
-      splits = separator.empty? ? text.chars : text.split(separator)
-
-      good_splits = []
-      splits.each do |s|
-        if s.length < @chunk_size
+      text.split(separator).each do |s|
+        if s.length < chunk_size
           good_splits << s
         else
-          unless good_splits.empty?
-            merged_text = merged(good_splits, separator)
-            final_chunks.concat(merged_text)
+          if good_splits.length.positive?
+            results += merged(good_splits, separator)
             good_splits.clear
           end
-          other_info = splitted(s)
-          final_chunks.concat(other_info)
+          results += splitted(s)
         end
       end
 
-      unless good_splits.empty?
-        merged_text = merged(good_splits, separator)
-        final_chunks.concat(merged_text)
+      if good_splits.length.positive?
+        results += merged(good_splits, separator)
       end
 
-      final_chunks
+      results
     end
   end
 end

data/lib/baran/text_splitter.rb

@@ -24,39 +24,33 @@ module Baran
       chunks
     end
 
-    def join_docs(docs, separator)
-      text = docs.join(separator).strip
+    def joined(items, separator)
+      text = items.join(separator).strip
       text.empty? ? nil : text
     end
 
     def merged(splits, separator)
-      docs = [] # Array of strings
-      current_doc = [] # Array of strings
+      results = [] # Array of strings
+      current_splits = [] # Array of strings
       total = 0
 
-      splits.each do |d|
-        len = d.length
+      splits.each do |split|
+        if total + split.length >= chunk_size && current_splits.length.positive?
+          results << joined(current_splits, separator)
 
-        if total + len >= @chunk_size
-          unless current_doc.empty?
-            doc = join_docs(current_doc, separator)
-            docs.push(doc) unless doc.nil?
-
-            while total > @chunk_overlap || (total + len > @chunk_size && total.positive?)
-              total -= current_doc.first.length
-              current_doc.shift
-            end
+          while total > chunk_overlap || (total + split.length >= chunk_size && total.positive?)
+            total -= current_splits.first.length
+            current_splits.shift
           end
         end
 
-        current_doc.push(d)
-        total += len
+        current_splits << split
+        total += split.length
       end
 
-      doc = join_docs(current_doc, separator)
-      docs.push(doc) unless doc.nil?
+      results << joined(current_splits, separator)
 
-      docs
+      results
     end
   end
 end

data/lib/baran/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Baran
-  VERSION = "0.1.4"
+  VERSION = "0.1.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baran
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.6
 platform: ruby
 authors:
 - Moeki Kawakami
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-05-28 00:00:00.000000000 Z
+date: 2023-06-15 00:00:00.000000000 Z
 dependencies: []
 description: Text Splitter for Large Language Model Datasets.
 email: