baran 0.1.7 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c2f275e2741118e6b0505c9f3ae20d195ca082b9338f28dd9e5fee3899b6d3a5
-  data.tar.gz: e0eaa0d7e56895c59ab33028f88efd5157a7c091bfb508d35f8cbd19fddf78c1
+  metadata.gz: 51599bea086ef467f089b64f9252aa87a881c1889c776219695cec4922318be8
+  data.tar.gz: b9c20815e90bf477c8b98b3fb2580013c791dcf1a387332552bf51ae33882768
 SHA512:
-  metadata.gz: 76da807e98e3ee3c8c11146b9fd960ad7e254eb8e185eaabda1cf43b524b63a70916eee29e5b698e6a90fae9ae7c5c638729f5b20299a5842e2a2780b41ce890
-  data.tar.gz: ae3d9ff9180cebb6ee57ab85f2c04239497ed0fd7713042cdc30dc3f8ca603457a3554b4ac05e0d7272293a928c27997172bdaa5562443252b8a9b81faf827a4
+  metadata.gz: 8add731f1eb06baa85e7bfe495f11ce58be02b8ca6947f3f0bbfc1d7ac85cad3b693d24dc7f4f9124e5a357f784d5f42eb6875e36d7d2fb1aac09dda22dd8862
+  data.tar.gz: 54e8924c8c2c86f524805b99cbeab0bb77a2784d72191fd35857def7450f9e9389176a63d684c4346c17787ad34bd7f3c9155b3803a6be2d0b22de24691a9ed2

data/Gemfile

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in baran.gemspec
 gemspec
 
-gem "minitest", "~> 5.18"
+gem "minitest", "~> 5.20"
 
 gem "rake", "~> 13.0"

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    baran (0.1.7)
+    baran (0.1.9)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.18.1)
+    minitest (5.20.0)
     rake (13.0.6)
 
 PLATFORMS
@@ -15,7 +15,7 @@ PLATFORMS
 
 DEPENDENCIES
   baran!
-  minitest (~> 5.18)
+  minitest (~> 5.20)
   rake (~> 13.0)
 
 BUNDLED WITH

data/lib/baran/character_text_splitter.rb

@@ -14,4 +14,4 @@ module Baran
       merged(splits, @separator)
     end
   end
-end
+end

data/lib/baran/text_splitter.rb

@@ -14,12 +14,14 @@ module Baran
       raise NotImplementedError, "splitted method should be implemented in a subclass"
     end
 
-    def chunks(text)
+    def chunks(text, metadata: nil)
       cursor = 0
       chunks = []
 
-      splitted(text).each do |chunk|
-        chunks << { text: chunk, cursor: cursor }
+      splitted(text).compact.each do |chunk|
+        chunk = { text: chunk, cursor: cursor }
+        chunk[:metadata] = metadata if metadata
+        chunks << chunk
         cursor += chunk.length
       end
 
@@ -56,4 +58,4 @@ module Baran
       results
     end
   end
-end
+end

data/lib/baran/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Baran
-  VERSION = "0.1.7"
+  VERSION = "0.1.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baran
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.9
 platform: ruby
 authors:
 - Moeki Kawakami
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-10 00:00:00.000000000 Z
+date: 2023-09-26 00:00:00.000000000 Z
 dependencies: []
 description: Text Splitter for Large Language Model Datasets.
 email: