baran 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cac2a83bc8fd8a978a3fdc706747ee8f023f3d143b7b6c67e22408ade389e4ef
-  data.tar.gz: b0f0fccc1e7e9a7872400d929fd4a1cb8db2ec6b18d092177d6973506a0caa61
+  metadata.gz: 51599bea086ef467f089b64f9252aa87a881c1889c776219695cec4922318be8
+  data.tar.gz: b9c20815e90bf477c8b98b3fb2580013c791dcf1a387332552bf51ae33882768
 SHA512:
-  metadata.gz: 22a196b7a71c83c362e207ae242d1c0863d2d26d9be82aa0e7c09baf193b69478f2ad78b7f6d952d724435afee0435ab922851685e6f78afc9ac1e2746ac8389
-  data.tar.gz: c2753c59488a1f9bc65bf6fb8b08395f1cb476053c0a108a7fb53f621ec3b95eeabe9527fd43b1f9420f4fe774057592be5cfb57b88248800b832fdfb0d85368
+  metadata.gz: 8add731f1eb06baa85e7bfe495f11ce58be02b8ca6947f3f0bbfc1d7ac85cad3b693d24dc7f4f9124e5a357f784d5f42eb6875e36d7d2fb1aac09dda22dd8862
+  data.tar.gz: 54e8924c8c2c86f524805b99cbeab0bb77a2784d72191fd35857def7450f9e9389176a63d684c4346c17787ad34bd7f3c9155b3803a6be2d0b22de24691a9ed2

data/Gemfile

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in baran.gemspec
 gemspec
 
-gem "minitest", "~> 5.19"
+gem "minitest", "~> 5.20"
 
 gem "rake", "~> 13.0"

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    baran (0.1.8)
+    baran (0.1.9)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.19.0)
+    minitest (5.20.0)
     rake (13.0.6)
 
 PLATFORMS
@@ -15,7 +15,7 @@ PLATFORMS
 
 DEPENDENCIES
   baran!
-  minitest (~> 5.19)
+  minitest (~> 5.20)
   rake (~> 13.0)
 
 BUNDLED WITH

data/lib/baran/character_text_splitter.rb

@@ -14,4 +14,4 @@ module Baran
       merged(splits, @separator)
     end
   end
-end
+end

data/lib/baran/text_splitter.rb

@@ -14,12 +14,14 @@ module Baran
       raise NotImplementedError, "splitted method should be implemented in a subclass"
     end
 
-    def chunks(text)
+    def chunks(text, metadata: nil)
       cursor = 0
       chunks = []
 
       splitted(text).compact.each do |chunk|
-        chunks << { text: chunk, cursor: cursor }
+        chunk = { text: chunk, cursor: cursor }
+        chunk[:metadata] = metadata if metadata
+        chunks << chunk
         cursor += chunk.length
       end
 
@@ -56,4 +58,4 @@ module Baran
       results
     end
   end
-end
+end

data/lib/baran/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Baran
-  VERSION = "0.1.8"
+  VERSION = "0.1.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baran
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - Moeki Kawakami
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-24 00:00:00.000000000 Z
+date: 2023-09-26 00:00:00.000000000 Z
 dependencies: []
 description: Text Splitter for Large Language Model Datasets.
 email: