baran 0.1.8 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cac2a83bc8fd8a978a3fdc706747ee8f023f3d143b7b6c67e22408ade389e4ef
-  data.tar.gz: b0f0fccc1e7e9a7872400d929fd4a1cb8db2ec6b18d092177d6973506a0caa61
+  metadata.gz: f6cd3f94a89b19931ba237a67a23119a23549a70a200d6074e072d4ccd84f459
+  data.tar.gz: e395826b7cde638a330dfbf8f66a0864053bf96cebd0c49127706d0b1556bee6
 SHA512:
-  metadata.gz: 22a196b7a71c83c362e207ae242d1c0863d2d26d9be82aa0e7c09baf193b69478f2ad78b7f6d952d724435afee0435ab922851685e6f78afc9ac1e2746ac8389
-  data.tar.gz: c2753c59488a1f9bc65bf6fb8b08395f1cb476053c0a108a7fb53f621ec3b95eeabe9527fd43b1f9420f4fe774057592be5cfb57b88248800b832fdfb0d85368
+  metadata.gz: 421472e67d0279cac41ce4a36db642dadc3ca5b1b3f1f6cde6cb86e7d08038eec007e32b622e099fd6169a23cca2b25fd994598cfabd154901edfee74fe22ce4
+  data.tar.gz: 54e61bb7255e7269a06aa8df912d8a839c3c124dc4f2f73bba502070d07eafa3d04728848842b80005e4a0cfe3fcd0fb6244ee413ac1c6e283e87c2e2f21f451

data/Gemfile

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in baran.gemspec
 gemspec
 
-gem "minitest", "~> 5.19"
+gem "minitest", "~> 5.20"
 
 gem "rake", "~> 13.0"

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    baran (0.1.8)
+    baran (0.1.10)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.19.0)
+    minitest (5.20.0)
     rake (13.0.6)
 
 PLATFORMS
@@ -15,7 +15,7 @@ PLATFORMS
 
 DEPENDENCIES
   baran!
-  minitest (~> 5.19)
+  minitest (~> 5.20)
   rake (~> 13.0)
 
 BUNDLED WITH

data/README.md

@@ -35,8 +35,8 @@ splitter = Baran::CharacterTextSplitter.new(
     chunk_overlap: 64,
     separator: "\n\n"
 )
-splitter.chunks(text)
-# => [{ cursor: 0, text: "..." }, ...]
+splitter.chunks(text, metadata: { ... })
+# => [{ cursor: 0, text: "...", metadata: { ... } }, ...]
 ```
 
 ### Recursive Character Text Splitter
@@ -47,8 +47,8 @@ Splitting by the specified characters recursively.
 splitter = Baran::RecursiveCharacterTextSplitter.new(
     separators: ["\n\n", "\n", " ", ""]
 )
-splitter.chunks(text)
-# => [{ cursor: 0, text: "..." }, ...]
+splitter.chunks(text, metadata: { ... })
+# => [{ cursor: 0, text: "...", metadata: { ... } }, ...]
 ```
 
 ### Markdown Text Splitter
@@ -57,8 +57,8 @@ Splitting by the Markdown descriptions.
 
 ```ruby
 splitter = Baran::MarkdownSplitter.new
-splitter.chunks(markdown)
-# => [{ cursor: 0, text: "..." }, ...]
+splitter.chunks(markdown, metadata: { ... })
+# => [{ cursor: 0, text: "...", metadata: { ... } }, ...]
 ```
 
 Split with the following priority.

data/lib/baran/character_text_splitter.rb

@@ -14,4 +14,4 @@ module Baran
       merged(splits, @separator)
     end
   end
-end
+end

data/lib/baran/markdown_splitter.rb

@@ -3,7 +3,7 @@ require_relative './recursive_character_text_splitter'
 module Baran
   class MarkdownSplitter < RecursiveCharacterTextSplitter
     def initialize(chunk_size: 1024, chunk_overlap: 64)
-      @separators = [
+      separators = [
         "\n# ", # h1
         "\n## ", # h2
         "\n### ", # h3
@@ -19,7 +19,7 @@ module Baran
         " ", # space
         "" # empty
       ]
-      super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
+      super(chunk_size: chunk_size, chunk_overlap: chunk_overlap, separators: separators)
     end
   end
-end
+end

data/lib/baran/text_splitter.rb

@@ -14,12 +14,14 @@ module Baran
       raise NotImplementedError, "splitted method should be implemented in a subclass"
     end
 
-    def chunks(text)
+    def chunks(text, metadata: nil)
       cursor = 0
       chunks = []
 
       splitted(text).compact.each do |chunk|
-        chunks << { text: chunk, cursor: cursor }
+        chunk = { text: chunk, cursor: cursor }
+        chunk[:metadata] = metadata if metadata
+        chunks << chunk
         cursor += chunk.length
       end
 
@@ -56,4 +58,4 @@ module Baran
       results
     end
   end
-end
+end

data/lib/baran/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Baran
-  VERSION = "0.1.8"
+  VERSION = "0.1.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baran
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.10
 platform: ruby
 authors:
 - Moeki Kawakami
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-24 00:00:00.000000000 Z
+date: 2023-11-14 00:00:00.000000000 Z
 dependencies: []
 description: Text Splitter for Large Language Model Datasets.
 email: