baran 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51599bea086ef467f089b64f9252aa87a881c1889c776219695cec4922318be8
-  data.tar.gz: b9c20815e90bf477c8b98b3fb2580013c791dcf1a387332552bf51ae33882768
+  metadata.gz: f6cd3f94a89b19931ba237a67a23119a23549a70a200d6074e072d4ccd84f459
+  data.tar.gz: e395826b7cde638a330dfbf8f66a0864053bf96cebd0c49127706d0b1556bee6
 SHA512:
-  metadata.gz: 8add731f1eb06baa85e7bfe495f11ce58be02b8ca6947f3f0bbfc1d7ac85cad3b693d24dc7f4f9124e5a357f784d5f42eb6875e36d7d2fb1aac09dda22dd8862
-  data.tar.gz: 54e8924c8c2c86f524805b99cbeab0bb77a2784d72191fd35857def7450f9e9389176a63d684c4346c17787ad34bd7f3c9155b3803a6be2d0b22de24691a9ed2
+  metadata.gz: 421472e67d0279cac41ce4a36db642dadc3ca5b1b3f1f6cde6cb86e7d08038eec007e32b622e099fd6169a23cca2b25fd994598cfabd154901edfee74fe22ce4
+  data.tar.gz: 54e61bb7255e7269a06aa8df912d8a839c3c124dc4f2f73bba502070d07eafa3d04728848842b80005e4a0cfe3fcd0fb6244ee413ac1c6e283e87c2e2f21f451

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    baran (0.1.9)
+    baran (0.1.10)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -35,8 +35,8 @@ splitter = Baran::CharacterTextSplitter.new(
     chunk_overlap: 64,
     separator: "\n\n"
 )
-splitter.chunks(text)
-# => [{ cursor: 0, text: "..." }, ...]
+splitter.chunks(text, metadata: { ... })
+# => [{ cursor: 0, text: "...", metadata: { ... } }, ...]
 ```
 
 ### Recursive Character Text Splitter
@@ -47,8 +47,8 @@ Splitting by the specified characters recursively.
 splitter = Baran::RecursiveCharacterTextSplitter.new(
     separators: ["\n\n", "\n", " ", ""]
 )
-splitter.chunks(text)
-# => [{ cursor: 0, text: "..." }, ...]
+splitter.chunks(text, metadata: { ... })
+# => [{ cursor: 0, text: "...", metadata: { ... } }, ...]
 ```
 
 ### Markdown Text Splitter
@@ -57,8 +57,8 @@ Splitting by the Markdown descriptions.
 
 ```ruby
 splitter = Baran::MarkdownSplitter.new
-splitter.chunks(markdown)
-# => [{ cursor: 0, text: "..." }, ...]
+splitter.chunks(markdown, metadata: { ... })
+# => [{ cursor: 0, text: "...", metadata: { ... } }, ...]
 ```
 
 Split with the following priority.

data/lib/baran/markdown_splitter.rb

@@ -3,7 +3,7 @@ require_relative './recursive_character_text_splitter'
 module Baran
   class MarkdownSplitter < RecursiveCharacterTextSplitter
     def initialize(chunk_size: 1024, chunk_overlap: 64)
-      @separators = [
+      separators = [
         "\n# ", # h1
         "\n## ", # h2
         "\n### ", # h3
@@ -19,7 +19,7 @@ module Baran
         " ", # space
         "" # empty
       ]
-      super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
+      super(chunk_size: chunk_size, chunk_overlap: chunk_overlap, separators: separators)
     end
   end
-end
+end

data/lib/baran/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Baran
-  VERSION = "0.1.9"
+  VERSION = "0.1.10"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baran
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Moeki Kawakami
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-26 00:00:00.000000000 Z
+date: 2023-11-14 00:00:00.000000000 Z
 dependencies: []
 description: Text Splitter for Large Language Model Datasets.
 email: