baran 0.1.9 → 0.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51599bea086ef467f089b64f9252aa87a881c1889c776219695cec4922318be8
-  data.tar.gz: b9c20815e90bf477c8b98b3fb2580013c791dcf1a387332552bf51ae33882768
+  metadata.gz: 95a0c57558fc237d12ab005d24e444381725f8ddbb1fbbfe1ee730f9e14384ba
+  data.tar.gz: a7eacd8e62b27478df98aaaf534a6b37348696d2baa78e67c5781891c1b74c03
 SHA512:
-  metadata.gz: 8add731f1eb06baa85e7bfe495f11ce58be02b8ca6947f3f0bbfc1d7ac85cad3b693d24dc7f4f9124e5a357f784d5f42eb6875e36d7d2fb1aac09dda22dd8862
-  data.tar.gz: 54e8924c8c2c86f524805b99cbeab0bb77a2784d72191fd35857def7450f9e9389176a63d684c4346c17787ad34bd7f3c9155b3803a6be2d0b22de24691a9ed2
+  metadata.gz: f09dd858f1dee1189ee543b440e8196871129fe33b6558762aedc85a1c9a26aebb1e439ff7c9b9e52bb0d92f6b1ffff37207105a146928c9350b26ed949019cd
+  data.tar.gz: 59ec6d83b1b7ce85e005dee095e8baf087ffffef750851a661957b431ef346521d75661cd266cf81e889eae64966b4876b3b8031ce43987fead5c2678437aace

data/CHANGELOG.md

@@ -8,7 +8,6 @@
 
 - Update README
 
-
 ## [0.1.2] - 2023-05-28
 
 - Fix README
@@ -24,4 +23,8 @@
 
 ## [0.1.5] - 2023-06-02
 
-- Refactor
+- Refactor
+
+## [0.1.10] - 2024-03-09
+
+- https://github.com/kawakamimoeki/baran/pull/17

data/Gemfile

@@ -5,6 +5,6 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in baran.gemspec
 gemspec
 
-gem "minitest", "~> 5.20"
+gem "minitest", "~> 5.21"
 
 gem "rake", "~> 13.0"

data/Gemfile.lock

@@ -1,12 +1,12 @@
 PATH
   remote: .
   specs:
-    baran (0.1.9)
+    baran (0.1.11)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    minitest (5.20.0)
+    minitest (5.21.2)
     rake (13.0.6)
 
 PLATFORMS
@@ -15,7 +15,7 @@ PLATFORMS
 
 DEPENDENCIES
   baran!
-  minitest (~> 5.20)
+  minitest (~> 5.21)
   rake (~> 13.0)
 
 BUNDLED WITH

data/README.md

@@ -35,8 +35,8 @@ splitter = Baran::CharacterTextSplitter.new(
     chunk_overlap: 64,
     separator: "\n\n"
 )
-splitter.chunks(text)
-# => [{ cursor: 0, text: "..." }, ...]
+splitter.chunks(text, metadata: { ... })
+# => [{ cursor: 0, text: "...", metadata: { ... } }, ...]
 ```
 
 ### Recursive Character Text Splitter
@@ -47,8 +47,8 @@ Splitting by the specified characters recursively.
 splitter = Baran::RecursiveCharacterTextSplitter.new(
     separators: ["\n\n", "\n", " ", ""]
 )
-splitter.chunks(text)
-# => [{ cursor: 0, text: "..." }, ...]
+splitter.chunks(text, metadata: { ... })
+# => [{ cursor: 0, text: "...", metadata: { ... } }, ...]
 ```
 
 ### Markdown Text Splitter
@@ -57,8 +57,8 @@ Splitting by the Markdown descriptions.
 
 ```ruby
 splitter = Baran::MarkdownSplitter.new
-splitter.chunks(markdown)
-# => [{ cursor: 0, text: "..." }, ...]
+splitter.chunks(markdown, metadata: { ... })
+# => [{ cursor: 0, text: "...", metadata: { ... } }, ...]
 ```
 
 Split with the following priority.

data/lib/baran/markdown_splitter.rb

@@ -3,7 +3,7 @@ require_relative './recursive_character_text_splitter'
 module Baran
   class MarkdownSplitter < RecursiveCharacterTextSplitter
     def initialize(chunk_size: 1024, chunk_overlap: 64)
-      @separators = [
+      separators = [
         "\n# ", # h1
         "\n## ", # h2
         "\n### ", # h3
@@ -19,7 +19,7 @@ module Baran
         " ", # space
         "" # empty
       ]
-      super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
+      super(chunk_size: chunk_size, chunk_overlap: chunk_overlap, separators: separators)
     end
   end
-end
+end

data/lib/baran/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Baran
-  VERSION = "0.1.9"
+  VERSION = "0.1.11"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: baran
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.11
 platform: ruby
 authors:
 - Moeki Kawakami
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-26 00:00:00.000000000 Z
+date: 2024-03-09 00:00:00.000000000 Z
 dependencies: []
 description: Text Splitter for Large Language Model Datasets.
 email: