baran 0.1.6 → 0.1.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/Gemfile.lock +3 -3
- data/README.md +4 -0
- data/lib/baran/text_splitter.rb +4 -1
- data/lib/baran/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: cac2a83bc8fd8a978a3fdc706747ee8f023f3d143b7b6c67e22408ade389e4ef
|
|
4
|
+
data.tar.gz: b0f0fccc1e7e9a7872400d929fd4a1cb8db2ec6b18d092177d6973506a0caa61
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 22a196b7a71c83c362e207ae242d1c0863d2d26d9be82aa0e7c09baf193b69478f2ad78b7f6d952d724435afee0435ab922851685e6f78afc9ac1e2746ac8389
|
|
7
|
+
data.tar.gz: c2753c59488a1f9bc65bf6fb8b08395f1cb476053c0a108a7fb53f621ec3b95eeabe9527fd43b1f9420f4fe774057592be5cfb57b88248800b832fdfb0d85368
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
baran (0.1.
|
|
4
|
+
baran (0.1.8)
|
|
5
5
|
|
|
6
6
|
GEM
|
|
7
7
|
remote: https://rubygems.org/
|
|
8
8
|
specs:
|
|
9
|
-
minitest (5.
|
|
9
|
+
minitest (5.19.0)
|
|
10
10
|
rake (13.0.6)
|
|
11
11
|
|
|
12
12
|
PLATFORMS
|
|
@@ -15,7 +15,7 @@ PLATFORMS
|
|
|
15
15
|
|
|
16
16
|
DEPENDENCIES
|
|
17
17
|
baran!
|
|
18
|
-
minitest (~> 5.
|
|
18
|
+
minitest (~> 5.19)
|
|
19
19
|
rake (~> 13.0)
|
|
20
20
|
|
|
21
21
|
BUNDLED WITH
|
data/README.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# Baran
|
|
2
2
|
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
3
7
|
Text Splitter for Large Language Model datasets.
|
|
4
8
|
|
|
5
9
|
To avoid token constraints and improve the accuracy of vector search in the Large Language Model, it is necessary to divide the document. This gem supports splitting the text in the specified manner.
|
data/lib/baran/text_splitter.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require 'logger'
|
|
2
|
+
|
|
1
3
|
module Baran
|
|
2
4
|
class TextSplitter
|
|
3
5
|
attr_accessor :chunk_size, :chunk_overlap
|
|
@@ -16,7 +18,7 @@ module Baran
|
|
|
16
18
|
cursor = 0
|
|
17
19
|
chunks = []
|
|
18
20
|
|
|
19
|
-
splitted(text).each do |chunk|
|
|
21
|
+
splitted(text).compact.each do |chunk|
|
|
20
22
|
chunks << { text: chunk, cursor: cursor }
|
|
21
23
|
cursor += chunk.length
|
|
22
24
|
end
|
|
@@ -46,6 +48,7 @@ module Baran
|
|
|
46
48
|
|
|
47
49
|
current_splits << split
|
|
48
50
|
total += split.length
|
|
51
|
+
Logger.new(STDOUT).warn("Created a chunk of size #{total}, which is longer than the specified #{@chunk_size}") if total > @chunk_size
|
|
49
52
|
end
|
|
50
53
|
|
|
51
54
|
results << joined(current_splits, separator)
|
data/lib/baran/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: baran
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Moeki Kawakami
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2023-
|
|
11
|
+
date: 2023-08-24 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Text Splitter for Large Language Model Datasets.
|
|
14
14
|
email:
|