baran 0.1.6 → 0.1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +2 -2
- data/README.md +4 -0
- data/lib/baran/text_splitter.rb +3 -0
- data/lib/baran/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c2f275e2741118e6b0505c9f3ae20d195ca082b9338f28dd9e5fee3899b6d3a5
|
|
4
|
+
data.tar.gz: e0eaa0d7e56895c59ab33028f88efd5157a7c091bfb508d35f8cbd19fddf78c1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 76da807e98e3ee3c8c11146b9fd960ad7e254eb8e185eaabda1cf43b524b63a70916eee29e5b698e6a90fae9ae7c5c638729f5b20299a5842e2a2780b41ce890
|
|
7
|
+
data.tar.gz: ae3d9ff9180cebb6ee57ab85f2c04239497ed0fd7713042cdc30dc3f8ca603457a3554b4ac05e0d7272293a928c27997172bdaa5562443252b8a9b81faf827a4
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# Baran
|
|
2
2
|
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
3
7
|
Text Splitter for Large Language Model datasets.
|
|
4
8
|
|
|
5
9
|
To avoid token constraints and improve the accuracy of vector search in the Large Language Model, it is necessary to divide the document. This gem supports splitting the text in the specified manner.
|
data/lib/baran/text_splitter.rb
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require 'logger'
|
|
2
|
+
|
|
1
3
|
module Baran
|
|
2
4
|
class TextSplitter
|
|
3
5
|
attr_accessor :chunk_size, :chunk_overlap
|
|
@@ -46,6 +48,7 @@ module Baran
|
|
|
46
48
|
|
|
47
49
|
current_splits << split
|
|
48
50
|
total += split.length
|
|
51
|
+
Logger.new(STDOUT).warn("Created a chunk of size #{total}, which is longer than the specified #{@chunk_size}") if total > @chunk_size
|
|
49
52
|
end
|
|
50
53
|
|
|
51
54
|
results << joined(current_splits, separator)
|
data/lib/baran/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: baran
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Moeki Kawakami
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2023-
|
|
11
|
+
date: 2023-07-10 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Text Splitter for Large Language Model Datasets.
|
|
14
14
|
email:
|