baran 0.1.12 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -1
- data/Gemfile.lock +3 -3
- data/README.md +3 -3
- data/lib/baran/sentence_text_splitter.rb +14 -0
- data/lib/baran/version.rb +1 -1
- data/lib/baran.rb +1 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bb9c125ca01ed50e901bb91c66be9b775e8ada6724c021c9ade1520615b74991
|
|
4
|
+
data.tar.gz: 6c94f295e48eb79ed37c45f7f431cfd47a5e668499daaa5b13b0b76bd3ca58b9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a4c6fba857e4a86b5a604c3241abe57798a523df601438e52ba4880426696041b4e5f950027bffe1e884fffd091c9ad0336f71011cdf42af019288cfbec80b37
|
|
7
|
+
data.tar.gz: 193ba60a25f5539c1d4b0d1d2bc6a552d1ade00cceab8a93fb11256e4ff5c7e92d78277159efa0a6908200ba015609fee83ec3de0a12aa280e10bf6f2e939541
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
baran (0.1
|
|
4
|
+
baran (0.2.1)
|
|
5
5
|
|
|
6
6
|
GEM
|
|
7
7
|
remote: https://rubygems.org/
|
|
8
8
|
specs:
|
|
9
|
-
minitest (5.
|
|
9
|
+
minitest (5.22.3)
|
|
10
10
|
rake (13.1.0)
|
|
11
11
|
|
|
12
12
|
PLATFORMS
|
|
@@ -15,7 +15,7 @@ PLATFORMS
|
|
|
15
15
|
|
|
16
16
|
DEPENDENCIES
|
|
17
17
|
baran!
|
|
18
|
-
minitest (~> 5.
|
|
18
|
+
minitest (~> 5.22)
|
|
19
19
|
rake (~> 13.1)
|
|
20
20
|
|
|
21
21
|
BUNDLED WITH
|
data/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
|
|
4
4
|
|
|
5
|
-
|
|
6
6
|
|
|
7
7
|
Text Splitter for Large Language Model datasets.
|
|
8
8
|
|
|
@@ -88,7 +88,7 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
|
|
|
88
88
|
|
|
89
89
|
## Contributing
|
|
90
90
|
|
|
91
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
|
91
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/kawakamimoeki/baran. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/kawakamimoeki/baran/blob/main/CODE_OF_CONDUCT.md).
|
|
92
92
|
|
|
93
93
|
## License
|
|
94
94
|
|
|
@@ -96,4 +96,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
|
|
|
96
96
|
|
|
97
97
|
## Code of Conduct
|
|
98
98
|
|
|
99
|
-
Everyone interacting in the Baran project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/
|
|
99
|
+
Everyone interacting in the Baran project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/kawakamimoeki/baran/blob/main/CODE_OF_CONDUCT.md).
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Baran
|
|
4
|
+
class SentenceTextSplitter < TextSplitter
|
|
5
|
+
def initialize(chunk_size: 1024, chunk_overlap: 64)
|
|
6
|
+
super(chunk_size: chunk_size, chunk_overlap: chunk_overlap)
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def splitted(text)
|
|
10
|
+
# Use a regex to split text based on the specified sentence-ending characters followed by whitespace
|
|
11
|
+
text.scan(/[^.!?]+[.!?]+(?:\s+|\z)/).map(&:strip)
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
data/lib/baran/version.rb
CHANGED
data/lib/baran.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative "baran/text_splitter"
|
|
|
5
5
|
require_relative "baran/markdown_splitter"
|
|
6
6
|
require_relative "baran/recursive_character_text_splitter"
|
|
7
7
|
require_relative "baran/character_text_splitter"
|
|
8
|
+
require_relative "baran/sentence_text_splitter"
|
|
8
9
|
|
|
9
10
|
module Baran
|
|
10
11
|
class Error < StandardError; end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: baran
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1
|
|
4
|
+
version: 0.2.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Moeki Kawakami
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-10-11 00:00:00.000000000 Z
|
|
12
12
|
dependencies: []
|
|
13
13
|
description: Text Splitter for Large Language Model Datasets.
|
|
14
14
|
email:
|
|
@@ -30,6 +30,7 @@ files:
|
|
|
30
30
|
- lib/baran/character_text_splitter.rb
|
|
31
31
|
- lib/baran/markdown_splitter.rb
|
|
32
32
|
- lib/baran/recursive_character_text_splitter.rb
|
|
33
|
+
- lib/baran/sentence_text_splitter.rb
|
|
33
34
|
- lib/baran/text_splitter.rb
|
|
34
35
|
- lib/baran/version.rb
|
|
35
36
|
- sig/baran.rbs
|