chunkify 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +2 -0
- data/Gemfile.lock +27 -0
- data/README.md +18 -32
- data/lib/chunkify/version.rb +1 -1
- data/lib/chunkify.rb +1 -1
- metadata +17 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b39d5a71f17cba9b3b93e216a9ecb51dcfc95a713947edf8a2561af47264ff6f
|
|
4
|
+
data.tar.gz: e632795b3f767bd03d7a3087f75dbcc5a081ec1b6a7c4dce1d98d625b3c0bb44
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e221ae6d69f0ad0e4420cd087a7044c6dad8fb5d22fedef0909629f475bbdacd1a62adb88fcacd43a77c5cbb7e4029378a2fab92406cbd144c1eb3713f76475c
|
|
7
|
+
data.tar.gz: 5996b1d6b6bc50c67649edfc73e6c7483d45ab1b4354e434a05e50eb9da6010f9eabe22274f8867e3fc9d8019fb05702d33bbef85995941869a9e2d236d87440
|
data/Gemfile
CHANGED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
chunkify (0.1.0)
|
|
5
|
+
pry
|
|
6
|
+
|
|
7
|
+
GEM
|
|
8
|
+
remote: https://rubygems.org/
|
|
9
|
+
specs:
|
|
10
|
+
coderay (1.1.3)
|
|
11
|
+
method_source (1.1.0)
|
|
12
|
+
pry (0.15.2)
|
|
13
|
+
coderay (~> 1.1)
|
|
14
|
+
method_source (~> 1.0)
|
|
15
|
+
rake (13.3.1)
|
|
16
|
+
yard (0.9.38)
|
|
17
|
+
|
|
18
|
+
PLATFORMS
|
|
19
|
+
x86_64-linux
|
|
20
|
+
|
|
21
|
+
DEPENDENCIES
|
|
22
|
+
chunkify!
|
|
23
|
+
rake (~> 13.0)
|
|
24
|
+
yard (~> 0.9.38)
|
|
25
|
+
|
|
26
|
+
BUNDLED WITH
|
|
27
|
+
2.3.15
|
data/README.md
CHANGED
|
@@ -1,37 +1,23 @@
|
|
|
1
|
-
# Chunkify
|
|
1
|
+
# Chunkify Installation
|
|
2
|
+
```
|
|
3
|
+
gem install chunkify
|
|
4
|
+
```
|
|
2
5
|
|
|
3
|
-
|
|
6
|
+
# Chunkify Usage
|
|
7
|
+
```
|
|
8
|
+
require 'chunkify'
|
|
4
9
|
|
|
5
|
-
|
|
10
|
+
# produce an array of chunks
|
|
11
|
+
Chunkify.split("... a very large text ...")
|
|
6
12
|
|
|
7
|
-
|
|
13
|
+
# map chunks
|
|
14
|
+
Chunkify.split("... a very large text ...") do |chunk|
|
|
15
|
+
puts "CHUNK: #{chunk}"
|
|
16
|
+
end
|
|
8
17
|
|
|
9
|
-
|
|
18
|
+
# explicitly set chunk size
|
|
19
|
+
Chunkify.size = 1024
|
|
10
20
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
$ gem install chunkify
|
|
16
|
-
|
|
17
|
-
## Usage
|
|
18
|
-
|
|
19
|
-
TODO: Write usage instructions here
|
|
20
|
-
|
|
21
|
-
## Development
|
|
22
|
-
|
|
23
|
-
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
24
|
-
|
|
25
|
-
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
26
|
-
|
|
27
|
-
## Contributing
|
|
28
|
-
|
|
29
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/chunkify. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/chunkify/blob/master/CODE_OF_CONDUCT.md).
|
|
30
|
-
|
|
31
|
-
## License
|
|
32
|
-
|
|
33
|
-
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
|
34
|
-
|
|
35
|
-
## Code of Conduct
|
|
36
|
-
|
|
37
|
-
Everyone interacting in the Chunkify project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/chunkify/blob/master/CODE_OF_CONDUCT.md).
|
|
21
|
+
# recallibrate chunk size based upon 2 ** input tokens length
|
|
22
|
+
Chunkify.size! "A size calibration string."
|
|
23
|
+
```
|
data/lib/chunkify/version.rb
CHANGED
data/lib/chunkify.rb
CHANGED
|
@@ -24,7 +24,7 @@ module Chunkify
|
|
|
24
24
|
b = lambda { |e| e }
|
|
25
25
|
end
|
|
26
26
|
chunk = []
|
|
27
|
-
paragraphs = text.split(/\n\n+/)
|
|
27
|
+
paragraphs = text.gsub(/\r/, "").gsub(/\n\n+/,"\n\n").split(/\n\n+/)
|
|
28
28
|
current_chunk = ""
|
|
29
29
|
paragraphs.each do |para|
|
|
30
30
|
if current_chunk.length + para.length > Chunkify.size && !current_chunk.empty?
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: chunkify
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Erik Olson
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2026-01-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: pry
|
|
@@ -24,6 +24,20 @@ dependencies:
|
|
|
24
24
|
- - ">="
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: yard
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - ">="
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '0'
|
|
34
|
+
type: :development
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - ">="
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '0'
|
|
27
41
|
description:
|
|
28
42
|
email:
|
|
29
43
|
- xorgnak@xorgnak.com
|
|
@@ -34,6 +48,7 @@ files:
|
|
|
34
48
|
- CHANGELOG.md
|
|
35
49
|
- CODE_OF_CONDUCT.md
|
|
36
50
|
- Gemfile
|
|
51
|
+
- Gemfile.lock
|
|
37
52
|
- LICENSE.txt
|
|
38
53
|
- README.md
|
|
39
54
|
- Rakefile
|