chunkify 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b65412205550d409af90f02a30709f5186e8c0e619225c88cb9d8602a049b84
-  data.tar.gz: 1d552da7a2fccc9bfd4147d1ea51e404059ef5b30b1c42cbe4366e61e92840cc
+  metadata.gz: 0e97f28d5ef6863efbe3de880495d714b0d05d8cd48cf9fad24f44aec7319f61
+  data.tar.gz: b85a362a843fc0130bc6125f3d6731f583fc8731104e6d9389150c0c000b3507
 SHA512:
-  metadata.gz: bdd42085fe657080776a3c45260d89149aa52517ef582a3ce1e48f972b07442291470b0abef4feab54408e24cbe33e0c56176f280bddd425988d9eb3b05367ce
-  data.tar.gz: 57ec9a2863800bc57cb967342bf3f0858c2d4ae9dc09a8b67d8f5d9918674135dd048a15cc62eb5045ea11735e6165da1da22d007f8bfb147c5426dc761190c1
+  metadata.gz: babf622c23a1f44dd64825cdbe9f828ab630badaef2c258edb8e90a004f72ae553a398038610a6a1dc174c3699f1a77eb0dfffe99ef5fc4285cd0cc899cbac0d
+  data.tar.gz: 3ae024a7ba605f5fa7581267ac4f983077902e028a99fe51bf38048b688064af415e2900b46258c2c5183f15b9116362cba5c1c4466eed265e1e37feebd636db

data/Gemfile

@@ -6,3 +6,5 @@ source "https://rubygems.org"
 gemspec
 
 gem "rake", "~> 13.0"
+
+gem "yard", "~> 0.9.38", :group => :development

data/Gemfile.lock

@@ -0,0 +1,27 @@
+PATH
+  remote: .
+  specs:
+    chunkify (0.1.0)
+      pry
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.3)
+    method_source (1.1.0)
+    pry (0.15.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rake (13.3.1)
+    yard (0.9.38)
+
+PLATFORMS
+  x86_64-linux
+
+DEPENDENCIES
+  chunkify!
+  rake (~> 13.0)
+  yard (~> 0.9.38)
+
+BUNDLED WITH
+   2.3.15

data/README.md

@@ -1,37 +1,23 @@
-# Chunkify
+# Chunkify Installation
+```
+gem install chunkify
+```
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/chunkify`. To experiment with that code, run `bin/console` for an interactive prompt.
+# Chunkify Usage
+```
+require 'chunkify'
 
-TODO: Delete this and the text above, and describe your gem
+# produce an array of chunks
+Chunkify.split("... a very large text ...")
 
-## Installation
+# map chunks
+Chunkify.split("... a very large text ...") do |chunk|
+  puts "CHUNK: #{chunk}"
+end
 
-Install the gem and add to the application's Gemfile by executing:
+# explicitly set chunk size
+Chunkify.size = 1024
 
-    $ bundle add chunkify
-
-If bundler is not being used to manage dependencies, install the gem by executing:
-
-    $ gem install chunkify
-
-## Usage
-
-TODO: Write usage instructions here
-
-## Development
-
-After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
-
-## Contributing
-
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/chunkify. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/chunkify/blob/master/CODE_OF_CONDUCT.md).
-
-## License
-
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
-
-## Code of Conduct
-
-Everyone interacting in the Chunkify project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/chunkify/blob/master/CODE_OF_CONDUCT.md).
+# recallibrate chunk size based upon 2 ** input tokens length
+Chunkify.size! "A size calibration string."
+```

data/lib/chunkify/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Chunkify
-  VERSION = "0.1.0"
+  VERSION = "0.1.2"
 end

data/lib/chunkify.rb

@@ -24,12 +24,12 @@ module Chunkify
       b = lambda { |e| e }
     end
     chunk = []
-    paragraphs = text.split(/\n\n+/)
+    paragraphs = text.gsub(/\r/, "").gsub(/\n\n+/,"\n\n").split(/\n\n+/)
     current_chunk = ""
     paragraphs.each do |para|
       if current_chunk.length + para.length > Chunkify.size && !current_chunk.empty?
         chunk << b.call(current_chunk)
-        current_chunk = para
+        current_chunk = para.split("\n").map { |e| e.strip.gsub(/  +/, "  ") }.join("\n")
       else
         current_chunk += (current_chunk.empty? ? "" : "\n\n") + para
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: chunkify
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Erik Olson
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-12-18 00:00:00.000000000 Z
+date: 2026-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - xorgnak@xorgnak.com
@@ -34,6 +48,7 @@ files:
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile