chunkify 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b1714738ae78ae142793b80dd761e0a64413fd25ae73fa76ce60e0f099722903
-  data.tar.gz: 376154406ea4b78983b5957030b3aa81767eca3b885c2a30b2b853f08bec724a
+  metadata.gz: bd15841c4753a975e1259857c96c524ec5e0c435d9adca98edf59a93cb9ca045
+  data.tar.gz: 3f06158f83273718b61c4a87809b53490acd209ff9e2c0e12593526cde195f1c
 SHA512:
-  metadata.gz: d7b0f5f0436434fe235f4729c755135e21fa2002149f311523b6900379433d66351e5feaaa2880de5e6ba86c873160d719c4fd7432b31df1362a95a27d1dc9ea
-  data.tar.gz: 494bcba38175ec9a10fffea9016708351f7b289e38d341cf2e8bc40b007de8c96ab7f61cb34bafb0b3637b8384a8a7c755b81f14e252e8a10323e8a12326786e
+  metadata.gz: e78c1a25870032ee80ba74d7e218743ccdbde6af6e64e5dcc4c12befceaa1f8d997adf77fa8be9529c6c885988a07f1504a5f38b9a8d45fde2611b7990f0c251
+  data.tar.gz: 4b778ecd6c56a9b413ef813f343bbba8dc18d9e051d62229339b7c10f5903e1b27ead3bb73ad7e3798b6125590503e56f3203aa4d62a9e42420dfdcfa01eabdd

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    chunkify (0.1.0)
+    chunkify (0.1.4)
       pry
 
 GEM
@@ -13,6 +13,7 @@ GEM
       coderay (~> 1.1)
       method_source (~> 1.0)
     rake (13.3.1)
+    tokenizer (0.3.0)
     yard (0.9.38)
 
 PLATFORMS
@@ -21,6 +22,7 @@ PLATFORMS
 DEPENDENCIES
   chunkify!
   rake (~> 13.0)
+  tokenizer
   yard (~> 0.9.38)
 
 BUNDLED WITH

data/lib/chunkify/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Chunkify
-  VERSION = "0.1.3"
+  VERSION = "0.1.5"
 end

data/lib/chunkify.rb

@@ -1,39 +1,59 @@
 # frozen_string_literal: true
 
+require 'tokenizer'
 require_relative "chunkify/version"
 
 module Chunkify
   class Error < StandardError; end
-
-  @@SIZE = 1024
-
-  def self.size! input
-    @@SIZE = 2 ** input.split(" ").length
-  end
-  
-  def self.size= s
-    @@SIZE = s
-  end
-
-  def self.size
-    @@SIZE
+  def self.tokens text
+    Tokenizer::WhitespaceTokenizer.new(:en).tokenize(text)
   end
-  
-  def self.split text, &b
-    if !block_given?
-      b = lambda { |e| e }
-    end
+  def self.split text, size: 1024
     chunk = []
-    paragraphs = text.gsub(/\r/, "").gsub(/\n\n+/,"\n\n").split(/\n\n+/)
+    paragraphs = text.gsub(/\r/, "").gsub(/\n\n+/,"\n\n").split(/\n\n+/).map { |e| e.split("\n").map { |ee| ee.strip.gsub(/  +/, "  ")  }.join("\n") } 
     current_chunk = ""
     paragraphs.each do |para|
-      if current_chunk.length + para.length > Chunkify.size && !current_chunk.empty?
-        chunk << b.call(current_chunk)
-        current_chunk = para.split("\n").map { |e| e.strip.gsub(/  +/, "  ") }.join("\n")
+      if /Project Gutenberg/.match(para) || /gutenberg.org/.match(para)
+        next
       else
-        current_chunk += (current_chunk.empty? ? "" : "\n\n") + para.split("\n").map { |e| e.strip.gsub(/  +/, "  ") }.join("\n")
+        if current_chunk.length + para.length > size && !current_chunk.empty?
+          # premptive newline
+          chunk << current_chunk
+          # set next chunk
+          current_chunk = para
+        else
+          # add normally
+          current_chunk += (current_chunk.empty? ? "" : "\n\n") + para
+        end
       end
     end
-    chunk << b.call(current_chunk) unless current_chunk.empty?
+    chunk << current_chunk unless current_chunk.empty?
+  end
+  @@DOC = Hash.new { |h,k| h[k] = Doc.new(k) }
+  class Doc
+    attr_accessor :doc
+    def initialize k
+      @id = k
+    end
+    # chunk document (... and handle)
+    def chunk &b
+      if block_given?
+        Chunkify.split(@doc).map { |e| b.call(e) }
+      else
+        Chunkify.split(@doc)
+      end
+    end
+    # chunk document to tokens (... and handle)
+    def tokens &b
+      if block_given?
+        chunk { |e| b.call(Chunkify.tokens(e)) }
+      else
+        chunk.map { |e| Chunkify.tokens(e) }
+      end
+    end
+  end
+  # document
+  def self.[] k
+    @@DOC[k]
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: chunkify
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Erik Olson
@@ -38,6 +38,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: tokenizer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email:
 - xorgnak@xorgnak.com