tokenizers 0.4.1-x86_64-darwin → 0.4.3-x86_64-darwin

data/README.md

@@ -34,15 +34,51 @@ Decode
 tokenizer.decode(ids)
 ```
 
-Load a tokenizer from files
+## Training
+
+Create a tokenizer
 
 ```ruby
-tokenizer = Tokenizers::CharBPETokenizer.new("vocab.json", "merges.txt")
+tokenizer = Tokenizers::Tokenizer.new(Tokenizers::Models::BPE.new(unk_token: "[UNK]"))
 ```
 
-## Training
+Set the pre-tokenizer
+
+```ruby
+tokenizer.pre_tokenizer = Tokenizers::PreTokenizers::Whitespace.new
+```
+
+Train the tokenizer ([example data](https://huggingface.co/docs/tokenizers/quicktour#build-a-tokenizer-from-scratch))
+
+```ruby
+trainer = Tokenizers::Trainers::BpeTrainer.new(special_tokens: ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"])
+tokenizer.train(["wiki.train.raw", "wiki.valid.raw", "wiki.test.raw"], trainer)
+```
+
+Encode
+
+```ruby
+output = tokenizer.encode("Hello, y'all! How are you 😁 ?")
+output.tokens
+```
+
+Save the tokenizer to a file
+
+```ruby
+tokenizer.save("tokenizer.json")
+```
+
+Load a tokenizer from a file
+
+```ruby
+tokenizer = Tokenizers.from_file("tokenizer.json")
+```
+
+Check out the [Quicktour](https://huggingface.co/docs/tokenizers/quicktour) and equivalent [Ruby code](https://github.com/ankane/tokenizers-ruby/blob/master/test/quicktour_test.rb#L8) for more info
+
+## API
 
-Check out the [Quicktour](https://huggingface.co/docs/tokenizers/quicktour) and equivalent [Ruby code](https://github.com/ankane/tokenizers-ruby/blob/master/test/quicktour_test.rb#L8)
+This library follows the [Tokenizers Python API](https://huggingface.co/docs/tokenizers/index). You can follow Python tutorials and convert the code to Ruby in many cases. Feel free to open an issue if you run into problems.
 
 ## History
 

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.14.0"
+    TOKENIZERS_VERSION = "0.15.0"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux
@@ -11,25 +11,27 @@ module Tokenizers
       require "digest"
       require "fileutils"
       require "json"
+      require "net/http"
       require "open-uri"
 
       cache_dir = ensure_cache_dir
 
-      # string options are headers
       options = {
         open_timeout: 3,
-        read_timeout: 30,
+        read_timeout: 30
+      }
+      headers = {
         "User-Agent" => "tokenizers/#{TOKENIZERS_VERSION}; bindings/Ruby; version/#{VERSION}"
       }
       if auth_token
-        options["Authorization"] = "Bearer #{auth_token}"
+        headers["Authorization"] = "Bearer #{auth_token}"
       end
 
       url = "https://huggingface.co/%s/resolve/%s/tokenizer.json" % [identifier, revision].map { |v| CGI.escape(v) }
 
       path =
         begin
-          cached_path(cache_dir, url, options)
+          cached_path(cache_dir, url, headers, options)
         rescue OpenURI::HTTPError
           raise Error, "Model \"#{identifier}\" on the Hub doesn't have a tokenizer"
         end
@@ -41,7 +43,7 @@ module Tokenizers
 
     # use same storage format as Rust version
     # https://github.com/epwalsh/rust-cached-path
-    def cached_path(cache_dir, url, options)
+    def cached_path(cache_dir, url, headers, options)
       fsum = Digest::SHA256.hexdigest(url)
       meta_paths = Dir[File.join(cache_dir, "#{fsum}.*.meta")]
       meta = meta_paths.map { |f| JSON.parse(File.read(f)) }.max_by { |m| m["creation_time"] }
@@ -50,21 +52,25 @@ module Tokenizers
       if etag
         esum = Digest::SHA256.hexdigest(etag)
         resource_path = File.join(cache_dir, "#{fsum}.#{esum}")
-        options["If-None-Match"] = etag if File.exist?(resource_path)
+        if File.exist?(resource_path)
+          uri = URI(url)
+          req = Net::HTTP::Head.new(uri)
+          headers.each do |k, v|
+            req[k] = v
+          end
+          res = Net::HTTP.start(uri.hostname, uri.port, options.merge(use_ssl: true)) do |http|
+            http.request(req)
+          end
+          if res["etag"] == etag
+            return resource_path
+          end
+        end
       end
 
       options[:content_length_proc] = -> (_) { puts "Downloading..." }
 
-      tempfile =
-        begin
-          URI.parse(url).open(options)
-        rescue OpenURI::HTTPError => e
-          if e.message == "304 Not Modified"
-            return resource_path
-          else
-            raise e
-          end
-        end
+      # string options are headers
+      tempfile = URI.parse(url).open(headers.merge(options))
 
       etag = tempfile.meta["etag"]
       esum = Digest::SHA256.hexdigest(etag)

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.4.1"
+  VERSION = "0.4.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.3
 platform: x86_64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-10-05 00:00:00.000000000 Z
+date: 2024-01-04 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.0/tokenizers.bundle
 - lib/tokenizers/3.1/tokenizers.bundle
 - lib/tokenizers/3.2/tokenizers.bundle
+- lib/tokenizers/3.3/tokenizers.bundle
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb
@@ -70,7 +71,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '3.0'
   - - "<"
     - !ruby/object:Gem::Version
-      version: 3.3.dev
+      version: 3.4.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="