tokenizers 0.4.4-x86_64-darwin → 0.5.0-x86_64-darwin

Sign up to get free protection for your applications and to get access to all the features.
Binary file
Binary file
Binary file
@@ -1,8 +1,8 @@
1
1
  module Tokenizers
2
2
  module Decoders
3
3
  class Metaspace
4
- def self.new(replacement: "\u2581", add_prefix_space: true)
5
- _new(replacement, add_prefix_space)
4
+ def self.new(replacement: "\u2581", prepend_scheme: "always", split: true)
5
+ _new(replacement, prepend_scheme, split)
6
6
  end
7
7
  end
8
8
  end
@@ -1,7 +1,7 @@
1
1
  module Tokenizers
2
2
  module FromPretrained
3
3
  # for user agent
4
- TOKENIZERS_VERSION = "0.15.2"
4
+ TOKENIZERS_VERSION = "0.19.1"
5
5
 
6
6
  # use Ruby for downloads
7
7
  # this avoids the need to vendor OpenSSL on Linux
@@ -1,8 +1,8 @@
1
1
  module Tokenizers
2
2
  module PreTokenizers
3
3
  class Metaspace
4
- def self.new(replacement: "\u2581", add_prefix_space: true)
5
- _new(replacement, add_prefix_space)
4
+ def self.new(replacement: "\u2581", prepend_scheme: "always", split: true)
5
+ _new(replacement, prepend_scheme, split)
6
6
  end
7
7
  end
8
8
  end
@@ -1,3 +1,3 @@
1
1
  module Tokenizers
2
- VERSION = "0.4.4"
2
+ VERSION = "0.5.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tokenizers
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.4
4
+ version: 0.5.0
5
5
  platform: x86_64-darwin
6
6
  authors:
7
7
  - Andrew Kane
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2024-02-27 00:00:00.000000000 Z
11
+ date: 2024-05-21 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description:
14
14
  email: andrew@ankane.org
@@ -23,7 +23,6 @@ files:
23
23
  - LICENSE.txt
24
24
  - README.md
25
25
  - lib/tokenizers.rb
26
- - lib/tokenizers/3.0/tokenizers.bundle
27
26
  - lib/tokenizers/3.1/tokenizers.bundle
28
27
  - lib/tokenizers/3.2/tokenizers.bundle
29
28
  - lib/tokenizers/3.3/tokenizers.bundle
@@ -68,7 +67,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
68
67
  requirements:
69
68
  - - ">="
70
69
  - !ruby/object:Gem::Version
71
- version: '3.0'
70
+ version: '3.1'
72
71
  - - "<"
73
72
  - !ruby/object:Gem::Version
74
73
  version: 3.4.dev
Binary file