tiktoken_ruby 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +33 -18
  3. data/Gemfile.lock +1 -1
  4. data/ext/tiktoken_ruby/Cargo.toml +2 -1
  5. data/ext/tiktoken_ruby/src/lib.rs +9 -5
  6. data/lib/tiktoken_ruby/version.rb +1 -1
  7. metadata +2 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ee8bc941d0d8de83fb99c33acf9c831d2bd7542f54726a2ff3ec8acf41fdaab8
4
- data.tar.gz: ca70753b8823328f08123328ad4409b351e99a2d564e0fc8c8e779a4b378a2b3
3
+ metadata.gz: e011f7a82a6ee3d09405cd2c817d2ba9fe06e5a3bdde3ee622c9aefd9b1e8dd8
4
+ data.tar.gz: 892539805e10e5e10456956aa65b72e7ab4ececae56db2c49ccd11848cd0d2c2
5
5
  SHA512:
6
- metadata.gz: '031225799558c1bd563ca6d40b7525f956155e7dabe899c0a5c440eedc29727fb7e8504136251623ca63ad0d2877aadf4673ceb6cda5c440aa8f9362d119d119'
7
- data.tar.gz: 9df34194052e5bdf958d214e6a0d5a9cdda9341401e8226956dc2de05406d22b67a3c7a67fd998af95bf981f1ced34daf8084aeaf9afec0f6e409f416eaa362a
6
+ metadata.gz: 5accbfe97904fb1e9b6c5fa793a9e326bb25befaca6dbf5a2f8e6cfcad5f3c80c7dd1d0f66aaa95bed732943a2c0999654ffa2cf40891b2c9443f8909efb1dc6
7
+ data.tar.gz: 5e95a59d2876fb54e8a76e7d855c42d4e314be211f495dae5ef0bf4004cc44eeb0ea433a620e037232349a10d107ed25afafa704e1fffcddb4ce29219a199e87
data/Cargo.lock CHANGED
@@ -31,11 +31,11 @@ checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
31
31
 
32
32
  [[package]]
33
33
  name = "bindgen"
34
- version = "0.60.1"
34
+ version = "0.66.1"
35
35
  source = "registry+https://github.com/rust-lang/crates.io-index"
36
- checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
36
+ checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7"
37
37
  dependencies = [
38
- "bitflags",
38
+ "bitflags 2.4.0",
39
39
  "cexpr",
40
40
  "clang-sys",
41
41
  "lazy_static",
@@ -46,6 +46,7 @@ dependencies = [
46
46
  "regex",
47
47
  "rustc-hash",
48
48
  "shlex",
49
+ "syn",
49
50
  ]
50
51
 
51
52
  [[package]]
@@ -69,6 +70,12 @@ version = "1.3.2"
69
70
  source = "registry+https://github.com/rust-lang/crates.io-index"
70
71
  checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
71
72
 
73
+ [[package]]
74
+ name = "bitflags"
75
+ version = "2.4.0"
76
+ source = "registry+https://github.com/rust-lang/crates.io-index"
77
+ checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
78
+
72
79
  [[package]]
73
80
  name = "bstr"
74
81
  version = "1.4.0"
@@ -163,20 +170,21 @@ dependencies = [
163
170
 
164
171
  [[package]]
165
172
  name = "magnus"
166
- version = "0.4.4"
173
+ version = "0.6.1"
167
174
  source = "registry+https://github.com/rust-lang/crates.io-index"
168
- checksum = "fc87660cd7daa49fddbfd524c836de54d5c927d520cd163f43700c5087c57d6c"
175
+ checksum = "0516897a45f8ce8270a8910bcb94cd83538b19b6ae3a0c281a765df170b64695"
169
176
  dependencies = [
170
177
  "magnus-macros",
171
178
  "rb-sys",
172
179
  "rb-sys-env",
180
+ "seq-macro",
173
181
  ]
174
182
 
175
183
  [[package]]
176
184
  name = "magnus-macros"
177
- version = "0.3.0"
185
+ version = "0.6.0"
178
186
  source = "registry+https://github.com/rust-lang/crates.io-index"
179
- checksum = "206cb23bfeea05180c97522ef6a3e52a4eb17b0ed2f30ee3ca9c4f994d2378ae"
187
+ checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
180
188
  dependencies = [
181
189
  "proc-macro2",
182
190
  "quote",
@@ -242,36 +250,36 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
242
250
 
243
251
  [[package]]
244
252
  name = "proc-macro2"
245
- version = "1.0.52"
253
+ version = "1.0.66"
246
254
  source = "registry+https://github.com/rust-lang/crates.io-index"
247
- checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
255
+ checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9"
248
256
  dependencies = [
249
257
  "unicode-ident",
250
258
  ]
251
259
 
252
260
  [[package]]
253
261
  name = "quote"
254
- version = "1.0.26"
262
+ version = "1.0.33"
255
263
  source = "registry+https://github.com/rust-lang/crates.io-index"
256
- checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
264
+ checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
257
265
  dependencies = [
258
266
  "proc-macro2",
259
267
  ]
260
268
 
261
269
  [[package]]
262
270
  name = "rb-sys"
263
- version = "0.9.68"
271
+ version = "0.9.81"
264
272
  source = "registry+https://github.com/rust-lang/crates.io-index"
265
- checksum = "4783528e031c3902524cfe685e0008d644e8b34bad4d1a19c1f39f4394d0777b"
273
+ checksum = "a57240b308b155b09dce81e32829966a99f52d1088b45957e4283e526c5317a1"
266
274
  dependencies = [
267
275
  "rb-sys-build",
268
276
  ]
269
277
 
270
278
  [[package]]
271
279
  name = "rb-sys-build"
272
- version = "0.9.68"
280
+ version = "0.9.81"
273
281
  source = "registry+https://github.com/rust-lang/crates.io-index"
274
- checksum = "14c04474335dd597126d20fe90d94504b3582a30cbc36bc72c7d6ed3ef8cf168"
282
+ checksum = "f24ce877a4c5d07f06f6aa6fec3ac95e4b357b9f73b0f5445d8cbb7266d410e8"
275
283
  dependencies = [
276
284
  "bindgen",
277
285
  "lazy_static",
@@ -294,7 +302,7 @@ version = "0.2.16"
294
302
  source = "registry+https://github.com/rust-lang/crates.io-index"
295
303
  checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
296
304
  dependencies = [
297
- "bitflags",
305
+ "bitflags 1.3.2",
298
306
  ]
299
307
 
300
308
  [[package]]
@@ -332,6 +340,12 @@ version = "1.1.0"
332
340
  source = "registry+https://github.com/rust-lang/crates.io-index"
333
341
  checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
334
342
 
343
+ [[package]]
344
+ name = "seq-macro"
345
+ version = "0.3.5"
346
+ source = "registry+https://github.com/rust-lang/crates.io-index"
347
+ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
348
+
335
349
  [[package]]
336
350
  name = "serde"
337
351
  version = "1.0.157"
@@ -358,9 +372,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
358
372
 
359
373
  [[package]]
360
374
  name = "syn"
361
- version = "1.0.109"
375
+ version = "2.0.31"
362
376
  source = "registry+https://github.com/rust-lang/crates.io-index"
363
- checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
377
+ checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398"
364
378
  dependencies = [
365
379
  "proc-macro2",
366
380
  "quote",
@@ -386,6 +400,7 @@ name = "tiktoken_ruby"
386
400
  version = "0.1.0"
387
401
  dependencies = [
388
402
  "magnus",
403
+ "rb-sys",
389
404
  "tiktoken-rs",
390
405
  ]
391
406
 
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- tiktoken_ruby (0.0.5)
4
+ tiktoken_ruby (0.0.6)
5
5
  rb_sys (~> 0.9.68)
6
6
 
7
7
  GEM
data/ext/tiktoken_ruby/Cargo.toml CHANGED
@@ -10,5 +10,6 @@ publish = false
10
10
  crate-type = ["cdylib"]
11
11
 
12
12
  [dependencies]
13
- magnus = { version = "0.4" }
13
+ magnus = { version = "0.6.1" }
14
+ rb-sys = { version = "*", features = ["stable-api-compiled-fallback"] }
14
15
  tiktoken-rs = { git = "https://github.com/IAPark/tiktoken-rs.git" }
data/ext/tiktoken_ruby/src/lib.rs CHANGED
@@ -1,7 +1,7 @@
1
1
  mod core_bpe_wrapper;
2
2
 
3
3
  use core_bpe_wrapper::CoreBPEWrapper;
4
- use magnus::{define_module, function, prelude::*, Error, method, class, RModule, ExceptionClass};
4
+ use magnus::{class, define_module, function, method, prelude::*, Error, ExceptionClass, RModule};
5
5
 
6
6
  fn r50k_base() -> CoreBPEWrapper {
7
7
  let core_bpe = tiktoken_rs::r50k_base().unwrap();
@@ -38,14 +38,18 @@ fn init() -> Result<(), Error> {
38
38
  factory_module.define_singleton_method("p50k_edit", function!(p50k_edit, 0))?;
39
39
  factory_module.define_singleton_method("cl100k_base", function!(cl100k_base, 0))?;
40
40
 
41
-
42
41
  let ext_module = module.define_module("Ext")?;
43
42
  let bpe_class = ext_module.define_class("CoreBPE", class::object())?;
44
43
 
45
- bpe_class.define_method("encode_ordinary", method!(CoreBPEWrapper::encode_ordinary, 1))?;
44
+ bpe_class.define_method(
45
+ "encode_ordinary",
46
+ method!(CoreBPEWrapper::encode_ordinary, 1),
47
+ )?;
46
48
  bpe_class.define_method("encode", method!(CoreBPEWrapper::encode, 2))?;
47
- bpe_class.define_method("encode_with_special_tokens", method!(CoreBPEWrapper::encode_with_special_tokens, 1))?;
48
-
49
+ bpe_class.define_method(
50
+ "encode_with_special_tokens",
51
+ method!(CoreBPEWrapper::encode_with_special_tokens, 1),
52
+ )?;
49
53
 
50
54
  bpe_class.define_method("decode", method!(CoreBPEWrapper::decode, 1))?;
51
55
  Ok(())
data/lib/tiktoken_ruby/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Tiktoken
4
- VERSION = "0.0.5"
4
+ VERSION = "0.0.6"
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tiktoken_ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - IAPark
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-05-11 00:00:00.000000000 Z
11
+ date: 2023-09-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rb_sys