tiktoken_ruby 0.0.10-x86_64-darwin → 0.0.11-x86_64-darwin

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: afa44352dbb69dae808847a17d8673ab148d9e2f70ef7a300d06fb5492978449
4
- data.tar.gz: 6207a6ef03a0de50acbbd8c48cd6e1bc8d757ccaffa26c0353ff6a2bd7dd9c22
3
+ metadata.gz: 6b6edbe160f475789f78974f7e5af957737f96338cc08d13abeb4fbbfef5b862
4
+ data.tar.gz: f1ad7a7ca6f49ae578146097cb6671cdc0d751490a622550c3101bde2352dfbf
5
5
  SHA512:
6
- metadata.gz: 6b8db3409bf361913490dd1181bd62f84af99080dbac8ecbdb3fdfea57ef8d0327fa75e9c0621607bcbb12ebccdaaf880722091667d696cd64ef46ccc58ab993
7
- data.tar.gz: 60d5b5eb85de133c3a2650b918a8b6e0e9365eb9c6d9771543bab663ce7c65489b4db1fabebea17664672d73f45a4367f28cd896b07d98d8f1d9dd6331857527
6
+ metadata.gz: 0043ce848e9514b4f1702514dc5d3047790beba55522fc9b00add2473ff93029ef5aa54ff33ffc958944c2f5423d42e30227bb661144ca62f0d49f18f61ffeda
7
+ data.tar.gz: c965158002ffe7a5f1a0fa28c8dba71d14b902e2e1a80dd5283420027ea9c2a5530ac45b7f83c887b7a50d9c6a30167394f668b751f1cbd783699d20ca1ae534
data/Gemfile.lock CHANGED
@@ -1,73 +1,73 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- tiktoken_ruby (0.0.10)
5
- rb_sys (>= 0.9.87)
4
+ tiktoken_ruby (0.0.11)
5
+ rb_sys (= 0.9.106)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
10
  ast (2.4.2)
11
- diff-lcs (1.5.0)
12
- json (2.7.1)
11
+ diff-lcs (1.5.1)
12
+ json (2.9.1)
13
13
  language_server-protocol (3.17.0.3)
14
14
  lint_roller (1.1.0)
15
15
  minitest (5.21.2)
16
- parallel (1.24.0)
17
- parser (3.3.0.4)
16
+ parallel (1.26.3)
17
+ parser (3.3.6.0)
18
18
  ast (~> 2.4.1)
19
19
  racc
20
- racc (1.7.3)
20
+ racc (1.8.1)
21
21
  rainbow (3.1.1)
22
- rake (13.1.0)
23
- rake-compiler (1.2.5)
22
+ rake (13.2.1)
23
+ rake-compiler (1.2.9)
24
24
  rake
25
- rb_sys (0.9.105)
26
- regexp_parser (2.9.0)
27
- rexml (3.2.6)
28
- rspec (3.12.0)
29
- rspec-core (~> 3.12.0)
30
- rspec-expectations (~> 3.12.0)
31
- rspec-mocks (~> 3.12.0)
32
- rspec-core (3.12.2)
33
- rspec-support (~> 3.12.0)
34
- rspec-expectations (3.12.3)
25
+ rb_sys (0.9.106)
26
+ regexp_parser (2.10.0)
27
+ rspec (3.13.0)
28
+ rspec-core (~> 3.13.0)
29
+ rspec-expectations (~> 3.13.0)
30
+ rspec-mocks (~> 3.13.0)
31
+ rspec-core (3.13.2)
32
+ rspec-support (~> 3.13.0)
33
+ rspec-expectations (3.13.3)
35
34
  diff-lcs (>= 1.2.0, < 2.0)
36
- rspec-support (~> 3.12.0)
37
- rspec-mocks (3.12.6)
35
+ rspec-support (~> 3.13.0)
36
+ rspec-mocks (3.13.2)
38
37
  diff-lcs (>= 1.2.0, < 2.0)
39
- rspec-support (~> 3.12.0)
40
- rspec-support (3.12.1)
41
- rubocop (1.59.0)
38
+ rspec-support (~> 3.13.0)
39
+ rspec-support (3.13.2)
40
+ rubocop (1.69.2)
42
41
  json (~> 2.3)
43
42
  language_server-protocol (>= 3.17.0)
44
43
  parallel (~> 1.10)
45
- parser (>= 3.2.2.4)
44
+ parser (>= 3.3.0.2)
46
45
  rainbow (>= 2.2.2, < 4.0)
47
- regexp_parser (>= 1.8, < 3.0)
48
- rexml (>= 3.2.5, < 4.0)
49
- rubocop-ast (>= 1.30.0, < 2.0)
46
+ regexp_parser (>= 2.9.3, < 3.0)
47
+ rubocop-ast (>= 1.36.2, < 2.0)
50
48
  ruby-progressbar (~> 1.7)
51
- unicode-display_width (>= 2.4.0, < 3.0)
52
- rubocop-ast (1.30.0)
53
- parser (>= 3.2.1.0)
54
- rubocop-performance (1.20.2)
49
+ unicode-display_width (>= 2.4.0, < 4.0)
50
+ rubocop-ast (1.37.0)
51
+ parser (>= 3.3.1.0)
52
+ rubocop-performance (1.23.0)
55
53
  rubocop (>= 1.48.1, < 2.0)
56
- rubocop-ast (>= 1.30.0, < 2.0)
54
+ rubocop-ast (>= 1.31.1, < 2.0)
57
55
  ruby-progressbar (1.13.0)
58
- standard (1.33.0)
56
+ standard (1.43.0)
59
57
  language_server-protocol (~> 3.17.0.2)
60
58
  lint_roller (~> 1.0)
61
- rubocop (~> 1.59.0)
59
+ rubocop (~> 1.69.1)
62
60
  standard-custom (~> 1.0.0)
63
- standard-performance (~> 1.3)
61
+ standard-performance (~> 1.6)
64
62
  standard-custom (1.0.2)
65
63
  lint_roller (~> 1.0)
66
64
  rubocop (~> 1.50)
67
- standard-performance (1.3.1)
65
+ standard-performance (1.6.0)
68
66
  lint_roller (~> 1.1)
69
- rubocop-performance (~> 1.20.2)
70
- unicode-display_width (2.5.0)
67
+ rubocop-performance (~> 1.23.0)
68
+ unicode-display_width (3.1.3)
69
+ unicode-emoji (~> 4.0, >= 4.0.4)
70
+ unicode-emoji (4.0.4)
71
71
  yard (0.9.34)
72
72
  yard-doctest (0.1.17)
73
73
  minitest
data/Rakefile CHANGED
@@ -21,3 +21,9 @@ end
21
21
  task build: :compile
22
22
 
23
23
  task default: %i[compile spec standard]
24
+
25
+ # Packaging default (non-precompiled) gem
26
+ require "rubygems/package_task"
27
+ gem_path = Gem::PackageTask.new(GEMSPEC).define
28
+ desc "Package the Ruby gem"
29
+ task "package" => [gem_path]
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Tiktoken
4
- VERSION = "0.0.10"
4
+ VERSION = "0.0.11"
5
5
  end
data/lib/tiktoken_ruby.rb CHANGED
@@ -69,9 +69,12 @@ module Tiktoken
69
69
  ]
70
70
 
71
71
  # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
72
- # that is also MIT licensed but by OpenAI
72
+ # that is also MIT licensed but by OpenAI;
73
+ # https://github.com/Congyuwang/tiktoken-rs/blob/main/tiktoken-rs/src/tokenizer.rs#L50
74
+ # is the source of the mapping for the Rust library
73
75
  MODEL_TO_ENCODING_NAME = {
74
76
  # chat
77
+ "chatgpt-4o-latest": "o200k_base",
75
78
  "gpt-4o": "o200k_base",
76
79
  "gpt-4": "cl100k_base",
77
80
  "gpt-3.5-turbo": "cl100k_base",
data/script/release CHANGED
@@ -7,23 +7,37 @@ if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
7
7
  exit 1
8
8
  fi
9
9
 
10
- version=$(grep VERSION lib/tiktoken_ruby/version.rb | head -n 1 | cut -d'"' -f2)
11
- echo "Building tiktoken_ruby v$version"
12
-
13
- targets=(
14
- "arm64-darwin"
15
- "x86_64-darwin"
16
- "aarch64-linux"
17
- "x86_64-linux"
18
- "x86_64-linux-musl"
19
- "arm-linux"
20
- "x64-mingw-ucrt"
21
- )
22
-
23
- for target in "${targets[@]}"; do
24
- bundle exec rb-sys-dock -p "$target" --ruby-versions 3.2 --build
10
+ run_id=""
11
+ # Parse arguments
12
+ while [[ "$#" -gt 0 ]]; do
13
+ case $1 in
14
+ --run-id)
15
+ run_id="$2"
16
+ shift 2
17
+ ;;
18
+ *)
19
+ echo "Unknown parameter passed: $1"
20
+ exit 1
21
+ ;;
22
+ esac
25
23
  done
26
24
 
27
- for gem in pkg/tiktoken_ruby-"$version"-*.gem ; do
28
- GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
25
+ if [ -z "${run_id}" ]; then
26
+ echo "Error: --run-id is not provided. Please provide the GitHub Action run id for the cross-compile workflow."
27
+ exit 1
28
+ fi
29
+
30
+ version=$(grep VERSION lib/tiktoken_ruby/version.rb | head -n 1 | cut -d'"' -f2)
31
+ echo "Building tiktoken_ruby v$version, using artifacts from run $run_id"
32
+
33
+ rm -rf pkg/cross-compiled
34
+ gh run download "$run_id" -D pkg/cross-compiled
35
+
36
+ for gem in pkg/cross-compiled/cross-gem-*/tiktoken_ruby-"$version"*.gem ; do
37
+ echo "Publishing $gem"
38
+ GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
29
39
  done
40
+
41
+ # last but not least, the uncompiled gem
42
+ bundle exec rake package
43
+ GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tiktoken_ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: x86_64-darwin
6
6
  authors:
7
7
  - IAPark
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-12-29 00:00:00.000000000 Z
11
+ date: 2025-01-08 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
14
14
  used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -29,8 +29,10 @@ files:
29
29
  - Rakefile
30
30
  - doctest_helper.rb
31
31
  - lib/tiktoken_ruby.rb
32
+ - lib/tiktoken_ruby/3.1/tiktoken_ruby.bundle
33
+ - lib/tiktoken_ruby/3.2/tiktoken_ruby.bundle
34
+ - lib/tiktoken_ruby/3.4/tiktoken_ruby.bundle
32
35
  - lib/tiktoken_ruby/encoding.rb
33
- - lib/tiktoken_ruby/tiktoken_ruby.bundle
34
36
  - lib/tiktoken_ruby/version.rb
35
37
  - script/release
36
38
  - sig/tiktoken_ruby.rbs
@@ -49,10 +51,10 @@ required_ruby_version: !ruby/object:Gem::Requirement
49
51
  requirements:
50
52
  - - ">="
51
53
  - !ruby/object:Gem::Version
52
- version: '3.2'
54
+ version: '3.1'
53
55
  - - "<"
54
56
  - !ruby/object:Gem::Version
55
- version: 3.3.dev
57
+ version: 3.5.dev
56
58
  required_rubygems_version: !ruby/object:Gem::Requirement
57
59
  requirements:
58
60
  - - ">="