tiktoken_ruby 0.0.10-aarch64-linux → 0.0.11-aarch64-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d592b9e024fad1efe1ffc3068fbf31846151b700226abafbe36c4758e93b226d
4
- data.tar.gz: b343edc7072fe9b3ec76dca6ce6971420540c43b496d7efbd95027671c21cb31
3
+ metadata.gz: 9ee1420c3c32061ec6ed2b0fbe15f458276c2f5f90fc6d461cd86cab80f08d62
4
+ data.tar.gz: 20526861e1daab913ec899b8784efccfaf25eac66dcf61fea424c9473c3dafe7
5
5
  SHA512:
6
- metadata.gz: d61a24f0174d8bd9a2a8851b22aa515feec00145cf2dd473a1f4add3f48248e30284a058d94ff980587e37df6a570c2174a29271b7f760561e9cbc93f325833c
7
- data.tar.gz: 3b477a5a3b47f8735e723e12439f76456b24c36f3b8dd6238454f467f983919d7b89f34dd578b36b29c8f7e253bfde6a3cdcfdef803a6f85b0ed35a3b3c57d09
6
+ metadata.gz: 31cc8b8c49edd0932dd11f5ceb5bdc745849a35003aef3556ee961e90061f6d9ce94a1ebbd346191dd2bf53ff32c25abc3c5104c14cad469e69236bef802c9e4
7
+ data.tar.gz: 7543b36b0a14a8aa8d8f73bb1c2ba7df85427e1d45c7baa93fa48fd592516e9e81434b7d8aba8722ace58e1ebb3c9bd9654dfdbdc460d1f0abb995789c8b5f29
data/Gemfile.lock CHANGED
@@ -1,73 +1,73 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- tiktoken_ruby (0.0.10)
5
- rb_sys (>= 0.9.87)
4
+ tiktoken_ruby (0.0.11)
5
+ rb_sys (= 0.9.106)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
10
  ast (2.4.2)
11
- diff-lcs (1.5.0)
12
- json (2.7.1)
11
+ diff-lcs (1.5.1)
12
+ json (2.9.1)
13
13
  language_server-protocol (3.17.0.3)
14
14
  lint_roller (1.1.0)
15
15
  minitest (5.21.2)
16
- parallel (1.24.0)
17
- parser (3.3.0.4)
16
+ parallel (1.26.3)
17
+ parser (3.3.6.0)
18
18
  ast (~> 2.4.1)
19
19
  racc
20
- racc (1.7.3)
20
+ racc (1.8.1)
21
21
  rainbow (3.1.1)
22
- rake (13.1.0)
23
- rake-compiler (1.2.5)
22
+ rake (13.2.1)
23
+ rake-compiler (1.2.9)
24
24
  rake
25
- rb_sys (0.9.105)
26
- regexp_parser (2.9.0)
27
- rexml (3.2.6)
28
- rspec (3.12.0)
29
- rspec-core (~> 3.12.0)
30
- rspec-expectations (~> 3.12.0)
31
- rspec-mocks (~> 3.12.0)
32
- rspec-core (3.12.2)
33
- rspec-support (~> 3.12.0)
34
- rspec-expectations (3.12.3)
25
+ rb_sys (0.9.106)
26
+ regexp_parser (2.10.0)
27
+ rspec (3.13.0)
28
+ rspec-core (~> 3.13.0)
29
+ rspec-expectations (~> 3.13.0)
30
+ rspec-mocks (~> 3.13.0)
31
+ rspec-core (3.13.2)
32
+ rspec-support (~> 3.13.0)
33
+ rspec-expectations (3.13.3)
35
34
  diff-lcs (>= 1.2.0, < 2.0)
36
- rspec-support (~> 3.12.0)
37
- rspec-mocks (3.12.6)
35
+ rspec-support (~> 3.13.0)
36
+ rspec-mocks (3.13.2)
38
37
  diff-lcs (>= 1.2.0, < 2.0)
39
- rspec-support (~> 3.12.0)
40
- rspec-support (3.12.1)
41
- rubocop (1.59.0)
38
+ rspec-support (~> 3.13.0)
39
+ rspec-support (3.13.2)
40
+ rubocop (1.69.2)
42
41
  json (~> 2.3)
43
42
  language_server-protocol (>= 3.17.0)
44
43
  parallel (~> 1.10)
45
- parser (>= 3.2.2.4)
44
+ parser (>= 3.3.0.2)
46
45
  rainbow (>= 2.2.2, < 4.0)
47
- regexp_parser (>= 1.8, < 3.0)
48
- rexml (>= 3.2.5, < 4.0)
49
- rubocop-ast (>= 1.30.0, < 2.0)
46
+ regexp_parser (>= 2.9.3, < 3.0)
47
+ rubocop-ast (>= 1.36.2, < 2.0)
50
48
  ruby-progressbar (~> 1.7)
51
- unicode-display_width (>= 2.4.0, < 3.0)
52
- rubocop-ast (1.30.0)
53
- parser (>= 3.2.1.0)
54
- rubocop-performance (1.20.2)
49
+ unicode-display_width (>= 2.4.0, < 4.0)
50
+ rubocop-ast (1.37.0)
51
+ parser (>= 3.3.1.0)
52
+ rubocop-performance (1.23.0)
55
53
  rubocop (>= 1.48.1, < 2.0)
56
- rubocop-ast (>= 1.30.0, < 2.0)
54
+ rubocop-ast (>= 1.31.1, < 2.0)
57
55
  ruby-progressbar (1.13.0)
58
- standard (1.33.0)
56
+ standard (1.43.0)
59
57
  language_server-protocol (~> 3.17.0.2)
60
58
  lint_roller (~> 1.0)
61
- rubocop (~> 1.59.0)
59
+ rubocop (~> 1.69.1)
62
60
  standard-custom (~> 1.0.0)
63
- standard-performance (~> 1.3)
61
+ standard-performance (~> 1.6)
64
62
  standard-custom (1.0.2)
65
63
  lint_roller (~> 1.0)
66
64
  rubocop (~> 1.50)
67
- standard-performance (1.3.1)
65
+ standard-performance (1.6.0)
68
66
  lint_roller (~> 1.1)
69
- rubocop-performance (~> 1.20.2)
70
- unicode-display_width (2.5.0)
67
+ rubocop-performance (~> 1.23.0)
68
+ unicode-display_width (3.1.3)
69
+ unicode-emoji (~> 4.0, >= 4.0.4)
70
+ unicode-emoji (4.0.4)
71
71
  yard (0.9.34)
72
72
  yard-doctest (0.1.17)
73
73
  minitest
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Tiktoken
4
- VERSION = "0.0.10"
4
+ VERSION = "0.0.11"
5
5
  end
data/lib/tiktoken_ruby.rb CHANGED
@@ -69,9 +69,12 @@ module Tiktoken
69
69
  ]
70
70
 
71
71
  # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
72
- # that is also MIT licensed but by OpenAI
72
+ # that is also MIT licensed but by OpenAI;
73
+ # https://github.com/Congyuwang/tiktoken-rs/blob/main/tiktoken-rs/src/tokenizer.rs#L50
74
+ # is the source of the mapping for the Rust library
73
75
  MODEL_TO_ENCODING_NAME = {
74
76
  # chat
77
+ "chatgpt-4o-latest": "o200k_base",
75
78
  "gpt-4o": "o200k_base",
76
79
  "gpt-4": "cl100k_base",
77
80
  "gpt-3.5-turbo": "cl100k_base",
data/script/release CHANGED
@@ -7,27 +7,37 @@ if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
7
7
  exit 1
8
8
  fi
9
9
 
10
- version=$(grep VERSION lib/tiktoken_ruby/version.rb | head -n 1 | cut -d'"' -f2)
11
- echo "Building tiktoken_ruby v$version"
12
-
13
- targets=(
14
- "arm64-darwin"
15
- "x86_64-darwin"
16
- "aarch64-linux"
17
- "x86_64-linux"
18
- "x86_64-linux-musl"
19
- "arm-linux"
20
- "x64-mingw-ucrt"
21
- )
22
-
23
- for target in "${targets[@]}"; do
24
- bundle exec rb-sys-dock -p "$target" --ruby-versions 3.2 --build
10
+ run_id=""
11
+ # Parse arguments
12
+ while [[ "$#" -gt 0 ]]; do
13
+ case $1 in
14
+ --run-id)
15
+ run_id="$2"
16
+ shift 2
17
+ ;;
18
+ *)
19
+ echo "Unknown parameter passed: $1"
20
+ exit 1
21
+ ;;
22
+ esac
25
23
  done
26
24
 
27
- for gem in pkg/tiktoken_ruby-"$version"-*.gem ; do
28
- GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
25
+ if [ -z "${run_id}" ]; then
26
+ echo "Error: --run-id is not provided. Please provide the GitHub Action run id for the cross-compile workflow."
27
+ exit 1
28
+ fi
29
+
30
+ version=$(grep VERSION lib/tiktoken_ruby/version.rb | head -n 1 | cut -d'"' -f2)
31
+ echo "Building tiktoken_ruby v$version, using artifacts from run $run_id"
32
+
33
+ rm -rf pkg/cross-compiled
34
+ gh run download "$run_id" -D pkg/cross-compiled
35
+
36
+ for gem in pkg/cross-compiled/cross-gem-*/tiktoken_ruby-"$version"*.gem ; do
37
+ echo "Publishing $gem"
38
+ GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
29
39
  done
30
40
 
31
- # last but not least
41
+ # last but not least, the uncompiled gem
32
42
  bundle exec rake package
33
43
  GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tiktoken_ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.10
4
+ version: 0.0.11
5
5
  platform: aarch64-linux
6
6
  authors:
7
7
  - IAPark
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-12-29 00:00:00.000000000 Z
11
+ date: 2025-01-08 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
14
14
  used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -29,8 +29,10 @@ files:
29
29
  - Rakefile
30
30
  - doctest_helper.rb
31
31
  - lib/tiktoken_ruby.rb
32
+ - lib/tiktoken_ruby/3.1/tiktoken_ruby.so
33
+ - lib/tiktoken_ruby/3.2/tiktoken_ruby.so
34
+ - lib/tiktoken_ruby/3.4/tiktoken_ruby.so
32
35
  - lib/tiktoken_ruby/encoding.rb
33
- - lib/tiktoken_ruby/tiktoken_ruby.so
34
36
  - lib/tiktoken_ruby/version.rb
35
37
  - script/release
36
38
  - sig/tiktoken_ruby.rbs
@@ -49,10 +51,10 @@ required_ruby_version: !ruby/object:Gem::Requirement
49
51
  requirements:
50
52
  - - ">="
51
53
  - !ruby/object:Gem::Version
52
- version: '3.2'
54
+ version: '3.1'
53
55
  - - "<"
54
56
  - !ruby/object:Gem::Version
55
- version: 3.3.dev
57
+ version: 3.5.dev
56
58
  required_rubygems_version: !ruby/object:Gem::Requirement
57
59
  requirements:
58
60
  - - ">="