tiktoken_ruby 0.0.9-arm-linux → 0.0.11.1-arm-linux

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: da4ac893803fed2e1f7feac48216d4f6b5e26a2b1f82ac390140e5b6ad713da9
4
- data.tar.gz: 75ead5e389c6f94477cb010d1e128985f27419ca544909ab3bec2ae125c41a45
3
+ metadata.gz: 3077759be498ef740196b12c300d1111f9a38a7c9f0d7d63d7a6b6e785a789ea
4
+ data.tar.gz: 6441e04eeb2c638dc663cc456d1db9decfb56ef14e143adb0731b4c39369b8b0
5
5
  SHA512:
6
- metadata.gz: e7c804a4af5e895f86d5ef485768f6a94f8411447e0015f7f796cbd83b1ccb18f2fa2a58a627dc000a8c01724c57ad4ef8f00524e39e99f245d3f934027c1051
7
- data.tar.gz: a498b685977f163fe6892fac2d429997ba76659df18c829fa688ea60258d5648c7bb98a86f2ab42959e9d9c2914dcaec143bfcc06dbf79ad0f38756b25259ddb
6
+ metadata.gz: b065e21826de85f30c825aa59535ed04d16254b82d8c3fff1344cbfba7a4619974b025444b1f14bc10e470b09629f0db416b1f04ae33ab244e9feae6a7c1e847
7
+ data.tar.gz: de67e9dae497859fec83f691375dfac1872ad395eb5188ea1595f99a3a9f0d1bd468d4e9f500544df1415bbfa9bd6124b33333bf477c4f489f6c9dc7b3dcaba5
@@ -0,0 +1,5 @@
1
+ {
2
+ "[ruby]": {
3
+ "editor.defaultFormatter": "Shopify.ruby-lsp"
4
+ }
5
+ }
data/Gemfile.lock CHANGED
@@ -1,73 +1,73 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- tiktoken_ruby (0.0.9)
5
- rb_sys (= 0.9.87)
4
+ tiktoken_ruby (0.0.11.1)
5
+ rb_sys (= 0.9.106)
6
6
 
7
7
  GEM
8
8
  remote: https://rubygems.org/
9
9
  specs:
10
10
  ast (2.4.2)
11
- diff-lcs (1.5.0)
12
- json (2.7.1)
11
+ diff-lcs (1.5.1)
12
+ json (2.9.1)
13
13
  language_server-protocol (3.17.0.3)
14
14
  lint_roller (1.1.0)
15
15
  minitest (5.21.2)
16
- parallel (1.24.0)
17
- parser (3.3.0.4)
16
+ parallel (1.26.3)
17
+ parser (3.3.6.0)
18
18
  ast (~> 2.4.1)
19
19
  racc
20
- racc (1.7.3)
20
+ racc (1.8.1)
21
21
  rainbow (3.1.1)
22
- rake (13.1.0)
23
- rake-compiler (1.2.5)
22
+ rake (13.2.1)
23
+ rake-compiler (1.2.9)
24
24
  rake
25
- rb_sys (0.9.87)
26
- regexp_parser (2.9.0)
27
- rexml (3.2.6)
28
- rspec (3.12.0)
29
- rspec-core (~> 3.12.0)
30
- rspec-expectations (~> 3.12.0)
31
- rspec-mocks (~> 3.12.0)
32
- rspec-core (3.12.2)
33
- rspec-support (~> 3.12.0)
34
- rspec-expectations (3.12.3)
25
+ rb_sys (0.9.106)
26
+ regexp_parser (2.10.0)
27
+ rspec (3.13.0)
28
+ rspec-core (~> 3.13.0)
29
+ rspec-expectations (~> 3.13.0)
30
+ rspec-mocks (~> 3.13.0)
31
+ rspec-core (3.13.2)
32
+ rspec-support (~> 3.13.0)
33
+ rspec-expectations (3.13.3)
35
34
  diff-lcs (>= 1.2.0, < 2.0)
36
- rspec-support (~> 3.12.0)
37
- rspec-mocks (3.12.6)
35
+ rspec-support (~> 3.13.0)
36
+ rspec-mocks (3.13.2)
38
37
  diff-lcs (>= 1.2.0, < 2.0)
39
- rspec-support (~> 3.12.0)
40
- rspec-support (3.12.1)
41
- rubocop (1.59.0)
38
+ rspec-support (~> 3.13.0)
39
+ rspec-support (3.13.2)
40
+ rubocop (1.69.2)
42
41
  json (~> 2.3)
43
42
  language_server-protocol (>= 3.17.0)
44
43
  parallel (~> 1.10)
45
- parser (>= 3.2.2.4)
44
+ parser (>= 3.3.0.2)
46
45
  rainbow (>= 2.2.2, < 4.0)
47
- regexp_parser (>= 1.8, < 3.0)
48
- rexml (>= 3.2.5, < 4.0)
49
- rubocop-ast (>= 1.30.0, < 2.0)
46
+ regexp_parser (>= 2.9.3, < 3.0)
47
+ rubocop-ast (>= 1.36.2, < 2.0)
50
48
  ruby-progressbar (~> 1.7)
51
- unicode-display_width (>= 2.4.0, < 3.0)
52
- rubocop-ast (1.30.0)
53
- parser (>= 3.2.1.0)
54
- rubocop-performance (1.20.2)
49
+ unicode-display_width (>= 2.4.0, < 4.0)
50
+ rubocop-ast (1.37.0)
51
+ parser (>= 3.3.1.0)
52
+ rubocop-performance (1.23.0)
55
53
  rubocop (>= 1.48.1, < 2.0)
56
- rubocop-ast (>= 1.30.0, < 2.0)
54
+ rubocop-ast (>= 1.31.1, < 2.0)
57
55
  ruby-progressbar (1.13.0)
58
- standard (1.33.0)
56
+ standard (1.43.0)
59
57
  language_server-protocol (~> 3.17.0.2)
60
58
  lint_roller (~> 1.0)
61
- rubocop (~> 1.59.0)
59
+ rubocop (~> 1.69.1)
62
60
  standard-custom (~> 1.0.0)
63
- standard-performance (~> 1.3)
61
+ standard-performance (~> 1.6)
64
62
  standard-custom (1.0.2)
65
63
  lint_roller (~> 1.0)
66
64
  rubocop (~> 1.50)
67
- standard-performance (1.3.1)
65
+ standard-performance (1.6.0)
68
66
  lint_roller (~> 1.1)
69
- rubocop-performance (~> 1.20.2)
70
- unicode-display_width (2.5.0)
67
+ rubocop-performance (~> 1.23.0)
68
+ unicode-display_width (3.1.3)
69
+ unicode-emoji (~> 4.0, >= 4.0.4)
70
+ unicode-emoji (4.0.4)
71
71
  yard (0.9.34)
72
72
  yard-doctest (0.1.17)
73
73
  minitest
@@ -89,4 +89,4 @@ DEPENDENCIES
89
89
  yard-doctest
90
90
 
91
91
  BUNDLED WITH
92
- 2.4.4
92
+ 2.5.18
data/Rakefile CHANGED
@@ -21,3 +21,9 @@ end
21
21
  task build: :compile
22
22
 
23
23
  task default: %i[compile spec standard]
24
+
25
+ # Packaging default (non-precompiled) gem
26
+ require "rubygems/package_task"
27
+ gem_path = Gem::PackageTask.new(GEMSPEC).define
28
+ desc "Package the Ruby gem"
29
+ task "package" => [gem_path]
Binary file
Binary file
Binary file
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Tiktoken
4
- VERSION = "0.0.9"
4
+ VERSION = "0.0.11.1"
5
5
  end
data/lib/tiktoken_ruby.rb CHANGED
@@ -69,9 +69,12 @@ module Tiktoken
69
69
  ]
70
70
 
71
71
  # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
72
- # that is also MIT licensed but by OpenAI
72
+ # that is also MIT licensed but by OpenAI;
73
+ # https://github.com/Congyuwang/tiktoken-rs/blob/main/tiktoken-rs/src/tokenizer.rs#L50
74
+ # is the source of the mapping for the Rust library
73
75
  MODEL_TO_ENCODING_NAME = {
74
76
  # chat
77
+ "chatgpt-4o-latest": "o200k_base",
75
78
  "gpt-4o": "o200k_base",
76
79
  "gpt-4": "cl100k_base",
77
80
  "gpt-3.5-turbo": "cl100k_base",
data/script/release ADDED
@@ -0,0 +1,43 @@
1
+ #!/usr/bin/env bash
2
+
3
+ set -e
4
+
5
+ if [ -z "${TIKTOKEN_PUBLISH_KEY}" ]; then
6
+ echo "Error: TIKTOKEN_PUBLISH_KEY is not set. This is the RubyGems API key to push the gem."
7
+ exit 1
8
+ fi
9
+
10
+ run_id=""
11
+ # Parse arguments
12
+ while [[ "$#" -gt 0 ]]; do
13
+ case $1 in
14
+ --run-id)
15
+ run_id="$2"
16
+ shift 2
17
+ ;;
18
+ *)
19
+ echo "Unknown parameter passed: $1"
20
+ exit 1
21
+ ;;
22
+ esac
23
+ done
24
+
25
+ if [ -z "${run_id}" ]; then
26
+ echo "Error: --run-id is not provided. Please provide the GitHub Action run id for the cross-compile workflow."
27
+ exit 1
28
+ fi
29
+
30
+ version=$(grep VERSION lib/tiktoken_ruby/version.rb | head -n 1 | cut -d'"' -f2)
31
+ echo "Building tiktoken_ruby v$version, using artifacts from run $run_id"
32
+
33
+ rm -rf pkg/cross-compiled
34
+ gh run download "$run_id" -D pkg/cross-compiled
35
+
36
+ for gem in pkg/cross-compiled/cross-gem-*/tiktoken_ruby-"$version"*.gem ; do
37
+ echo "Publishing $gem"
38
+ GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "$gem" --host https://rubygems.org
39
+ done
40
+
41
+ # last but not least, the uncompiled gem
42
+ bundle exec rake package
43
+ GEM_HOST_API_KEY="${TIKTOKEN_PUBLISH_KEY}" gem push "pkg/tiktoken_ruby-$version.gem" --host https://rubygems.org
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: tiktoken_ruby
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.9
4
+ version: 0.0.11.1
5
5
  platform: arm-linux
6
6
  authors:
7
7
  - IAPark
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2024-05-16 00:00:00.000000000 Z
11
+ date: 2025-01-08 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: An unofficial Ruby wrapper for Tiktoken, a BPE tokenizer written by and
14
14
  used by OpenAI. It can be used to count the number of tokens in text before sending
@@ -21,6 +21,7 @@ extra_rdoc_files: []
21
21
  files:
22
22
  - ".rspec"
23
23
  - ".standard.yml"
24
+ - ".vscode/settings.json"
24
25
  - Gemfile
25
26
  - Gemfile.lock
26
27
  - LICENSE.txt
@@ -31,8 +32,10 @@ files:
31
32
  - lib/tiktoken_ruby/3.1/tiktoken_ruby.so
32
33
  - lib/tiktoken_ruby/3.2/tiktoken_ruby.so
33
34
  - lib/tiktoken_ruby/3.3/tiktoken_ruby.so
35
+ - lib/tiktoken_ruby/3.4/tiktoken_ruby.so
34
36
  - lib/tiktoken_ruby/encoding.rb
35
37
  - lib/tiktoken_ruby/version.rb
38
+ - script/release
36
39
  - sig/tiktoken_ruby.rbs
37
40
  homepage: https://github.com/IAPark/tiktoken_ruby
38
41
  licenses:
@@ -52,14 +55,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
52
55
  version: '3.1'
53
56
  - - "<"
54
57
  - !ruby/object:Gem::Version
55
- version: 3.4.dev
58
+ version: 3.5.dev
56
59
  required_rubygems_version: !ruby/object:Gem::Requirement
57
60
  requirements:
58
61
  - - ">="
59
62
  - !ruby/object:Gem::Version
60
- version: 3.1.0
63
+ version: 3.4.0
61
64
  requirements: []
62
- rubygems_version: 3.4.4
65
+ rubygems_version: 3.5.23
63
66
  signing_key:
64
67
  specification_version: 4
65
68
  summary: Ruby wrapper for Tiktoken