riktoken 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. checksums.yaml +7 -0
  2. data/.rubocop.yml +10 -0
  3. data/CHANGELOG.md +5 -0
  4. data/CODE_OF_CONDUCT.md +132 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +205 -0
  7. data/Rakefile +24 -0
  8. data/lib/riktoken/bpe.rb +139 -0
  9. data/lib/riktoken/encoding.rb +52 -0
  10. data/lib/riktoken/encodings/cl100k_base.rb +35 -0
  11. data/lib/riktoken/encodings/o200k_base.rb +40 -0
  12. data/lib/riktoken/encodings/p50k_base.rb +31 -0
  13. data/lib/riktoken/encodings/p50k_edit.rb +35 -0
  14. data/lib/riktoken/encodings/r50k_base.rb +31 -0
  15. data/lib/riktoken/encodings.rb +28 -0
  16. data/lib/riktoken/tiktoken_file.rb +42 -0
  17. data/lib/riktoken/version.rb +5 -0
  18. data/lib/riktoken.rb +155 -0
  19. data/renovate.json +6 -0
  20. data/sig/generated/riktoken/bpe.rbs +55 -0
  21. data/sig/generated/riktoken/encoding.rbs +34 -0
  22. data/sig/generated/riktoken/encodings/cl100k_base.rbs +15 -0
  23. data/sig/generated/riktoken/encodings/o200k_base.rbs +15 -0
  24. data/sig/generated/riktoken/encodings/p50k_base.rbs +15 -0
  25. data/sig/generated/riktoken/encodings/p50k_edit.rbs +17 -0
  26. data/sig/generated/riktoken/encodings/r50k_base.rbs +15 -0
  27. data/sig/generated/riktoken/encodings.rbs +18 -0
  28. data/sig/generated/riktoken/tiktoken_file.rbs +15 -0
  29. data/sig/generated/riktoken/version.rbs +5 -0
  30. data/sig/generated/riktoken.rbs +55 -0
  31. metadata +87 -0
metadata ADDED
@@ -0,0 +1,87 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: riktoken
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - moznion
8
+ bindir: exe
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: base64
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '0'
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - ">="
24
+ - !ruby/object:Gem::Version
25
+ version: '0'
26
+ description: A pure Ruby implementation of OpenAI's tiktoken library for BPE tokenization
27
+ email:
28
+ - moznion@mail.moznion.net
29
+ executables: []
30
+ extensions: []
31
+ extra_rdoc_files: []
32
+ files:
33
+ - ".rubocop.yml"
34
+ - CHANGELOG.md
35
+ - CODE_OF_CONDUCT.md
36
+ - LICENSE.txt
37
+ - README.md
38
+ - Rakefile
39
+ - lib/riktoken.rb
40
+ - lib/riktoken/bpe.rb
41
+ - lib/riktoken/encoding.rb
42
+ - lib/riktoken/encodings.rb
43
+ - lib/riktoken/encodings/cl100k_base.rb
44
+ - lib/riktoken/encodings/o200k_base.rb
45
+ - lib/riktoken/encodings/p50k_base.rb
46
+ - lib/riktoken/encodings/p50k_edit.rb
47
+ - lib/riktoken/encodings/r50k_base.rb
48
+ - lib/riktoken/tiktoken_file.rb
49
+ - lib/riktoken/version.rb
50
+ - renovate.json
51
+ - sig/generated/riktoken.rbs
52
+ - sig/generated/riktoken/bpe.rbs
53
+ - sig/generated/riktoken/encoding.rbs
54
+ - sig/generated/riktoken/encodings.rbs
55
+ - sig/generated/riktoken/encodings/cl100k_base.rbs
56
+ - sig/generated/riktoken/encodings/o200k_base.rbs
57
+ - sig/generated/riktoken/encodings/p50k_base.rbs
58
+ - sig/generated/riktoken/encodings/p50k_edit.rbs
59
+ - sig/generated/riktoken/encodings/r50k_base.rbs
60
+ - sig/generated/riktoken/tiktoken_file.rbs
61
+ - sig/generated/riktoken/version.rbs
62
+ homepage: https://github.com/moznion/riktoken
63
+ licenses:
64
+ - MIT
65
+ metadata:
66
+ allowed_push_host: https://rubygems.org
67
+ homepage_uri: https://github.com/moznion/riktoken
68
+ source_code_uri: https://github.com/moznion/riktoken
69
+ changelog_uri: https://github.com/moznion/riktoken/blob/main/CHANGELOG.md
70
+ rdoc_options: []
71
+ require_paths:
72
+ - lib
73
+ required_ruby_version: !ruby/object:Gem::Requirement
74
+ requirements:
75
+ - - ">="
76
+ - !ruby/object:Gem::Version
77
+ version: 3.1.0
78
+ required_rubygems_version: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ requirements: []
84
+ rubygems_version: 3.6.7
85
+ specification_version: 4
86
+ summary: Pure Ruby implementation of OpenAI's tiktoken
87
+ test_files: []