gigatoken 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +7 -0
  2. data/Cargo.lock +3016 -0
  3. data/Cargo.toml +135 -0
  4. data/LICENSE +21 -0
  5. data/README.md +141 -0
  6. data/exe/gigatoken +9 -0
  7. data/ext/gigatoken/Cargo.toml +24 -0
  8. data/ext/gigatoken/extconf.rb +19 -0
  9. data/ext/gigatoken/src/error.rs +20 -0
  10. data/ext/gigatoken/src/gvl.rs +122 -0
  11. data/ext/gigatoken/src/lib.rs +50 -0
  12. data/ext/gigatoken/src/sentencepiece.rs +205 -0
  13. data/ext/gigatoken/src/sources.rs +207 -0
  14. data/ext/gigatoken/src/tokenizer.rs +571 -0
  15. data/lib/gigatoken/cli/bench.rb +64 -0
  16. data/lib/gigatoken/cli/support.rb +132 -0
  17. data/lib/gigatoken/cli/validate.rb +55 -0
  18. data/lib/gigatoken/cli.rb +18 -0
  19. data/lib/gigatoken/hub.rb +242 -0
  20. data/lib/gigatoken/packed_result.rb +48 -0
  21. data/lib/gigatoken/tokenizer.rb +117 -0
  22. data/lib/gigatoken/version.rb +5 -0
  23. data/lib/gigatoken.rb +29 -0
  24. data/rust-toolchain.toml +8 -0
  25. data/src/batch.rs +1808 -0
  26. data/src/bindings/bridge.rs +396 -0
  27. data/src/bindings/hub.rs +42 -0
  28. data/src/bindings/matcher.rs +114 -0
  29. data/src/bindings/mod.rs +14 -0
  30. data/src/bindings/padding.rs +177 -0
  31. data/src/bindings/pretokenize.rs +53 -0
  32. data/src/bindings/sources.rs +273 -0
  33. data/src/bindings/train.rs +125 -0
  34. data/src/bpe/mod.rs +1217 -0
  35. data/src/bpe/pretoken_cache.rs +495 -0
  36. data/src/bpe/sentencepiece.rs +1485 -0
  37. data/src/bpe/tiktoken.rs +2555 -0
  38. data/src/bpe_train.rs +351 -0
  39. data/src/input/decompress.rs +11 -0
  40. data/src/input/file_source.rs +514 -0
  41. data/src/input/jsonl.rs +94 -0
  42. data/src/input/mod.rs +333 -0
  43. data/src/input/parquet.rs +303 -0
  44. data/src/lib.rs +578 -0
  45. data/src/load_tokenizer/hf.rs +1036 -0
  46. data/src/load_tokenizer/hub.rs +344 -0
  47. data/src/load_tokenizer/mod.rs +3 -0
  48. data/src/load_tokenizer/tiktoken.rs +87 -0
  49. data/src/main.rs +95 -0
  50. data/src/pretokenize/fast/cl100k.rs +426 -0
  51. data/src/pretokenize/fast/cl100k_family.rs +891 -0
  52. data/src/pretokenize/fast/deepseek_v3.rs +605 -0
  53. data/src/pretokenize/fast/kimi.rs +281 -0
  54. data/src/pretokenize/fast/mask.rs +1486 -0
  55. data/src/pretokenize/fast/mod.rs +446 -0
  56. data/src/pretokenize/fast/nemotron.rs +138 -0
  57. data/src/pretokenize/fast/o200k.rs +347 -0
  58. data/src/pretokenize/fast/o200k_family.rs +1734 -0
  59. data/src/pretokenize/fast/olmo3.rs +505 -0
  60. data/src/pretokenize/fast/qwen2.rs +429 -0
  61. data/src/pretokenize/fast/qwen3_5.rs +541 -0
  62. data/src/pretokenize/fast/r50k.rs +1250 -0
  63. data/src/pretokenize/mod.rs +1079 -0
  64. data/src/pretokenize/options.rs +188 -0
  65. data/src/pretokenize/pretoken.rs +20 -0
  66. data/src/pretokenize/pretokenize_traits.rs +49 -0
  67. data/src/pretokenize/reference/avx512.rs +522 -0
  68. data/src/pretokenize/reference/combinator.rs +572 -0
  69. data/src/pretokenize/reference/mod.rs +28 -0
  70. data/src/pretokenize/reference/simd.rs +852 -0
  71. data/src/pretokenize/reference/state_machine.rs +365 -0
  72. data/src/pretokenize/unicode.rs +546 -0
  73. data/src/test_hub.rs +28 -0
  74. data/src/token.rs +42 -0
  75. metadata +161 -0
metadata ADDED
@@ -0,0 +1,161 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: gigatoken
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Eric Jacobs
8
+ bindir: exe
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: async
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '2.43'
19
+ type: :runtime
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - "~>"
24
+ - !ruby/object:Gem::Version
25
+ version: '2.43'
26
+ - !ruby/object:Gem::Dependency
27
+ name: async-http
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '0.96'
33
+ type: :runtime
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '0.96'
40
+ - !ruby/object:Gem::Dependency
41
+ name: dry-cli
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '1.0'
47
+ type: :runtime
48
+ prerelease: false
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '1.0'
54
+ description: 'Ruby bindings to the gigatoken core crate: BPE and SentencePiece tokenization
55
+ at GB/s.'
56
+ email:
57
+ - eric@ebj.dev
58
+ executables:
59
+ - gigatoken
60
+ extensions:
61
+ - ext/gigatoken/extconf.rb
62
+ extra_rdoc_files: []
63
+ files:
64
+ - Cargo.lock
65
+ - Cargo.toml
66
+ - LICENSE
67
+ - README.md
68
+ - exe/gigatoken
69
+ - ext/gigatoken/Cargo.toml
70
+ - ext/gigatoken/extconf.rb
71
+ - ext/gigatoken/src/error.rs
72
+ - ext/gigatoken/src/gvl.rs
73
+ - ext/gigatoken/src/lib.rs
74
+ - ext/gigatoken/src/sentencepiece.rs
75
+ - ext/gigatoken/src/sources.rs
76
+ - ext/gigatoken/src/tokenizer.rs
77
+ - lib/gigatoken.rb
78
+ - lib/gigatoken/cli.rb
79
+ - lib/gigatoken/cli/bench.rb
80
+ - lib/gigatoken/cli/support.rb
81
+ - lib/gigatoken/cli/validate.rb
82
+ - lib/gigatoken/hub.rb
83
+ - lib/gigatoken/packed_result.rb
84
+ - lib/gigatoken/tokenizer.rb
85
+ - lib/gigatoken/version.rb
86
+ - rust-toolchain.toml
87
+ - src/batch.rs
88
+ - src/bindings/bridge.rs
89
+ - src/bindings/hub.rs
90
+ - src/bindings/matcher.rs
91
+ - src/bindings/mod.rs
92
+ - src/bindings/padding.rs
93
+ - src/bindings/pretokenize.rs
94
+ - src/bindings/sources.rs
95
+ - src/bindings/train.rs
96
+ - src/bpe/mod.rs
97
+ - src/bpe/pretoken_cache.rs
98
+ - src/bpe/sentencepiece.rs
99
+ - src/bpe/tiktoken.rs
100
+ - src/bpe_train.rs
101
+ - src/input/decompress.rs
102
+ - src/input/file_source.rs
103
+ - src/input/jsonl.rs
104
+ - src/input/mod.rs
105
+ - src/input/parquet.rs
106
+ - src/lib.rs
107
+ - src/load_tokenizer/hf.rs
108
+ - src/load_tokenizer/hub.rs
109
+ - src/load_tokenizer/mod.rs
110
+ - src/load_tokenizer/tiktoken.rs
111
+ - src/main.rs
112
+ - src/pretokenize/fast/cl100k.rs
113
+ - src/pretokenize/fast/cl100k_family.rs
114
+ - src/pretokenize/fast/deepseek_v3.rs
115
+ - src/pretokenize/fast/kimi.rs
116
+ - src/pretokenize/fast/mask.rs
117
+ - src/pretokenize/fast/mod.rs
118
+ - src/pretokenize/fast/nemotron.rs
119
+ - src/pretokenize/fast/o200k.rs
120
+ - src/pretokenize/fast/o200k_family.rs
121
+ - src/pretokenize/fast/olmo3.rs
122
+ - src/pretokenize/fast/qwen2.rs
123
+ - src/pretokenize/fast/qwen3_5.rs
124
+ - src/pretokenize/fast/r50k.rs
125
+ - src/pretokenize/mod.rs
126
+ - src/pretokenize/options.rs
127
+ - src/pretokenize/pretoken.rs
128
+ - src/pretokenize/pretokenize_traits.rs
129
+ - src/pretokenize/reference/avx512.rs
130
+ - src/pretokenize/reference/combinator.rs
131
+ - src/pretokenize/reference/mod.rs
132
+ - src/pretokenize/reference/simd.rs
133
+ - src/pretokenize/reference/state_machine.rs
134
+ - src/pretokenize/unicode.rs
135
+ - src/test_hub.rs
136
+ - src/token.rs
137
+ homepage: https://github.com/jetpks/gigatoken-rb
138
+ licenses:
139
+ - MIT
140
+ metadata:
141
+ bug_tracker_uri: https://github.com/jetpks/gigatoken-rb/issues
142
+ homepage_uri: https://github.com/jetpks/gigatoken-rb
143
+ source_code_uri: https://github.com/jetpks/gigatoken-rb
144
+ rdoc_options: []
145
+ require_paths:
146
+ - lib
147
+ required_ruby_version: !ruby/object:Gem::Requirement
148
+ requirements:
149
+ - - ">="
150
+ - !ruby/object:Gem::Version
151
+ version: 3.3.0
152
+ required_rubygems_version: !ruby/object:Gem::Requirement
153
+ requirements:
154
+ - - ">="
155
+ - !ruby/object:Gem::Version
156
+ version: '0'
157
+ requirements: []
158
+ rubygems_version: 3.6.9
159
+ specification_version: 4
160
+ summary: Tokenize your documents at GB/s
161
+ test_files: []