gigatoken 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +7 -0
  2. data/Cargo.lock +3016 -0
  3. data/Cargo.toml +135 -0
  4. data/LICENSE +21 -0
  5. data/README.md +141 -0
  6. data/exe/gigatoken +9 -0
  7. data/ext/gigatoken/Cargo.toml +24 -0
  8. data/ext/gigatoken/extconf.rb +19 -0
  9. data/ext/gigatoken/src/error.rs +20 -0
  10. data/ext/gigatoken/src/gvl.rs +122 -0
  11. data/ext/gigatoken/src/lib.rs +50 -0
  12. data/ext/gigatoken/src/sentencepiece.rs +205 -0
  13. data/ext/gigatoken/src/sources.rs +207 -0
  14. data/ext/gigatoken/src/tokenizer.rs +571 -0
  15. data/lib/gigatoken/cli/bench.rb +64 -0
  16. data/lib/gigatoken/cli/support.rb +132 -0
  17. data/lib/gigatoken/cli/validate.rb +55 -0
  18. data/lib/gigatoken/cli.rb +18 -0
  19. data/lib/gigatoken/hub.rb +242 -0
  20. data/lib/gigatoken/packed_result.rb +48 -0
  21. data/lib/gigatoken/tokenizer.rb +117 -0
  22. data/lib/gigatoken/version.rb +5 -0
  23. data/lib/gigatoken.rb +29 -0
  24. data/rust-toolchain.toml +8 -0
  25. data/src/batch.rs +1808 -0
  26. data/src/bindings/bridge.rs +396 -0
  27. data/src/bindings/hub.rs +42 -0
  28. data/src/bindings/matcher.rs +114 -0
  29. data/src/bindings/mod.rs +14 -0
  30. data/src/bindings/padding.rs +177 -0
  31. data/src/bindings/pretokenize.rs +53 -0
  32. data/src/bindings/sources.rs +273 -0
  33. data/src/bindings/train.rs +125 -0
  34. data/src/bpe/mod.rs +1217 -0
  35. data/src/bpe/pretoken_cache.rs +495 -0
  36. data/src/bpe/sentencepiece.rs +1485 -0
  37. data/src/bpe/tiktoken.rs +2555 -0
  38. data/src/bpe_train.rs +351 -0
  39. data/src/input/decompress.rs +11 -0
  40. data/src/input/file_source.rs +514 -0
  41. data/src/input/jsonl.rs +94 -0
  42. data/src/input/mod.rs +333 -0
  43. data/src/input/parquet.rs +303 -0
  44. data/src/lib.rs +578 -0
  45. data/src/load_tokenizer/hf.rs +1036 -0
  46. data/src/load_tokenizer/hub.rs +344 -0
  47. data/src/load_tokenizer/mod.rs +3 -0
  48. data/src/load_tokenizer/tiktoken.rs +87 -0
  49. data/src/main.rs +95 -0
  50. data/src/pretokenize/fast/cl100k.rs +426 -0
  51. data/src/pretokenize/fast/cl100k_family.rs +891 -0
  52. data/src/pretokenize/fast/deepseek_v3.rs +605 -0
  53. data/src/pretokenize/fast/kimi.rs +281 -0
  54. data/src/pretokenize/fast/mask.rs +1486 -0
  55. data/src/pretokenize/fast/mod.rs +446 -0
  56. data/src/pretokenize/fast/nemotron.rs +138 -0
  57. data/src/pretokenize/fast/o200k.rs +347 -0
  58. data/src/pretokenize/fast/o200k_family.rs +1734 -0
  59. data/src/pretokenize/fast/olmo3.rs +505 -0
  60. data/src/pretokenize/fast/qwen2.rs +429 -0
  61. data/src/pretokenize/fast/qwen3_5.rs +541 -0
  62. data/src/pretokenize/fast/r50k.rs +1250 -0
  63. data/src/pretokenize/mod.rs +1079 -0
  64. data/src/pretokenize/options.rs +188 -0
  65. data/src/pretokenize/pretoken.rs +20 -0
  66. data/src/pretokenize/pretokenize_traits.rs +49 -0
  67. data/src/pretokenize/reference/avx512.rs +522 -0
  68. data/src/pretokenize/reference/combinator.rs +572 -0
  69. data/src/pretokenize/reference/mod.rs +28 -0
  70. data/src/pretokenize/reference/simd.rs +852 -0
  71. data/src/pretokenize/reference/state_machine.rs +365 -0
  72. data/src/pretokenize/unicode.rs +546 -0
  73. data/src/test_hub.rs +28 -0
  74. data/src/token.rs +42 -0
  75. metadata +161 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 8d646f2ee02894c38fc931c9a3a0f0b24e548e12225a3df6c8c49a4c7f9eec1c
4
+ data.tar.gz: 6b8fb459b7ea9869b6f6e0624e3a86ffdfe1adce176fc7bf530106f749150c13
5
+ SHA512:
6
+ metadata.gz: 0a7f6f40b1450e384364ba17ef18dd40a16c408cc9d7101e941b070eefe44349eb3368db29838337f49cc3189ebaf2ae03b21c3acf9d416c10f96fb9a64ef3fa
7
+ data.tar.gz: f549192a5290172dd0bf58ce1a0e26ad2baaa23fd3a4c767f199e6678a6f206aec8318935272396f4d0bfab9a954353f27145ad532616c5ce449714d0fbf4b49