tiktoken_ruby 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 81166d50bf3f152dd63a68d0d40f364b8232c906d8e4534f6c47360ea72f99a7
+  data.tar.gz: 93b7db86bfd25bc04dffd1c7568b95ac7421c0445865c86776a4a6b58248fbd5
+SHA512:
+  metadata.gz: '03950362233a94958d1811f183e5151d4f52d38b11c913810e819bba78728d9c89b0cf49489346b19a3d735cf67efa2e4718a34cea9d3e65166e333d115c2536'
+  data.tar.gz: 9ee22a55ee71ec3e75fa6090984faa1f9b565532331fa91e6295ab0760264f75d94aee1688e59bc2191aeac8e90423a2ac48d73dfa3e7c833966532adae67788

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.standard.yml

@@ -0,0 +1,3 @@
+# For available configuration options, see:
+#   https://github.com/testdouble/standard
+ruby_version: 2.6

data/Cargo.lock

@@ -0,0 +1,484 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "aho-corasick"
+version = "0.7.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.70"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4"
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "base64"
+version = "0.21.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a"
+
+[[package]]
+name = "bindgen"
+version = "0.60.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6"
+dependencies = [
+ "bitflags",
+ "cexpr",
+ "clang-sys",
+ "lazy_static",
+ "lazycell",
+ "peeking_take_while",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "rustc-hash",
+ "shlex",
+]
+
+[[package]]
+name = "bit-set"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+dependencies = [
+ "bit-vec",
+]
+
+[[package]]
+name = "bit-vec"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bstr"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09"
+dependencies = [
+ "memchr",
+ "once_cell",
+ "regex-automata",
+ "serde",
+]
+
+[[package]]
+name = "cexpr"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
+dependencies = [
+ "nom",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clang-sys"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a"
+dependencies = [
+ "glob",
+ "libc",
+ "libloading",
+]
+
+[[package]]
+name = "fancy-regex"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2"
+dependencies = [
+ "bit-set",
+ "regex",
+]
+
+[[package]]
+name = "glob"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "lazycell"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
+
+[[package]]
+name = "libc"
+version = "0.2.140"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c"
+
+[[package]]
+name = "libloading"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f"
+dependencies = [
+ "cfg-if",
+ "winapi",
+]
+
+[[package]]
+name = "lock_api"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
+[[package]]
+name = "magnus"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc87660cd7daa49fddbfd524c836de54d5c927d520cd163f43700c5087c57d6c"
+dependencies = [
+ "magnus-macros",
+ "rb-sys",
+ "rb-sys-env",
+]
+
+[[package]]
+name = "magnus-macros"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "206cb23bfeea05180c97522ef6a3e52a4eb17b0ed2f30ee3ca9c4f994d2378ae"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
+
+[[package]]
+name = "parking_lot"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-sys",
+]
+
+[[package]]
+name = "peeking_take_while"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rb-sys"
+version = "0.9.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4783528e031c3902524cfe685e0008d644e8b34bad4d1a19c1f39f4394d0777b"
+dependencies = [
+ "rb-sys-build",
+]
+
+[[package]]
+name = "rb-sys-build"
+version = "0.9.68"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c04474335dd597126d20fe90d94504b3582a30cbc36bc72c7d6ed3ef8cf168"
+dependencies = [
+ "bindgen",
+ "lazy_static",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "shell-words",
+ "syn",
+]
+
+[[package]]
+name = "rb-sys-env"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "regex"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
+
+[[package]]
+name = "rustc-hash"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
+[[package]]
+name = "serde"
+version = "1.0.157"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "707de5fcf5df2b5788fca98dd7eab490bc2fd9b7ef1404defc462833b83f25ca"
+
+[[package]]
+name = "shell-words"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde"
+
+[[package]]
+name = "shlex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
+
+[[package]]
+name = "smallvec"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "tiktoken-rs"
+version = "0.3.2"
+source = "git+https://github.com/IAPark/tiktoken-rs.git#5231fbf4a91d9221a713522e755445d0dde341fa"
+dependencies = [
+ "anyhow",
+ "base64",
+ "bstr",
+ "fancy-regex",
+ "lazy_static",
+ "parking_lot",
+ "rustc-hash",
+]
+
+[[package]]
+name = "tiktoken_ruby"
+version = "0.1.0"
+dependencies = [
+ "magnus",
+ "tiktoken-rs",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-sys"
+version = "0.45.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.42.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"

data/Cargo.toml

@@ -0,0 +1,7 @@
+# This Cargo.toml is here to let externals tools (IDEs, etc.) know that this is
+# a Rust project. Your extensions dependencies should be added to the Cargo.toml
+# in the ext/ directory.
+
+[workspace]
+members = ["./ext/tiktoken_ruby"]
+resolver = "2"

data/Gemfile

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in tiktoken_ruby.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rake-compiler"
+gem "rb_sys"
+
+gem "rspec", "~> 3.0"
+
+gem "standard", "~> 1.3"
+gem 'pry', '~> 0.14.2'

data/Gemfile.lock

@@ -0,0 +1,77 @@
+PATH
+  remote: .
+  specs:
+    tiktoken_ruby (0.0.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    coderay (1.1.3)
+    diff-lcs (1.5.0)
+    json (2.6.3)
+    language_server-protocol (3.17.0.3)
+    method_source (1.0.0)
+    parallel (1.22.1)
+    parser (3.2.1.1)
+      ast (~> 2.4.1)
+    pry (0.14.2)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    rake-compiler (1.2.1)
+      rake
+    rb_sys (0.9.68)
+    regexp_parser (2.7.0)
+    rexml (3.2.5)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.4)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rubocop (1.48.1)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.2.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.26.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.27.0)
+      parser (>= 3.2.1.0)
+    rubocop-performance (1.16.0)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    ruby-progressbar (1.13.0)
+    standard (1.25.1)
+      language_server-protocol (~> 3.17.0.2)
+      rubocop (= 1.48.1)
+      rubocop-performance (= 1.16.0)
+    unicode-display_width (2.4.2)
+
+PLATFORMS
+  arm64-darwin-22
+  x86_64-linux
+
+DEPENDENCIES
+  pry (~> 0.14.2)
+  rake (~> 13.0)
+  rake-compiler
+  rb_sys
+  rspec (~> 3.0)
+  standard (~> 1.3)
+  tiktoken_ruby!
+
+BUNDLED WITH
+   2.4.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 IAPark
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,37 @@
+[![Gem Version](https://badge.fury.io/rb/tiktoken_ruby.svg)](https://badge.fury.io/rb/tiktoken_ruby)
+# tiktoken_ruby
+
+[Tiktoken](https://github.com/openai/tiktoken) is BPE tokenizer from OpenAI used with their GPT models.
+This is a wrapper around it aimed primarily at enabling accurate counts of GPT model tokens used. 
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add tiktoken_ruby
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install tiktoken_ruby
+
+## Usage
+
+```ruby
+encoding = Tiktoken::Encoding.r50k_base
+tokens = encoding.encode("Hello world!")
+puts encoding.decode(tokens)
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/iapark/tiktoken_ruby.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+require "standard/rake"
+require "rake/extensiontask"
+require "rb_sys/extensiontask"
+
+GEMSPEC = Gem::Specification.load("tiktoken_ruby.gemspec")
+
+RbSys::ExtensionTask.new("tiktoken_ruby", GEMSPEC) do |ext|
+  ext.lib_dir = "lib/tiktoken_ruby"
+end
+
+RSpec::Core::RakeTask.new(:spec)
+
+
+task :native, [:platform] do |_t, platform:|
+  sh "bundle", "exec", "rb-sys-dock", "--platform", platform, "--build"
+end
+
+task build: :compile
+
+task default: %i[compile spec standard]

data/ext/tiktoken_ruby/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "tiktoken_ruby"
+version = "0.1.0"
+edition = "2021"
+authors = ["IAPark <isaac.a.park@gmail.com>"]
+license = "MIT"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = { version = "0.4" }
+tiktoken-rs = { git = "https://github.com/IAPark/tiktoken-rs.git" }

data/ext/tiktoken_ruby/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("tiktoken_ruby/tiktoken_ruby")

data/ext/tiktoken_ruby/src/core_bpe_wrapper.rs

@@ -0,0 +1,44 @@
+use std::collections::HashSet;
+
+use crate::uncicode_error;
+
+
+#[magnus::wrap(class = "Tiktoken::Ext::CoreBPE")]
+pub struct CoreBPEWrapper {
+    core_bpe: tiktoken_rs::CoreBPE,
+}
+
+impl CoreBPEWrapper {
+    pub fn new(core_bpe: tiktoken_rs::CoreBPE) -> Self {
+        Self { core_bpe }
+    }
+
+    pub fn encode_ordinary(&self, text: String) -> Vec<usize> {
+        self.core_bpe.encode_ordinary(text.as_str())
+    }
+
+    pub fn encode(&self, text: String, allowed_special: magnus::RArray) -> Result<Vec<usize>, magnus::Error> {
+        let allowed_special: Vec<String> = allowed_special.to_vec()?;
+        let allowed_special: Vec<&str> = allowed_special.iter().map(|s| s.as_str()).collect();
+        let allowed_special: HashSet<&str> = HashSet::from_iter(allowed_special.iter().cloned());
+
+        Ok(self.core_bpe.encode(text.as_str(), allowed_special))
+    }
+
+    pub fn encode_with_special_tokens(&self, text: String) -> Vec<usize> {
+        self.core_bpe.encode_with_special_tokens(text.as_str())
+    }
+
+    pub fn decode(&self, ids: Vec<usize>) -> Result<String, magnus::Error> {
+        self.core_bpe.decode(ids)
+            .map_err(|e| {
+                let error = match uncicode_error() {
+                    Ok(error) => error,
+                    Err(e) => return e
+                };
+                
+                magnus::Error::new(error, e.to_string())
+        })
+
+    }
+}

data/ext/tiktoken_ruby/src/lib.rs

@@ -0,0 +1,52 @@
+mod core_bpe_wrapper;
+
+use core_bpe_wrapper::CoreBPEWrapper;
+use magnus::{define_module, function, prelude::*, Error, method, class, RModule, ExceptionClass};
+
+fn r50k_base() -> CoreBPEWrapper {
+    let core_bpe = tiktoken_rs::r50k_base().unwrap();
+    CoreBPEWrapper::new(core_bpe)
+}
+fn p50k_base() -> CoreBPEWrapper {
+    let core_bpe = tiktoken_rs::p50k_base().unwrap();
+    CoreBPEWrapper::new(core_bpe)
+}
+fn p50k_edit() -> CoreBPEWrapper {
+    let core_bpe = tiktoken_rs::p50k_edit().unwrap();
+    CoreBPEWrapper::new(core_bpe)
+}
+fn cl100k_base() -> CoreBPEWrapper {
+    let core_bpe = tiktoken_rs::cl100k_base().unwrap();
+    CoreBPEWrapper::new(core_bpe)
+}
+
+fn module() -> Result<RModule, magnus::Error> {
+    define_module("Tiktoken")
+}
+
+fn uncicode_error() -> Result<ExceptionClass, magnus::Error> {
+    module()?.define_error("UnicodeError", magnus::exception::standard_error())
+}
+
+#[magnus::init]
+fn init() -> Result<(), Error> {
+    let module = module()?;
+
+    let factory_module = module.define_module("BpeFactory")?;
+    factory_module.define_singleton_method("r50k_base", function!(r50k_base, 0))?;
+    factory_module.define_singleton_method("p50k_base", function!(p50k_base, 0))?;
+    factory_module.define_singleton_method("p50k_base", function!(p50k_edit, 0))?;
+    factory_module.define_singleton_method("cl100k_base", function!(cl100k_base, 0))?;
+
+
+    let ext_module = module.define_module("Ext")?;
+    let bpe_class = ext_module.define_class("CoreBPE", class::object())?;
+
+    bpe_class.define_method("encode_ordinary", method!(CoreBPEWrapper::encode_ordinary, 1))?;
+    bpe_class.define_method("encode", method!(CoreBPEWrapper::encode, 2))?;
+    bpe_class.define_method("encode_with_special_tokens", method!(CoreBPEWrapper::encode_with_special_tokens, 1))?;
+
+
+    bpe_class.define_method("decode", method!(CoreBPEWrapper::decode, 1))?;
+    Ok(())
+}

data/lib/tiktoken_ruby/encoding.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class Tiktoken::Encoding 
+    def self.method_missing(method)
+        Tiktoken::Encoding.new(Tiktoken::BpeFactory.send(method))
+    end
+
+    def initialize(ext_base_bpe)
+        @ext_base_bpe = ext_base_bpe
+    end
+
+    def encode_ordinary(text)
+        @ext_base_bpe.encode_ordinary(text)
+    end
+
+    def encode(text, allowed_special: [])
+        @ext_base_bpe.encode(text, allowed_special)
+    end
+
+    def decode(tokens)
+        @ext_base_bpe.decode(tokens)
+    end
+end

data/lib/tiktoken_ruby/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Tiktoken
+  VERSION = "0.0.1"
+end

data/lib/tiktoken_ruby.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "tiktoken_ruby/version"
+require_relative "tiktoken_ruby/encoding.rb"
+require_relative "tiktoken_ruby/tiktoken_ruby"
+
+module Tiktoken
+  class Error < StandardError; end
+end

data/sig/tiktoken_ruby.rbs

@@ -0,0 +1,4 @@
+module TiktokenRuby
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

data/tiktoken_ruby.gemspec

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "lib/tiktoken_ruby/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "tiktoken_ruby"
+  spec.version = Tiktoken::VERSION
+  spec.authors = ["IAPark"]
+  spec.email = ["isaac.a.park@gmail.com"]
+
+  spec.summary = "Ruby wrapper for Tiktoken"
+  spec.description = "Unofficial Ruby wrapper for Tiktoken by way of the unofficial rust bindings"
+  spec.homepage = "https://github.com/IAPark/tiktoken_ruby"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+  spec.required_rubygems_version = ">= 3.3.11"
+  spec.platform = Gem::Platform::RUBY
+
+  #spec.metadata["allowed_push_host"] = "TODO: Set to your gem server 'https://example.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/IAPark/tiktoken_ruby"
+  #spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.extensions = ["ext/tiktoken_ruby/Cargo.toml"]
+
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "example-gem", "~> 1.0"
+  # spec.add_dependency "rb_sys", "~> 0.9"
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: tiktoken_ruby
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- IAPark
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-03-19 00:00:00.000000000 Z
+dependencies: []
+description: Unofficial Ruby wrapper for Tiktoken by way of the unofficial rust bindings
+email:
+- isaac.a.park@gmail.com
+executables: []
+extensions:
+- ext/tiktoken_ruby/Cargo.toml
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".standard.yml"
+- Cargo.lock
+- Cargo.toml
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- ext/tiktoken_ruby/Cargo.toml
+- ext/tiktoken_ruby/extconf.rb
+- ext/tiktoken_ruby/src/core_bpe_wrapper.rs
+- ext/tiktoken_ruby/src/lib.rs
+- lib/tiktoken_ruby.rb
+- lib/tiktoken_ruby/encoding.rb
+- lib/tiktoken_ruby/version.rb
+- sig/tiktoken_ruby.rbs
+- tiktoken_ruby.gemspec
+homepage: https://github.com/IAPark/tiktoken_ruby
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/IAPark/tiktoken_ruby
+  source_code_uri: https://github.com/IAPark/tiktoken_ruby
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.11
+requirements: []
+rubygems_version: 3.4.6
+signing_key:
+specification_version: 4
+summary: Ruby wrapper for Tiktoken
+test_files: []