tiktoken 0.1.0-aarch64-linux

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ac0d02721f78da67082a4f51d7928a82d39ba61b999f3e2e3d0e4b4e5b2eb58d
+  data.tar.gz: ea2cbb00c20eb439bf663e5a15ca3af58de81f94116f7385a44bc9fbf4d02999
+SHA512:
+  metadata.gz: 3380bbc03c730351ea2ae714fb81acc1ecb2ae2b85b6296d534f6875f85f35cbd0b13f9ab4975f5e7d936af2ba3e5841a84938f6eed9b53e5c23529bae0ca878
+  data.tar.gz: 52a2badf3349c19c0b2cdf611af47e52876168b59d5c637e576de0474cebcd8bcc109efb93d09217e144541d77e57e6c389b9b1b9d3e254d17b4b6bd9196c640

data/Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+members = ["ext/tiktoken"]
+
+[profile.release]
+strip = true

data/lib/tiktoken/encoder.rb

@@ -0,0 +1,16 @@
+module Tiktoken
+    class Encoder
+        def encode(text, allowed_special=[], disallowed_special="all")
+            _encode(text, allowed_special, disallowed_special)
+        end
+
+        def encode_ordinary(text)
+            _encode_ordinary(text)
+        end
+
+        def decode(tokens, utf_opts={invalid: :replace, undef: :replace})
+            _bytes = _decode(tokens)
+            _bytes.pack('C*').encode('UTF-8', **utf_opts)
+        end
+    end
+end

data/lib/tiktoken/version.rb

@@ -0,0 +1,3 @@
+module Tiktoken
+  VERSION = "0.1.0"
+end

data/lib/tiktoken.rb

@@ -0,0 +1,18 @@
+begin
+  require_relative "tiktoken/#{RUBY_VERSION.to_f}/tiktoken"
+rescue LoadError
+  require_relative "tiktoken/tiktoken"
+end
+
+require_relative "tiktoken/version"
+require_relative "tiktoken/encoder"
+
+module Tiktoken
+  def self.get_encoding(encoding, extra_special_tokens={})
+    Tiktoken._get_encoding(encoding, extra_special_tokens)
+  end
+
+  def self.encoding_for_model(model, extra_special_tokens={})
+    Tiktoken._encoding_for_model(model, extra_special_tokens)
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: tiktoken
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: aarch64-linux
+authors:
+- Arjun Singh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-03-21 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: arjun@volition.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Cargo.toml
+- lib/tiktoken.rb
+- lib/tiktoken/encoder.rb
+- lib/tiktoken/tiktoken.so
+- lib/tiktoken/version.rb
+homepage: https://github.com/volition-co/tiktoken
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 3.3.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.4
+signing_key: 
+specification_version: 4
+summary: Wrapper for OpenAI's tiktoken library
+test_files: []