tiktoken 0.1.0-x86_64-linux

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b7f9da1fa201b52dd5233c02575c113070bd298f386f3880965f4d70b627309c
+  data.tar.gz: 38c6f10bf03a80eae89151319a9ac9a9ea23489478f8d1701d1d91c4015c83c4
+SHA512:
+  metadata.gz: 6de91cb53e377e29c1f2e61774eb299385ad7f80f1b5dc94eb26ca312c5845b515ecad2c6cbe2ac337995c6d9afe6b3b022e06e5591876b8b202559e558d2aa6
+  data.tar.gz: b15a47ba5bb27e4baee2f83e74bae233ee205e95eaccdf9c8e64257296d32d674dc45d6549504f0779726e0e4e77bf6c26b2757b14e3bc79e579ea8272c5a41a

data/Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+members = ["ext/tiktoken"]
+
+[profile.release]
+strip = true

data/lib/tiktoken/encoder.rb

@@ -0,0 +1,16 @@
+module Tiktoken
+    class Encoder
+        def encode(text, allowed_special=[], disallowed_special="all")
+            _encode(text, allowed_special, disallowed_special)
+        end
+
+        def encode_ordinary(text)
+            _encode_ordinary(text)
+        end
+
+        def decode(tokens, utf_opts={invalid: :replace, undef: :replace})
+            _bytes = _decode(tokens)
+            _bytes.pack('C*').encode('UTF-8', **utf_opts)
+        end
+    end
+end

data/lib/tiktoken/version.rb

@@ -0,0 +1,3 @@
+module Tiktoken
+  VERSION = "0.1.0"
+end

data/lib/tiktoken.rb

@@ -0,0 +1,18 @@
+begin
+  require_relative "tiktoken/#{RUBY_VERSION.to_f}/tiktoken"
+rescue LoadError
+  require_relative "tiktoken/tiktoken"
+end
+
+require_relative "tiktoken/version"
+require_relative "tiktoken/encoder"
+
+module Tiktoken
+  def self.get_encoding(encoding, extra_special_tokens={})
+    Tiktoken._get_encoding(encoding, extra_special_tokens)
+  end
+
+  def self.encoding_for_model(model, extra_special_tokens={})
+    Tiktoken._encoding_for_model(model, extra_special_tokens)
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: tiktoken
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: x86_64-linux
+authors:
+- Arjun Singh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-03-21 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: arjun@volition.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Cargo.toml
+- lib/tiktoken.rb
+- lib/tiktoken/encoder.rb
+- lib/tiktoken/tiktoken.so
+- lib/tiktoken/version.rb
+homepage: https://github.com/volition-co/tiktoken
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 3.3.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.4
+signing_key: 
+specification_version: 4
+summary: Wrapper for OpenAI's tiktoken library
+test_files: []