tiktoken 0.1.0-x86_64-linux-musl

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e030266426a077f984b90c71a19f1a1ee8f30a442e96c43b69c11a0aec141a03
+  data.tar.gz: a653d53784a0c7412c4a9bfde4c39d9bad796b7dce231ab45137b7cac1e3b099
+SHA512:
+  metadata.gz: a718b9dd3c7cda67504af8fc29fbaa6e691cbd244310a1d50124441963650d2cf716506bb00f1f0d1e6834ca5d996d280b6bd098fd441763fbb21279267436b9
+  data.tar.gz: 84b6ce34d9b6d493acef322a0e0ab97b5f83715d4bce9ed9c38f4e3629ea54b8bd50ca1fd1998cdff1175056aaf1c9839f92d7155356e4d2d6ca986da1addae1

data/Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+members = ["ext/tiktoken"]
+
+[profile.release]
+strip = true

data/lib/tiktoken/encoder.rb

@@ -0,0 +1,16 @@
+module Tiktoken
+    class Encoder
+        def encode(text, allowed_special=[], disallowed_special="all")
+            _encode(text, allowed_special, disallowed_special)
+        end
+
+        def encode_ordinary(text)
+            _encode_ordinary(text)
+        end
+
+        def decode(tokens, utf_opts={invalid: :replace, undef: :replace})
+            _bytes = _decode(tokens)
+            _bytes.pack('C*').encode('UTF-8', **utf_opts)
+        end
+    end
+end

data/lib/tiktoken/version.rb

@@ -0,0 +1,3 @@
+module Tiktoken
+  VERSION = "0.1.0"
+end

data/lib/tiktoken.rb

@@ -0,0 +1,18 @@
+begin
+  require_relative "tiktoken/#{RUBY_VERSION.to_f}/tiktoken"
+rescue LoadError
+  require_relative "tiktoken/tiktoken"
+end
+
+require_relative "tiktoken/version"
+require_relative "tiktoken/encoder"
+
+module Tiktoken
+  def self.get_encoding(encoding, extra_special_tokens={})
+    Tiktoken._get_encoding(encoding, extra_special_tokens)
+  end
+
+  def self.encoding_for_model(model, extra_special_tokens={})
+    Tiktoken._encoding_for_model(model, extra_special_tokens)
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: tiktoken
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: x86_64-linux-musl
+authors:
+- Arjun Singh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-03-21 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: arjun@volition.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Cargo.toml
+- lib/tiktoken.rb
+- lib/tiktoken/encoder.rb
+- lib/tiktoken/tiktoken.so
+- lib/tiktoken/version.rb
+homepage: https://github.com/volition-co/tiktoken
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 3.3.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.4
+signing_key: 
+specification_version: 4
+summary: Wrapper for OpenAI's tiktoken library
+test_files: []