tiktoken 0.1.0-x86_64-darwin

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f233e52639df448d294faf9376334e70230a897e8e6fb8cc3c160a3a644a0eda
+  data.tar.gz: 1ab52551e05386079680e0a15324a90ba0a304ff1acf5ee5f8f1a0cd6db7ff19
+SHA512:
+  metadata.gz: 8433b108d97c3733105b54262a31744ef403d64f0b676136b425d353d8c61fc10e20e3cd4d690153764d9f226df88e272d2a4849c13b123a5297c16936194069
+  data.tar.gz: ec14e446e1a3f356f9d180d94d72ca4d4a27b66e04107c1a02db728f2196c58ffbe9fc0bdc7f23c33c97c3ee6e8c2157b400aee54580b4010408d8d65240b45a

data/Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+members = ["ext/tiktoken"]
+
+[profile.release]
+strip = true

data/lib/tiktoken/encoder.rb

@@ -0,0 +1,16 @@
+module Tiktoken
+    class Encoder
+        def encode(text, allowed_special=[], disallowed_special="all")
+            _encode(text, allowed_special, disallowed_special)
+        end
+
+        def encode_ordinary(text)
+            _encode_ordinary(text)
+        end
+
+        def decode(tokens, utf_opts={invalid: :replace, undef: :replace})
+            _bytes = _decode(tokens)
+            _bytes.pack('C*').encode('UTF-8', **utf_opts)
+        end
+    end
+end

data/lib/tiktoken/version.rb

@@ -0,0 +1,3 @@
+module Tiktoken
+  VERSION = "0.1.0"
+end

data/lib/tiktoken.rb

@@ -0,0 +1,18 @@
+begin
+  require_relative "tiktoken/#{RUBY_VERSION.to_f}/tiktoken"
+rescue LoadError
+  require_relative "tiktoken/tiktoken"
+end
+
+require_relative "tiktoken/version"
+require_relative "tiktoken/encoder"
+
+module Tiktoken
+  def self.get_encoding(encoding, extra_special_tokens={})
+    Tiktoken._get_encoding(encoding, extra_special_tokens)
+  end
+
+  def self.encoding_for_model(model, extra_special_tokens={})
+    Tiktoken._encoding_for_model(model, extra_special_tokens)
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: tiktoken
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: x86_64-darwin
+authors:
+- Arjun Singh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-03-21 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: arjun@volition.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Cargo.toml
+- lib/tiktoken.rb
+- lib/tiktoken/encoder.rb
+- lib/tiktoken/tiktoken.bundle
+- lib/tiktoken/version.rb
+homepage: https://github.com/volition-co/tiktoken
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 3.3.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.4
+signing_key: 
+specification_version: 4
+summary: Wrapper for OpenAI's tiktoken library
+test_files: []