tiktoken 0.1.0-arm64-darwin

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cafb591c23c31570a835ea89c633eaab2b8a185b9962aa814e79c069ae338728
+  data.tar.gz: b619e8e84c5afeb22e7e0727d1f2372363a705943530ace0727a59ecdad7a2e1
+SHA512:
+  metadata.gz: a1b23fe99dc76cfdfcdd51ff52144f5ba8ff8c6f773f93642da78cc87b7d28a6a50828bc56c0c2a86093fdcc6f483a7a8b6e213a378dc8ec53f7c53c78fc7dbd
+  data.tar.gz: e18018ae5d01450452d97327ce28f260ce99f01bdac89c5e3ff1cd5b277792312c9c3532c6708f9cafa61cbd7e323d44b4bb3fb77a8ed77b4147c92e271a0d97

data/Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+members = ["ext/tiktoken"]
+
+[profile.release]
+strip = true

data/lib/tiktoken/encoder.rb

@@ -0,0 +1,16 @@
+module Tiktoken
+    class Encoder
+        def encode(text, allowed_special=[], disallowed_special="all")
+            _encode(text, allowed_special, disallowed_special)
+        end
+
+        def encode_ordinary(text)
+            _encode_ordinary(text)
+        end
+
+        def decode(tokens, utf_opts={invalid: :replace, undef: :replace})
+            _bytes = _decode(tokens)
+            _bytes.pack('C*').encode('UTF-8', **utf_opts)
+        end
+    end
+end

data/lib/tiktoken/version.rb

@@ -0,0 +1,3 @@
+module Tiktoken
+  VERSION = "0.1.0"
+end

data/lib/tiktoken.rb

@@ -0,0 +1,18 @@
+begin
+  require_relative "tiktoken/#{RUBY_VERSION.to_f}/tiktoken"
+rescue LoadError
+  require_relative "tiktoken/tiktoken"
+end
+
+require_relative "tiktoken/version"
+require_relative "tiktoken/encoder"
+
+module Tiktoken
+  def self.get_encoding(encoding, extra_special_tokens={})
+    Tiktoken._get_encoding(encoding, extra_special_tokens)
+  end
+
+  def self.encoding_for_model(model, extra_special_tokens={})
+    Tiktoken._encoding_for_model(model, extra_special_tokens)
+  end
+end

metadata

@@ -0,0 +1,50 @@
+--- !ruby/object:Gem::Specification
+name: tiktoken
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: arm64-darwin
+authors:
+- Arjun Singh
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2023-03-21 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: arjun@volition.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Cargo.toml
+- lib/tiktoken.rb
+- lib/tiktoken/encoder.rb
+- lib/tiktoken/tiktoken.bundle
+- lib/tiktoken/version.rb
+homepage: https://github.com/volition-co/tiktoken
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 3.3.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.4
+signing_key: 
+specification_version: 4
+summary: Wrapper for OpenAI's tiktoken library
+test_files: []