token_estimator 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4eb2770739cb655fed20189d0f63414c489d8d943066870b3cb6e18260d7523f
-  data.tar.gz: 367201d551861d9fb49a398baddc3537c87fa4c929ee3a5d82ee3f7c501a3dc1
+  metadata.gz: e76555160bf7038e96625963f99f434979da61bab11f70c0e7293d83db8c1588
+  data.tar.gz: 46d04371c15c88c39d96f87223c7a0713f9ceae367c52a48b50ca59b57d417e5
 SHA512:
-  metadata.gz: 00a175350809995a880b31b79db68fb2bfc184f4da9e180b89438959a4b39d17e01e169ca1896f97b984a36284712260d7e7f434d2bd12e3c4ee60599cb38280
-  data.tar.gz: dae8df4734be69cea8760a642c3e764d25340cf7b27058e831c3e72fc999b3a24cc93ea3c0706cf7fe0b937f6602c63d5649a5ff3cad4bfa06e6df860b31703a
+  metadata.gz: 110fb3caff83e609a5b4fa9e28ff92c1b0a866d9f30b669ea72b071e36f80e96377c451581dd94fa1155280311c9fd2fbb9ab5e9516c66c6c6b507aaba4972c7
+  data.tar.gz: 339ca2a1c63812c424cc2d1771d4dd665151ca069965511e469dd17587764a31957f463a2c5f5222e677413c9b1117f5331c1bd2762bd4d7c96260496e586f94

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [0.1.1] - 2024-07-15
+
+- Added `TokenEstimator::Estimator::SUPPORTED_FILE_TYPES` method to specify supported file types.
+
 ## [0.1.0] - 2024-07-11
 
 - Initial release

data/README.md

@@ -72,6 +72,9 @@ Counts tokens from a JSON object.
 #### `count_tokens_from_html`
 Counts tokens from an HTML string.
 
+#### `TokenEstimator::Estimator::SUPPORTED_FILE_TYPES`
+Return the supported file types.
+
 ## Roadmap
 Here is a checklist of the formats we currently support for token counting and those we plan to support in the future:
 

data/lib/token_estimator/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TokenEstimator
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/token_estimator.rb

@@ -13,6 +13,8 @@ module TokenEstimator
   class UnsupportedFileTypeError < StandardError; end
 
   class Estimator
+    SUPPORTED_FILE_TYPES = [".txt", ".csv", ".pdf", ".json", ".md", ".html", ".xlsx"]
+
     def initialize(tokenizer_name)
       @tokenizer = Tokenizers.from_pretrained(tokenizer_name)
     end
@@ -95,10 +97,6 @@ module TokenEstimator
       tokens.count
     end
 
-    def supported_file_types
-      [".txt", ".csv", ".pdf", ".json", ".md", ".html", ".xlsx"]
-    end
-
     private
 
     def extract_text_from_excel(xlsx)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: token_estimator
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - aemabit
@@ -139,6 +139,7 @@ files:
 - lib/token_estimator/version.rb
 - sig/token_estimator.rbs
 - token_estimator-0.1.0.gem
+- token_estimator-0.1.1.gem
 homepage: https://github.com/aemabit/token_estimator
 licenses:
 - MIT