context_spook 1.5.1 → 1.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bb51df13e01f4262026aea154cc901997a0b5cec6566f4bc71e42467ec3dfed
-  data.tar.gz: b26ead5c765e99a931026d267e6de374dc0c93f8a5c64e46bfe09c28d892dd99
+  metadata.gz: 6b21e74b5db171841cc7af7c0e5204f252cb7474de2f45512c3597f61774ffcb
+  data.tar.gz: 3068b2faa4a58ef85722f9e756b712d89e45127eff52d7ca9695535611e04670
 SHA512:
-  metadata.gz: 4f8995fb526a7f6fd8be4c15c2a869775fdc5a826571bd84723689e695ca3b982325d56bf971da3982afc6cecdb2575d20d0319ea073d505d85e19a6415fdffc
-  data.tar.gz: 56bc054c3fe75f245ed97a7d92c0c91cb31dcfb1e7b101b2ee3a2092324a7290b7ec0449c5dba310b809a102e64d2bd40a656d126feef5a3d03ec588eaead314
+  metadata.gz: 41f8ac31162e123cf27b6ad109bcd7cc58c6b1e687ac1e4f3af57e6e446c1ebc07ab4a4290af659e7f5e8cd3026f2342a53c109550628bb4f2cf536ec1aea646
+  data.tar.gz: dde406feb3c651bd99ea4a0aa4c10342690459604fb27c7e1d2d3df0302326dbabb83ef78350f860b23773bbd5d56ef31fcb66841b595be937006ae29160dbd8

data/Rakefile

@@ -31,14 +31,14 @@ GemHadar do
 
   required_ruby_version  '>= 3.1'
 
-  dependency 'tins',           '~>1.46'
-  dependency 'json',           '~>2.0'
-  dependency 'term-ansicolor', '~> 1.11'
-  dependency 'mize',           '~> 0.6'
-  dependency 'mime-types',     '~> 3.0'
-  dependency 'yaml',           '~> 0.4'
-  dependency 'pathname',       '~> 0.4'
-  dependency 'toon-ruby',      '>= 0.1'
+  dependency 'tins',                   '~> 1.46'
+  dependency 'json',                   '~> 2.0'
+  dependency 'term-ansicolor',         '~> 1.11'
+  dependency 'mize',                   '~> 0.6'
+  dependency 'mime-types',             '~> 3.0'
+  dependency 'yaml',                   '~> 0.4'
+  dependency 'pathname',               '~> 0.4'
+  dependency 'toon-ruby',              '>= 0.1'
   development_dependency 'all_images', '~> 0.6'
   development_dependency 'rspec',      '~> 3.2'
   development_dependency 'debug'

data/context_spook.gemspec

@@ -1,9 +1,9 @@
 # -*- encoding: utf-8 -*-
-# stub: context_spook 1.5.1 ruby lib
+# stub: context_spook 1.6.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "context_spook".freeze
-  s.version = "1.5.1".freeze
+  s.version = "1.6.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -18,13 +18,13 @@ Gem::Specification.new do |s|
   s.licenses = ["MIT".freeze]
   s.rdoc_options = ["--title".freeze, "ContextSpook - context_spook collects project context for AI".freeze, "--main".freeze, "README.md".freeze]
   s.required_ruby_version = Gem::Requirement.new(">= 3.1".freeze)
-  s.rubygems_version = "4.0.3".freeze
+  s.rubygems_version = "4.0.10".freeze
   s.summary = "context_spook collects project context for AI".freeze
   s.test_files = ["spec/context_spook/generator_spec.rb".freeze, "spec/context_spook/toon_spec.rb".freeze, "spec/spec_helper.rb".freeze]
 
   s.specification_version = 4
 
-  s.add_development_dependency(%q<gem_hadar>.freeze, [">= 2.17.0".freeze])
+  s.add_development_dependency(%q<gem_hadar>.freeze, [">= 2.17.1".freeze])
   s.add_development_dependency(%q<all_images>.freeze, ["~> 0.6".freeze])
   s.add_development_dependency(%q<rspec>.freeze, ["~> 3.2".freeze])
   s.add_development_dependency(%q<debug>.freeze, [">= 0".freeze])

data/lib/context_spook/generator.rb

@@ -72,6 +72,13 @@ module ContextSpook
 
     private_class_method :new
 
+    # The token_estimator attribute allows setting the logic used to estimate
+    # the number of tokens in a serialized context.
+    #
+    # It expects an object that responds to #call(text), where text is the
+    # serialized context string.
+    attr_writer :token_estimator
+
     # The initialize method sets up the object by evaluating the provided block
     # in the object's context.
     #
@@ -85,6 +92,7 @@ module ContextSpook
       %w[ TOON JSON ].include?(@format) or
         raise ArgumentError,
           "format needs to be either JSON or TOON, was #{@format.inspect}"
+      self.token_estimator = ContextSpook::Utils.method(:estimate_tokens)
       block and instance_eval(&block)
     end
 

data/lib/context_spook/output_context.rb

@@ -5,16 +5,21 @@
 # functionality for generated context objects.
 module ContextSpook::OutputContext
   # The output_context_size method prints the total size of the generated
-  # context JSON representation.
+  # context representation and an estimated token count.
   #
   # This method calculates the size of the context object when serialized to
-  # JSON, formats it using binary units (KiB, MiB, etc.), and outputs the
-  # result to standard error.
+  # JSON (or TOON), formats it using binary units, estimates the token count
+  # based on bytes, and outputs the result to standard error.
   def output_context_size
-    context_size =
-      (@format == 'TOON' ? @context&.toon_size : @context&.size).to_i
-    context_size = ContextSpook::Utils.format_size(context_size)
-    verbose_puts "Built #{context_size} of #@format context in total."
+    context          = ''
+    if @context
+      context        = @format == 'TOON' ? @context.to_toon : @context.to_json
+    end
+    formatted_size   = ContextSpook::Utils.format_size(context.size)
+    tokens           = @token_estimator.call(context)
+    formatted_tokens = ContextSpook::Utils.format_tokens(tokens)
+
+    verbose_puts "Built #{formatted_size} (#{formatted_tokens} tokens) of #@format context in total."
   end
 
   # The output_context method writes the generated context to the specified

data/lib/context_spook/utils.rb

@@ -16,4 +16,26 @@ module ContextSpook::Utils
   def format_size(context_size)
     Tins::Unit.format(context_size, format: '%.2f %U', unit: ?b, prefix: 1024)
   end
+
+  # The format_tokens method converts a token count into a human-readable
+  # string using SI prefixes (e.g., 1.2 kT).
+  #
+  # @param tokens [ Integer ] the number of tokens to be formatted
+  #
+  # @return [ String ] the formatted token string
+  def format_tokens(tokens)
+    Tins::Unit.format(tokens, unit: ?T, prefix: :si_uc, format: '%.1f %U')
+  end
+
+  # The estimate_tokens method provides a crude estimation of token counts
+  # based on the byte size of the input content.
+  #
+  # This follows a heuristic where one token is roughly equivalent to 3.5 bytes.
+  #
+  # @param text [ String ] the content to be estimated
+  #
+  # @return [ Integer ] the estimated number of tokens
+  def estimate_tokens(text)
+    (text.size.to_f / 3.5).ceil
+  end
 end

data/lib/context_spook/version.rb

@@ -1,6 +1,6 @@
 module ContextSpook
   # ContextSpook version
-  VERSION         = '1.5.1'
+  VERSION         = '1.6.0'
   VERSION_ARRAY   = VERSION.split('.').map(&:to_i) # :nodoc:
   VERSION_MAJOR   = VERSION_ARRAY[0] # :nodoc:
   VERSION_MINOR   = VERSION_ARRAY[1] # :nodoc:

data/spec/context_spook/generator_spec.rb

@@ -9,7 +9,10 @@ describe ContextSpook::Generator do
     it 'context can be generated from block' do
       expect_any_instance_of(described_class).to\
         receive(:output_context_size).and_call_original
+      expect_any_instance_of(described_class).to receive(:verbose_puts).
+        with(/Built.*?31.0 T tokens.*?of JSON context in total/)
       my_context = ContextSpook.generate_context do
+        self.token_estimator = -> text { (text.size.to_f / Math::PI).ceil }
         context do
           variable foo: 'bar'
           metadata version: '1.0'
@@ -89,7 +92,7 @@ describe ContextSpook::Generator do
       expect_any_instance_of(described_class).to receive(:output_context_size).
         and_call_original
       expect_any_instance_of(described_class).to receive(:verbose_puts).
-        with(/Built.*of JSON context in total/)
+        with(/Built.*?Kb.*?KT.*?of JSON context in total/)
       allow_any_instance_of(described_class::Context).to receive(:verbose_puts)
       my_context
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: context_spook
 version: !ruby/object:Gem::Version
-  version: 1.5.1
+  version: 1.6.0
 platform: ruby
 authors:
 - Florian Frank
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.17.0
+        version: 2.17.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.17.0
+        version: 2.17.1
 - !ruby/object:Gem::Dependency
   name: all_images
   requirement: !ruby/object:Gem::Requirement
@@ -250,7 +250,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.10
 specification_version: 4
 summary: context_spook collects project context for AI
 test_files: