context_spook 1.5.1 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2bb51df13e01f4262026aea154cc901997a0b5cec6566f4bc71e42467ec3dfed
-  data.tar.gz: b26ead5c765e99a931026d267e6de374dc0c93f8a5c64e46bfe09c28d892dd99
+  metadata.gz: 3992c09d3428a0268385de6fbf268f96b92ff8cc8f84cc11d464e3c1cd9c1cfe
+  data.tar.gz: d8cbd80e0ca8a3df54f464d8eb120dd8d9864b51e1aafc12b23f60e9e8b42eef
 SHA512:
-  metadata.gz: 4f8995fb526a7f6fd8be4c15c2a869775fdc5a826571bd84723689e695ca3b982325d56bf971da3982afc6cecdb2575d20d0319ea073d505d85e19a6415fdffc
-  data.tar.gz: 56bc054c3fe75f245ed97a7d92c0c91cb31dcfb1e7b101b2ee3a2092324a7290b7ec0449c5dba310b809a102e64d2bd40a656d126feef5a3d03ec588eaead314
+  metadata.gz: d186a38e1e0285498700f73d790ed0caa3fd4e19e38565c21f39657320311d1bf37bf32405ea176c44e8d7e4b8ef3978d853ce35f89a1e922ecbbf739b2dafa6
+  data.tar.gz: 9a86582d2c45a68a913ae82f64e76ca2235e96d4d3250901b49eced799ff7d236b08f869fe1dc20fe4c3453c21d35e4cb7e9c8b42c67d8ed67c7bb79d3e469f5

data/README.md

@@ -3,7 +3,10 @@
 ContextSpook is a Ruby library that collects and organizes project information
 to help AI assistants understand codebases better. It provides a
 domain-specific language (DSL) for describing project context, which can then
-be exported as structured JSON data.
+be exported as structured JSON or TOON (Token-Oriented Object Notation) data.
+
+TOON output is produced by a native encoder conformant to the TOON v4.4
+specification, validated against 186 official conformance fixtures.
 
 The DSL is general-purpose and can be used for any kind of project or
 collection of files, whether software development, documentation, research
@@ -14,7 +17,8 @@ domains.
 
 ## Documentation
 
-Complete API documentation is available at: [GitHub.io](https://flori.github.io/context_spook/)
+Complete API documentation is available at:
+[GitHub.io](https://flori.github.io/context_spook/)
 
 ## Installation
 
@@ -25,11 +29,13 @@ gem 'context_spook'
 ```
 
 And then execute:
+
 ```bash
 $ bundle install
 ```
 
 Or install it yourself as:
+
 ```bash
 $ gem install context_spook
 ```
@@ -72,7 +78,18 @@ application.
 ```ruby
 File.write 'context.json', context.to_json
 ```
+You can also generate a manifest-only context by passing `manifest: true`.
+File entries are reduced to their path plus metadata (size, tokens, MIME
+types, tags), giving the model a lightweight index of file paths it can
+use with `read_file` to load only what it needs:
 
+```ruby
+context = ContextSpook::generate_context(manifest: true) do
+  context do
+    file 'lib/context_spook.rb'
+  end
+end
+```
 #### From a context definition file
 
 Alternatively store the block's content above to a file `.contexts/project.rb`:
@@ -136,6 +153,28 @@ Or pipe directly to another tool:
 context_spook .contexts/project.rb | ollama_chat_send
 ```
 
+You can choose the output format with the `-F` flag (JSON is the default):
+
+```bash
+# Generate context as TOON
+context_spook .contexts/project.rb -F TOON > context.toon
+
+# Save TOON directly to a file
+context_spook .contexts/project.rb -F TOON -o context.toon
+```
+You can use manifest mode with `-m` to omit file content. Each file entry
+is reduced to its path plus metadata (size, tokens, MIME types, tags),
+giving the model a lightweight index of file paths it can use with
+`read_file` to load only what it needs:
+
+```bash
+# Generate manifest-only context as TOON
+context_spook .contexts/project.rb -F TOON -m > context.toon
+
+# Generate manifest-only context as JSON
+context_spook .contexts/project.rb -m > context.json
+```
+
 The CLI tool also supports verbose output:
 
 ```bash
@@ -178,7 +217,8 @@ To estimate the size of your context without generating the large output file:
 context_spook .contexts/project.rb -S
 ```
 
-This is useful for optimizing context size before committing to large output files.
+This is useful for optimizing context size before committing to large output
+files.
 
 This is how you can show the usage message:
 
@@ -192,8 +232,10 @@ context_spook -h
 The DSL collects various types of project information:
 
 - **Variables**: Key-value pairs (e.g., git branch)
-- **Files**: Content, size, line count, and tags for each file
-- **Commands**: Shell command outputs with exit codes and working directory
+- **Files**: Line-numbered content, byte size, token estimate, MIME types, line
+  count, and tags (or metadata-only in manifest mode)
+- **Commands**: Shell command outputs with size, token estimate, exit codes,
+  working directory, and tags
 - **Metadata**: Project-wide attributes (e.g., Ruby version, coverage data)
 
 ## Intended Use Case
@@ -215,9 +257,20 @@ assistants understand:
   "files": {
     "lib/context_spook.rb": {
       "namespace": "lib",
-      "content": "...",
-      "size": 1234,
-      "lines": 56,
+      "content": [
+        { "linenumber": 1, "line": "module ContextSpook" },
+        { "linenumber": 2, "line": "" },
+        { "linenumber": 3, "line": "require 'context_spook/version'" },
+        { "linenumber": 4, "line": "require 'context_spook/utils'" },
+        { "linenumber": 5, "line": "require 'context_spook/generator'" }
+      ],
+      "size": 120,
+      "tokens": 35,
+      "lines": 5,
+      "content_types": [
+        "application/x-ruby",
+        "text/x-ruby"
+      ],
       "tags": [
         "lib"
       ]
@@ -227,20 +280,58 @@ assistants understand:
     "tree": {
       "namespace": "structure",
       "output": "lib\n├── context_spook\n│   └── generator.rb\n└── context_spook.rb\n\n2 directories, 3 files",
+      "size": 88,
+      "tokens": 26,
       "exit_code": 0,
-      "working_directory": "/path/to/project"
+      "working_directory": "/path/to/project",
+      "tags": [
+        "project_structure"
+      ]
     }
   },
   "metadata": {
-    "ruby": "ruby 3.1.0 ...",
-    "code_coverage": {}
+    "ruby": "ruby 4.0.7 (2026-09-15 ...) +PRISM [x86_64-linux]",
+    "code_coverage": null
   },
   "variables": {
-    "branch": "main"
+    "branch": "master"
   }
 }
 ```
 
+The same context in TOON v4 format:
+
+```
+files:
+  "lib/context_spook.rb":
+    namespace: lib
+    content[5]{linenumber,line}:
+      1,module ContextSpook
+      2,""
+      3,require 'context_spook/version'
+      4,require 'context_spook/utils'
+      5,require 'context_spook/generator'
+    size: 120
+    tokens: 35
+    lines: 5
+    content_types[2]: application/x-ruby,text/x-ruby
+    tags[1]: lib
+commands:
+  tree:
+    namespace: structure
+    output: "lib\n├── context_spook\n│   └── generator.rb\n└── context_spook.rb\n\n2 directories, 3 files"
+    size: 88
+    tokens: 26
+    exit_code: 0
+    working_directory: /path/to/project
+    tags[1]: project_structure
+metadata:
+  ruby: "ruby 4.0.7 (2026-09-15 ...) +PRISM [x86_64-linux]"
+  code_coverage: null
+variables:
+  branch: master
+```
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](LICENSE)

data/Rakefile

@@ -31,14 +31,13 @@ GemHadar do
 
   required_ruby_version  '>= 3.1'
 
-  dependency 'tins',           '~>1.46'
-  dependency 'json',           '~>2.0'
-  dependency 'term-ansicolor', '~> 1.11'
-  dependency 'mize',           '~> 0.6'
-  dependency 'mime-types',     '~> 3.0'
-  dependency 'yaml',           '~> 0.4'
-  dependency 'pathname',       '~> 0.4'
-  dependency 'toon-ruby',      '>= 0.1'
+  dependency 'tins',                   '~> 1.46'
+  dependency 'json',                   '~> 2.0'
+  dependency 'term-ansicolor',         '~> 1.11'
+  dependency 'mize',                   '~> 0.6'
+  dependency 'mime-types',             '~> 3.0'
+  dependency 'yaml',                   '~> 0.4'
+  dependency 'pathname',               '~> 0.4'
   development_dependency 'all_images', '~> 0.6'
   development_dependency 'rspec',      '~> 3.2'
   development_dependency 'debug'

data/bin/context_spook

@@ -27,6 +27,7 @@ def usage
     -i PATTERN   Include files matching PATTERN (can be used multiple times)
                  Supports glob patterns like **, *, ?, [abc], {a,b,c}
     -F FORMAT    Output context as format JSON (the default) or TOON
+    -m           Manifest mode: omit file content, carry only metadata
     -h           Show this help message
 
   Examples:
@@ -49,19 +50,21 @@ def usage
     # Generate informational output only to estimate context size
     #{File.basename($0)} .contexts/project.rb -S
 
+    # Generate context in manifest mode (metadata only, no file content)
+    #{File.basename($0)} .contexts/project.rb -m
     # Generate context and save to file
     #{File.basename($0)} .contexts/project.rb -o context.json
   EOT
   exit 0
 end
 
-opts    = go 'o:i:F:pvSh', defaults: { ?v => true }
+opts    = go 'o:i:F:mvSh', defaults: { ?v => true }
 opts[?h] and usage
 context = nil
 output  = nil
 
 if opts[?i]
-  context = ContextSpook.generate_context(verbose: opts[?v], format: opts[?F]) do
+  context = ContextSpook.generate_context(verbose: opts[?v], format: opts[?F], manifest: opts[?m]) do
     context do
       opts[?i].to_a.each do |glob|
         glob = File.expand_path(glob)
@@ -74,7 +77,7 @@ if opts[?i]
   end
 else
   filename = ARGV.shift or fail 'require context definition file as an argument'
-  context = ContextSpook.generate_context(filename, verbose: opts[?v], format: opts[?F])
+  context = ContextSpook.generate_context(filename, verbose: opts[?v], format: opts[?F], manifest: opts[?m])
 end
 
 output = STDOUT

data/context_spook.gemspec

@@ -1,9 +1,9 @@
 # -*- encoding: utf-8 -*-
-# stub: context_spook 1.5.1 ruby lib
+# stub: context_spook 1.7.0 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "context_spook".freeze
-  s.version = "1.5.1".freeze
+  s.version = "1.7.0".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -12,19 +12,19 @@ Gem::Specification.new do |s|
   s.description = "context_spook is a library that collects and organizes project\ninformation to help AI assistants understand codebases better.\n".freeze
   s.email = "flori@ping.de".freeze
   s.executables = ["context_spook".freeze]
-  s.extra_rdoc_files = ["README.md".freeze, "lib/context_spook.rb".freeze, "lib/context_spook/generator.rb".freeze, "lib/context_spook/output_context.rb".freeze, "lib/context_spook/toon.rb".freeze, "lib/context_spook/utils.rb".freeze, "lib/context_spook/verbose_puts.rb".freeze, "lib/context_spook/version.rb".freeze]
-  s.files = [".contexts/project.rb".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "bin/context_spook".freeze, "context_spook.gemspec".freeze, "hello_world.json".freeze, "hey_world.yaml".freeze, "lib/context_spook.rb".freeze, "lib/context_spook/generator.rb".freeze, "lib/context_spook/output_context.rb".freeze, "lib/context_spook/toon.rb".freeze, "lib/context_spook/toon/example.md".freeze, "lib/context_spook/utils.rb".freeze, "lib/context_spook/verbose_puts.rb".freeze, "lib/context_spook/version.rb".freeze, "spec/context_spook/generator_spec.rb".freeze, "spec/context_spook/toon_spec.rb".freeze, "spec/spec_helper.rb".freeze]
+  s.extra_rdoc_files = ["README.md".freeze, "lib/context_spook.rb".freeze, "lib/context_spook/generator.rb".freeze, "lib/context_spook/output_context.rb".freeze, "lib/context_spook/toon.rb".freeze, "lib/context_spook/toon/encoder.rb".freeze, "lib/context_spook/utils.rb".freeze, "lib/context_spook/verbose_puts.rb".freeze, "lib/context_spook/version.rb".freeze]
+  s.files = [".contexts/project.rb".freeze, "Gemfile".freeze, "LICENSE".freeze, "README.md".freeze, "Rakefile".freeze, "bin/context_spook".freeze, "context_spook.gemspec".freeze, "hello_world.json".freeze, "hey_world.yaml".freeze, "lib/context_spook.rb".freeze, "lib/context_spook/generator.rb".freeze, "lib/context_spook/output_context.rb".freeze, "lib/context_spook/toon.rb".freeze, "lib/context_spook/toon/encoder.rb".freeze, "lib/context_spook/toon/example.md".freeze, "lib/context_spook/utils.rb".freeze, "lib/context_spook/verbose_puts.rb".freeze, "lib/context_spook/version.rb".freeze, "spec/context_spook/generator_spec.rb".freeze, "spec/context_spook/toon_encoder_spec.rb".freeze, "spec/context_spook/toon_spec.rb".freeze, "spec/examples/README.md".freeze, "spec/examples/conversions/keyed-tabular-objects.json".freeze, "spec/examples/conversions/keyed-tabular-objects.toon".freeze, "spec/examples/conversions/nested-field-groups.json".freeze, "spec/examples/conversions/nested-field-groups.toon".freeze, "spec/examples/conversions/tabular-arrays.json".freeze, "spec/examples/conversions/tabular-arrays.toon".freeze, "spec/examples/invalid/delimiter-mismatch.toon".freeze, "spec/examples/invalid/length-mismatch.toon".freeze, "spec/examples/invalid/multiple-root-primitives.toon".freeze, "spec/examples/valid/comments.toon".freeze, "spec/examples/valid/delimiter-scoping.toon".freeze, "spec/examples/valid/mixed-arrays.toon".freeze, "spec/examples/valid/nested-objects.toon".freeze, "spec/examples/valid/objects.toon".freeze, "spec/examples/valid/pipe-delimiter.toon".freeze, "spec/examples/valid/primitive-arrays.toon".freeze, "spec/examples/valid/tab-delimiter.toon".freeze, "spec/fixtures/encode/arrays-nested.json".freeze, "spec/fixtures/encode/arrays-objects.json".freeze, "spec/fixtures/encode/arrays-primitive.json".freeze, "spec/fixtures/encode/arrays-tabular.json".freeze, "spec/fixtures/encode/delimiters.json".freeze, "spec/fixtures/encode/objects-keyed.json".freeze, "spec/fixtures/encode/objects.json".freeze, "spec/fixtures/encode/primitives.json".freeze, "spec/fixtures/encode/whitespace.json".freeze, "spec/spec_helper.rb".freeze]
   s.homepage = "https://github.com/flori/context_spook".freeze
   s.licenses = ["MIT".freeze]
   s.rdoc_options = ["--title".freeze, "ContextSpook - context_spook collects project context for AI".freeze, "--main".freeze, "README.md".freeze]
   s.required_ruby_version = Gem::Requirement.new(">= 3.1".freeze)
-  s.rubygems_version = "4.0.3".freeze
+  s.rubygems_version = "4.0.22".freeze
   s.summary = "context_spook collects project context for AI".freeze
-  s.test_files = ["spec/context_spook/generator_spec.rb".freeze, "spec/context_spook/toon_spec.rb".freeze, "spec/spec_helper.rb".freeze]
+  s.test_files = ["spec/context_spook/generator_spec.rb".freeze, "spec/context_spook/toon_encoder_spec.rb".freeze, "spec/context_spook/toon_spec.rb".freeze, "spec/spec_helper.rb".freeze]
 
   s.specification_version = 4
 
-  s.add_development_dependency(%q<gem_hadar>.freeze, [">= 2.17.0".freeze])
+  s.add_development_dependency(%q<gem_hadar>.freeze, [">= 2.18.0".freeze])
   s.add_development_dependency(%q<all_images>.freeze, ["~> 0.6".freeze])
   s.add_development_dependency(%q<rspec>.freeze, ["~> 3.2".freeze])
   s.add_development_dependency(%q<debug>.freeze, [">= 0".freeze])
@@ -36,5 +36,4 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency(%q<mime-types>.freeze, ["~> 3.0".freeze])
   s.add_runtime_dependency(%q<yaml>.freeze, ["~> 0.4".freeze])
   s.add_runtime_dependency(%q<pathname>.freeze, ["~> 0.4".freeze])
-  s.add_runtime_dependency(%q<toon-ruby>.freeze, [">= 0.1".freeze])
 end

data/lib/context_spook/generator.rb

@@ -28,6 +28,8 @@ module ContextSpook
   #   during processing
   # @param format [ String, nil ] the output format for the context, either
   #   'JSON' or 'TOON'
+  # @param manifest [ TrueClass, FalseClass ] if true, file entries
+  #   omit their content and only carry metadata
   # @param block [ Proc ] a block containing the context definition, used if
   #   filename is nil
   #
@@ -37,15 +39,18 @@ module ContextSpook
   # @raise [ ArgumentError ] if neither a filename nor a block is provided
   # @raise [ ArgumentError ] if an invalid format is specified
   # @raise [ ArgumentError ] if the context definition file cannot be read or parsed
-  def self.generate_context(filename = nil, verbose: false, format: nil, &block)
+  def self.generate_context(filename = nil, verbose: false, format: nil,
+                            manifest: false, &block)
     verbose = !!verbose
     filename.present? ^ block or
       raise ArgumentError, 'need either a filename or a &block argument'
-    generator = if filename
-                  Generator.send(:new, verbose:, format:).send(:parse, File.read(filename))
-                else
-                  Generator.send(:new, verbose:, format:, &block)
-                end
+    generator =
+      if filename
+        source = File.read(filename)
+        Generator.send(:new, verbose:, format:, manifest:).send(:parse, source)
+      else
+        Generator.send(:new, verbose:, format:, manifest:, &block)
+      end
     generator.output_context_size
     generator.context
   end
@@ -70,8 +75,21 @@ module ContextSpook
     # @return [ String ] the format identifier, either 'JSON' or 'TOON'
     attr_reader :format
 
+    # The manifest method returns whether the generator is in manifest mode,
+    # where file entries omit their content.
+    #
+    # @return [ TrueClass, FalseClass ] true if in manifest mode
+    attr_reader :manifest
+
     private_class_method :new
 
+    # The token_estimator attribute allows setting the logic used to estimate
+    # the number of tokens in a serialized context.
+    #
+    # It expects an object that responds to #call(text), where text is the
+    # serialized context string.
+    attr_writer :token_estimator
+
     # The initialize method sets up the object by evaluating the provided block
     # in the object's context.
     #
@@ -79,12 +97,14 @@ module ContextSpook
     #   output during processing, defaults to lfalse.
     # @param block [ Proc ] a block of code to be evaluated within the object's context
     #                       If no block is given, the method does nothing.
-    def initialize(verbose: false, format: nil, &block)
-      @verbose = !!verbose
-      @format  = (format || 'JSON').upcase
+    def initialize(verbose: false, format: nil, manifest: false, &block)
+      @verbose  = !!verbose
+      @format   = (format || 'JSON').upcase
+      @manifest = !!manifest
       %w[ TOON JSON ].include?(@format) or
         raise ArgumentError,
           "format needs to be either JSON or TOON, was #{@format.inspect}"
+      self.token_estimator = ContextSpook::Utils.method(:estimate_tokens)
       block and instance_eval(&block)
     end
 
@@ -111,21 +131,18 @@ module ContextSpook
       include Tins::DSLAccessor
       include Term::ANSIColor
       include ContextSpook::TOON
+      include ContextSpook::Utils
 
       # The initialize method sets up the object by evaluating the provided block
       # in the object's context.
       #
-      # @param verbose [ TrueClass, FalseClass ] flag to enable verbose output
-      #   during processing, defaults to false.
       # @param block [ Proc ] a block of code to be evaluated within the object's context
       #                       If no block is given, the method does nothing.
       def initialize(generator:, &block)
         @generator = generator
         block and instance_eval(&block)
         meta(format: generator.format)
-        if generator.format == 'TOON'
-          meta(toon_example: toon_example)
-        end
+        meta(manifest: true) if generator.manifest
       end
 
       # The generator method returns the generator object associated with this
@@ -137,6 +154,8 @@ module ContextSpook
 
       delegate :verbose, to: :generator
 
+      delegate :manifest, to: :generator
+
       # The namespace method creates a scoped block with a given name.
       #
       # @param name [ Object ] the name to scope the block with
@@ -197,11 +216,10 @@ module ContextSpook
       #
       # @param filename [ String ] the path to the JSON file to be read and parsed
       #
-      # @return [ Object, nil ] the parsed JSON data structure or nil if the file cannot be read
+      # @return [ Object, nil ] the parsed JSON data structure or nil if the
+      #   file cannot be read
       def json(filename)
-        file_size = Tins::Unit.format(
-          File.size(filename), format: '%.2f %U', unit: ?b, prefix: 1024
-        )
+        file_size = format_size(File.size(filename))
         verbose_puts "Read #{filename.inspect} as JSON (%s) for context." % file_size
         JSON.load_file(filename)
       rescue Errno::ENOENT, JSON::ParserError => e
@@ -223,9 +241,7 @@ module ContextSpook
       # @return [ Object, nil ] the parsed YAML data structure or nil if the
       #   file cannot be read
       def yaml(filename)
-        file_size = Tins::Unit.format(
-          File.size(filename), format: '%.2f %U', unit: ?b, prefix: 1024
-        )
+        file_size = format_size(File.size(filename))
         verbose_puts "Read #{filename.inspect} as YAML (%s) for context." % file_size
         YAML.load_file(filename)
       rescue Errno::ENOENT, Psych::SyntaxError => e
@@ -244,32 +260,51 @@ module ContextSpook
       # content.
       #
       # It reads the specified file and creates an entry in the files hash with
-      # the file's content, along with its namespace and optional tags.
+      # the file's content, along with its namespace and optional tags. In
+      # manifest mode, only metadata fields are stored.
       #
       # @param filename [ String ] the path to the file to be read and stored
       # @param tags [ Array<String>, nil ] optional array of tags to associate with the file
       #
       # @return [ Hash ] the created file entry with content, namespace, and tags
       def file(filename, tags: nil)
-        content = File.read(filename)
-        files[filename] = {
-          namespace: scope_top,
-          content:,
-          size: content.size,
-          lines: content.lines.size,
-          content_types: MIME::Types.type_for(filename).map(&:content_type).full?,
-          tags: (Array(tags) if tags),
-        }.compact
-        file_size = Tins::Unit.format(
-          content.size, format: '%.2f %U', unit: ?b, prefix: 1024
-        )
-        verbose_puts "Read #{filename.inspect} (%s) for context." % file_size
+        size   = File.size(filename)
+        tokens = estimate_tokens(size)
+        if manifest
+          files[filename] = {
+            namespace:     scope_top,
+            size:          ,
+            tokens:        ,
+            content_types: MIME::Types.type_for(filename)
+                             .map(&:content_type).full?,
+            tags:          (Array(tags) if tags),
+          }.compact
+        else
+          content        = File.read(filename)
+          content_lines  = content.lines
+          numbered_lines = content_lines.each_with_index.map {
+            { linenumber: _2 + 1, line: _1.chomp }
+          }
+          files[filename] = {
+            namespace:     scope_top,
+            content:       numbered_lines,
+            size:          ,
+            tokens:        ,
+            lines:         content_lines.size,
+            content_types: MIME::Types.type_for(filename)
+                             .map(&:content_type).full?,
+            tags:          (Array(tags) if tags),
+          }.compact
+        end
+        verbose_puts "Read #{filename.inspect} (%s, %s) for context." %
+          [ format_size(size), format_tokens(tokens) ]
         nil
       rescue Errno::ENOENT => e
         verbose_puts color(208) { "Reading #{filename.inspect} caused #{e.class}: #{e}" }
       end
 
-      # The commands method sets up a DSL accessor for provided command outputs.
+      # The commands method sets up a DSL accessor for provided command
+      # outputs.
       #
       # @param value [ Hash ] the hash of commands to be set
       dsl_accessor :commands do
@@ -278,8 +313,8 @@ module ContextSpook
 
       # The command method executes a shell command and stores its result.
       #
-      # This method runs a given shell command and records the output, exit code,
-      # working directory, and optional tags in the commands hash.
+      # This method runs a given shell command and records the output, exit
+      # code, working directory, and optional tags in the commands hash.
       #
       # @param shell_command [ String ] the shell command to execute
       # @param tags [ Array<String>, nil ] optional array of tags to associate
@@ -295,17 +330,20 @@ module ContextSpook
             "Executing #{shell_command.inspect} resulted in exit code #{exit_code}."
           }
         end
+        output_size   = output.bytesize
+        output_tokens = estimate_tokens(output_size)
         commands[shell_command] = {
-          namespace: scope_top,
+          namespace:         scope_top,
           output:,
+          size:              output_size,
+          tokens:            output_tokens,
           exit_code:,
           working_directory: Dir.pwd,
-          tags: (Array(tags) if tags),
+          tags:              (Array(tags) if tags),
         }.compact
-        output_size = Tins::Unit.format(
-          output.size, format: '%.2f %U', unit: ?b, prefix: 1024
-        )
-        verbose_puts "Executed #{shell_command.inspect} with output (%s) for context." % output_size
+        verbose_puts "Executed #{shell_command.inspect} with output "\
+          "(%s, %s) for context." %
+          [ format_size(output_size), format_tokens(output_tokens) ]
         nil
       end
 
@@ -343,7 +381,7 @@ module ContextSpook
       # @return [ Integer ] the size in bytes of the JSON representation of the
       # context
       def size
-        to_json.size
+        to_json.bytesize
       end
     end
 

data/lib/context_spook/output_context.rb

@@ -5,16 +5,21 @@
 # functionality for generated context objects.
 module ContextSpook::OutputContext
   # The output_context_size method prints the total size of the generated
-  # context JSON representation.
+  # context representation and an estimated token count.
   #
   # This method calculates the size of the context object when serialized to
-  # JSON, formats it using binary units (KiB, MiB, etc.), and outputs the
-  # result to standard error.
+  # JSON (or TOON), formats it using binary units, estimates the token count
+  # based on bytes, and outputs the result to standard error.
   def output_context_size
-    context_size =
-      (@format == 'TOON' ? @context&.toon_size : @context&.size).to_i
-    context_size = ContextSpook::Utils.format_size(context_size)
-    verbose_puts "Built #{context_size} of #@format context in total."
+    context          = ''
+    if @context
+      context        = @format == 'TOON' ? @context.to_toon : @context.to_json
+    end
+    formatted_size   = ContextSpook::Utils.format_size(context.bytesize)
+    tokens           = @token_estimator.call(context)
+    formatted_tokens = ContextSpook::Utils.format_tokens(tokens)
+
+    verbose_puts "Built #{formatted_size} (#{formatted_tokens} tokens) of #@format context in total."
   end
 
   # The output_context method writes the generated context to the specified
@@ -28,7 +33,7 @@ module ContextSpook::OutputContext
   #   defaults to STDOUT
   def output_context(output: STDOUT)
     if @format == 'TOON'
-      output.puts(@context.to_toon)
+      @context.to_toon_stream(output)
     else
       JSON.dump(@context.as_json, output)
     end