swarm_memory 2.1.4 → 2.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. checksums.yaml +4 -4
  2. data/lib/swarm_memory/version.rb +1 -1
  3. data/lib/swarm_memory.rb +7 -2
  4. metadata +6 -185
  5. data/lib/claude_swarm/base_executor.rb +0 -133
  6. data/lib/claude_swarm/claude_code_executor.rb +0 -349
  7. data/lib/claude_swarm/claude_mcp_server.rb +0 -78
  8. data/lib/claude_swarm/cli.rb +0 -697
  9. data/lib/claude_swarm/commands/ps.rb +0 -215
  10. data/lib/claude_swarm/commands/show.rb +0 -139
  11. data/lib/claude_swarm/configuration.rb +0 -373
  12. data/lib/claude_swarm/hooks/session_start_hook.rb +0 -42
  13. data/lib/claude_swarm/json_handler.rb +0 -91
  14. data/lib/claude_swarm/mcp_generator.rb +0 -243
  15. data/lib/claude_swarm/openai/chat_completion.rb +0 -256
  16. data/lib/claude_swarm/openai/executor.rb +0 -256
  17. data/lib/claude_swarm/openai/responses.rb +0 -319
  18. data/lib/claude_swarm/orchestrator.rb +0 -878
  19. data/lib/claude_swarm/process_tracker.rb +0 -78
  20. data/lib/claude_swarm/session_cost_calculator.rb +0 -209
  21. data/lib/claude_swarm/session_path.rb +0 -42
  22. data/lib/claude_swarm/settings_generator.rb +0 -77
  23. data/lib/claude_swarm/system_utils.rb +0 -46
  24. data/lib/claude_swarm/templates/generation_prompt.md.erb +0 -230
  25. data/lib/claude_swarm/tools/reset_session_tool.rb +0 -24
  26. data/lib/claude_swarm/tools/session_info_tool.rb +0 -24
  27. data/lib/claude_swarm/tools/task_tool.rb +0 -63
  28. data/lib/claude_swarm/version.rb +0 -5
  29. data/lib/claude_swarm/worktree_manager.rb +0 -475
  30. data/lib/claude_swarm/yaml_loader.rb +0 -22
  31. data/lib/claude_swarm.rb +0 -67
  32. data/lib/swarm_cli/cli.rb +0 -201
  33. data/lib/swarm_cli/command_registry.rb +0 -61
  34. data/lib/swarm_cli/commands/mcp_serve.rb +0 -130
  35. data/lib/swarm_cli/commands/mcp_tools.rb +0 -148
  36. data/lib/swarm_cli/commands/migrate.rb +0 -55
  37. data/lib/swarm_cli/commands/run.rb +0 -173
  38. data/lib/swarm_cli/config_loader.rb +0 -98
  39. data/lib/swarm_cli/formatters/human_formatter.rb +0 -781
  40. data/lib/swarm_cli/formatters/json_formatter.rb +0 -51
  41. data/lib/swarm_cli/interactive_repl.rb +0 -924
  42. data/lib/swarm_cli/mcp_serve_options.rb +0 -44
  43. data/lib/swarm_cli/mcp_tools_options.rb +0 -59
  44. data/lib/swarm_cli/migrate_options.rb +0 -54
  45. data/lib/swarm_cli/migrator.rb +0 -132
  46. data/lib/swarm_cli/options.rb +0 -151
  47. data/lib/swarm_cli/ui/components/agent_badge.rb +0 -33
  48. data/lib/swarm_cli/ui/components/content_block.rb +0 -120
  49. data/lib/swarm_cli/ui/components/divider.rb +0 -57
  50. data/lib/swarm_cli/ui/components/panel.rb +0 -62
  51. data/lib/swarm_cli/ui/components/usage_stats.rb +0 -70
  52. data/lib/swarm_cli/ui/formatters/cost.rb +0 -49
  53. data/lib/swarm_cli/ui/formatters/number.rb +0 -58
  54. data/lib/swarm_cli/ui/formatters/text.rb +0 -77
  55. data/lib/swarm_cli/ui/formatters/time.rb +0 -73
  56. data/lib/swarm_cli/ui/icons.rb +0 -36
  57. data/lib/swarm_cli/ui/renderers/event_renderer.rb +0 -188
  58. data/lib/swarm_cli/ui/state/agent_color_cache.rb +0 -45
  59. data/lib/swarm_cli/ui/state/depth_tracker.rb +0 -40
  60. data/lib/swarm_cli/ui/state/spinner_manager.rb +0 -170
  61. data/lib/swarm_cli/ui/state/usage_tracker.rb +0 -62
  62. data/lib/swarm_cli/version.rb +0 -5
  63. data/lib/swarm_cli.rb +0 -46
  64. data/lib/swarm_sdk/agent/RETRY_LOGIC.md +0 -127
  65. data/lib/swarm_sdk/agent/builder.rb +0 -552
  66. data/lib/swarm_sdk/agent/chat.rb +0 -774
  67. data/lib/swarm_sdk/agent/chat_helpers/context_tracker.rb +0 -268
  68. data/lib/swarm_sdk/agent/chat_helpers/event_emitter.rb +0 -204
  69. data/lib/swarm_sdk/agent/chat_helpers/hook_integration.rb +0 -480
  70. data/lib/swarm_sdk/agent/chat_helpers/instrumentation.rb +0 -78
  71. data/lib/swarm_sdk/agent/chat_helpers/llm_configuration.rb +0 -233
  72. data/lib/swarm_sdk/agent/chat_helpers/logging_helpers.rb +0 -116
  73. data/lib/swarm_sdk/agent/chat_helpers/serialization.rb +0 -83
  74. data/lib/swarm_sdk/agent/chat_helpers/system_reminder_injector.rb +0 -136
  75. data/lib/swarm_sdk/agent/chat_helpers/system_reminders.rb +0 -79
  76. data/lib/swarm_sdk/agent/chat_helpers/token_tracking.rb +0 -98
  77. data/lib/swarm_sdk/agent/context.rb +0 -116
  78. data/lib/swarm_sdk/agent/context_manager.rb +0 -315
  79. data/lib/swarm_sdk/agent/definition.rb +0 -477
  80. data/lib/swarm_sdk/agent/llm_instrumentation_middleware.rb +0 -182
  81. data/lib/swarm_sdk/agent/system_prompt_builder.rb +0 -161
  82. data/lib/swarm_sdk/builders/base_builder.rb +0 -409
  83. data/lib/swarm_sdk/claude_code_agent_adapter.rb +0 -205
  84. data/lib/swarm_sdk/concerns/cleanupable.rb +0 -39
  85. data/lib/swarm_sdk/concerns/snapshotable.rb +0 -67
  86. data/lib/swarm_sdk/concerns/validatable.rb +0 -55
  87. data/lib/swarm_sdk/configuration/parser.rb +0 -353
  88. data/lib/swarm_sdk/configuration/translator.rb +0 -255
  89. data/lib/swarm_sdk/configuration.rb +0 -135
  90. data/lib/swarm_sdk/context_compactor/metrics.rb +0 -147
  91. data/lib/swarm_sdk/context_compactor/token_counter.rb +0 -106
  92. data/lib/swarm_sdk/context_compactor.rb +0 -335
  93. data/lib/swarm_sdk/context_management/builder.rb +0 -128
  94. data/lib/swarm_sdk/context_management/context.rb +0 -328
  95. data/lib/swarm_sdk/defaults.rb +0 -196
  96. data/lib/swarm_sdk/events_to_messages.rb +0 -199
  97. data/lib/swarm_sdk/hooks/adapter.rb +0 -359
  98. data/lib/swarm_sdk/hooks/context.rb +0 -197
  99. data/lib/swarm_sdk/hooks/definition.rb +0 -80
  100. data/lib/swarm_sdk/hooks/error.rb +0 -29
  101. data/lib/swarm_sdk/hooks/executor.rb +0 -146
  102. data/lib/swarm_sdk/hooks/registry.rb +0 -147
  103. data/lib/swarm_sdk/hooks/result.rb +0 -150
  104. data/lib/swarm_sdk/hooks/shell_executor.rb +0 -255
  105. data/lib/swarm_sdk/hooks/tool_call.rb +0 -35
  106. data/lib/swarm_sdk/hooks/tool_result.rb +0 -62
  107. data/lib/swarm_sdk/log_collector.rb +0 -227
  108. data/lib/swarm_sdk/log_stream.rb +0 -127
  109. data/lib/swarm_sdk/markdown_parser.rb +0 -75
  110. data/lib/swarm_sdk/model_aliases.json +0 -8
  111. data/lib/swarm_sdk/models.json +0 -1
  112. data/lib/swarm_sdk/models.rb +0 -120
  113. data/lib/swarm_sdk/node_context.rb +0 -245
  114. data/lib/swarm_sdk/observer/builder.rb +0 -81
  115. data/lib/swarm_sdk/observer/config.rb +0 -45
  116. data/lib/swarm_sdk/observer/manager.rb +0 -236
  117. data/lib/swarm_sdk/patterns/agent_observer.rb +0 -160
  118. data/lib/swarm_sdk/permissions/config.rb +0 -239
  119. data/lib/swarm_sdk/permissions/error_formatter.rb +0 -121
  120. data/lib/swarm_sdk/permissions/path_matcher.rb +0 -35
  121. data/lib/swarm_sdk/permissions/validator.rb +0 -173
  122. data/lib/swarm_sdk/permissions_builder.rb +0 -122
  123. data/lib/swarm_sdk/plugin.rb +0 -309
  124. data/lib/swarm_sdk/plugin_registry.rb +0 -101
  125. data/lib/swarm_sdk/proc_helpers.rb +0 -53
  126. data/lib/swarm_sdk/prompts/base_system_prompt.md.erb +0 -117
  127. data/lib/swarm_sdk/restore_result.rb +0 -65
  128. data/lib/swarm_sdk/result.rb +0 -123
  129. data/lib/swarm_sdk/snapshot.rb +0 -156
  130. data/lib/swarm_sdk/snapshot_from_events.rb +0 -397
  131. data/lib/swarm_sdk/state_restorer.rb +0 -476
  132. data/lib/swarm_sdk/state_snapshot.rb +0 -334
  133. data/lib/swarm_sdk/swarm/agent_initializer.rb +0 -683
  134. data/lib/swarm_sdk/swarm/all_agents_builder.rb +0 -167
  135. data/lib/swarm_sdk/swarm/builder.rb +0 -249
  136. data/lib/swarm_sdk/swarm/executor.rb +0 -213
  137. data/lib/swarm_sdk/swarm/hook_triggers.rb +0 -150
  138. data/lib/swarm_sdk/swarm/logging_callbacks.rb +0 -340
  139. data/lib/swarm_sdk/swarm/mcp_configurator.rb +0 -154
  140. data/lib/swarm_sdk/swarm/swarm_registry_builder.rb +0 -67
  141. data/lib/swarm_sdk/swarm/tool_configurator.rb +0 -358
  142. data/lib/swarm_sdk/swarm.rb +0 -717
  143. data/lib/swarm_sdk/swarm_loader.rb +0 -145
  144. data/lib/swarm_sdk/swarm_registry.rb +0 -136
  145. data/lib/swarm_sdk/tools/bash.rb +0 -282
  146. data/lib/swarm_sdk/tools/clock.rb +0 -44
  147. data/lib/swarm_sdk/tools/delegate.rb +0 -267
  148. data/lib/swarm_sdk/tools/document_converters/base_converter.rb +0 -83
  149. data/lib/swarm_sdk/tools/document_converters/docx_converter.rb +0 -99
  150. data/lib/swarm_sdk/tools/document_converters/html_converter.rb +0 -101
  151. data/lib/swarm_sdk/tools/document_converters/pdf_converter.rb +0 -78
  152. data/lib/swarm_sdk/tools/document_converters/xlsx_converter.rb +0 -194
  153. data/lib/swarm_sdk/tools/edit.rb +0 -145
  154. data/lib/swarm_sdk/tools/glob.rb +0 -166
  155. data/lib/swarm_sdk/tools/grep.rb +0 -235
  156. data/lib/swarm_sdk/tools/image_extractors/docx_image_extractor.rb +0 -43
  157. data/lib/swarm_sdk/tools/image_extractors/pdf_image_extractor.rb +0 -163
  158. data/lib/swarm_sdk/tools/image_formats/tiff_builder.rb +0 -65
  159. data/lib/swarm_sdk/tools/multi_edit.rb +0 -236
  160. data/lib/swarm_sdk/tools/path_resolver.rb +0 -92
  161. data/lib/swarm_sdk/tools/read.rb +0 -261
  162. data/lib/swarm_sdk/tools/registry.rb +0 -205
  163. data/lib/swarm_sdk/tools/scratchpad/scratchpad_list.rb +0 -117
  164. data/lib/swarm_sdk/tools/scratchpad/scratchpad_read.rb +0 -97
  165. data/lib/swarm_sdk/tools/scratchpad/scratchpad_write.rb +0 -108
  166. data/lib/swarm_sdk/tools/stores/read_tracker.rb +0 -96
  167. data/lib/swarm_sdk/tools/stores/scratchpad_storage.rb +0 -272
  168. data/lib/swarm_sdk/tools/stores/storage.rb +0 -142
  169. data/lib/swarm_sdk/tools/stores/todo_manager.rb +0 -65
  170. data/lib/swarm_sdk/tools/think.rb +0 -98
  171. data/lib/swarm_sdk/tools/todo_write.rb +0 -235
  172. data/lib/swarm_sdk/tools/web_fetch.rb +0 -262
  173. data/lib/swarm_sdk/tools/write.rb +0 -112
  174. data/lib/swarm_sdk/utils.rb +0 -68
  175. data/lib/swarm_sdk/validation_result.rb +0 -33
  176. data/lib/swarm_sdk/version.rb +0 -5
  177. data/lib/swarm_sdk/workflow/agent_config.rb +0 -79
  178. data/lib/swarm_sdk/workflow/builder.rb +0 -143
  179. data/lib/swarm_sdk/workflow/executor.rb +0 -497
  180. data/lib/swarm_sdk/workflow/node_builder.rb +0 -555
  181. data/lib/swarm_sdk/workflow/transformer_executor.rb +0 -249
  182. data/lib/swarm_sdk/workflow.rb +0 -554
  183. data/lib/swarm_sdk.rb +0 -524
  184. /data/lib/swarm_memory/{errors.rb → error.rb} +0 -0
@@ -1,135 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SwarmSDK
4
- # Configuration facade that delegates to Parser and Translator
5
- #
6
- # This class maintains the public API while internally delegating to:
7
- # - Configuration::Parser - YAML parsing, validation, and normalization
8
- # - Configuration::Translator - Translation to Swarm/Workflow DSL builders
9
- #
10
- # ## Public API (unchanged)
11
- # - Configuration.load_file(path) - Load from file
12
- # - Configuration.new(yaml_content, base_dir:) - Load from string
13
- # - config.load_and_validate - Parse and validate
14
- # - config.to_swarm(allow_filesystem_tools:) - Convert to Swarm/Workflow
15
- # - config.agent_names - Get list of agent names
16
- # - config.connections_for(agent_name) - Get delegation targets
17
- #
18
- # ## Architecture
19
- # The facade pattern keeps backward compatibility while separating concerns:
20
- # - Parser handles all YAML parsing and validation logic
21
- # - Translator handles all DSL builder translation logic
22
- # - Configuration delegates to both, exposing parsed data via attr_readers
23
- class Configuration
24
- attr_reader :config_type,
25
- :swarm_name,
26
- :swarm_id,
27
- :lead_agent,
28
- :start_node,
29
- :agents,
30
- :all_agents_config,
31
- :swarm_hooks,
32
- :all_agents_hooks,
33
- :scratchpad_enabled,
34
- :nodes,
35
- :external_swarms
36
-
37
- class << self
38
- # Load configuration from YAML file
39
- #
40
- # @param path [String, Pathname] Path to YAML configuration file
41
- # @return [Configuration] Validated configuration instance
42
- # @raise [ConfigurationError] If file not found or invalid
43
- def load_file(path)
44
- path = Pathname.new(path).expand_path
45
-
46
- unless path.exist?
47
- raise ConfigurationError, "Configuration file not found: #{path}"
48
- end
49
-
50
- yaml_content = File.read(path)
51
- base_dir = path.dirname
52
-
53
- new(yaml_content, base_dir: base_dir).tap(&:load_and_validate)
54
- rescue Errno::ENOENT
55
- raise ConfigurationError, "Configuration file not found: #{path}"
56
- end
57
- end
58
-
59
- # Initialize configuration from YAML string
60
- #
61
- # @param yaml_content [String] YAML configuration content
62
- # @param base_dir [String, Pathname] Base directory for resolving agent file paths (default: Dir.pwd)
63
- def initialize(yaml_content, base_dir: Dir.pwd)
64
- raise ArgumentError, "yaml_content cannot be nil" if yaml_content.nil?
65
- raise ArgumentError, "base_dir cannot be nil" if base_dir.nil?
66
-
67
- @yaml_content = yaml_content
68
- @base_dir = Pathname.new(base_dir).expand_path
69
- @parser = nil
70
- @translator = nil
71
- end
72
-
73
- # Parse and validate YAML configuration
74
- #
75
- # Delegates to Parser for all parsing logic, then syncs parsed data
76
- # to instance variables for backward compatibility.
77
- #
78
- # @return [self]
79
- def load_and_validate
80
- @parser = Parser.new(@yaml_content, base_dir: @base_dir)
81
- @parser.parse
82
-
83
- # Sync parsed data to instance variables for backward compatibility
84
- sync_from_parser
85
-
86
- self
87
- end
88
-
89
- def agent_names
90
- @agents.keys
91
- end
92
-
93
- def connections_for(agent_name)
94
- agent_config = @agents[agent_name]
95
- return [] unless agent_config
96
-
97
- delegates = agent_config[:delegates_to] || []
98
- Array(delegates).map(&:to_sym)
99
- end
100
-
101
- # Convert configuration to Swarm or Workflow using appropriate builder
102
- #
103
- # Delegates to Translator for all DSL translation logic.
104
- #
105
- # @param allow_filesystem_tools [Boolean, nil] Whether to allow filesystem tools (nil uses global setting)
106
- # @return [Swarm, Workflow] Configured swarm or workflow
107
- def to_swarm(allow_filesystem_tools: nil)
108
- raise ConfigurationError, "Configuration not loaded. Call load_and_validate first." unless @parser
109
-
110
- @translator = Translator.new(@parser)
111
- @translator.to_swarm(allow_filesystem_tools: allow_filesystem_tools)
112
- end
113
-
114
- private
115
-
116
- # Sync parsed data from Parser to instance variables
117
- #
118
- # This maintains backward compatibility with code that accesses
119
- # @config_type, @agents, etc. directly via attr_readers.
120
- def sync_from_parser
121
- @config_type = @parser.config_type
122
- @swarm_name = @parser.swarm_name
123
- @swarm_id = @parser.swarm_id
124
- @lead_agent = @parser.lead_agent
125
- @start_node = @parser.start_node
126
- @agents = @parser.agents
127
- @all_agents_config = @parser.all_agents_config
128
- @swarm_hooks = @parser.swarm_hooks
129
- @all_agents_hooks = @parser.all_agents_hooks
130
- @external_swarms = @parser.external_swarms
131
- @nodes = @parser.nodes
132
- @scratchpad_enabled = @parser.scratchpad_mode # NOTE: attr_reader says scratchpad_enabled
133
- end
134
- end
135
- end
@@ -1,147 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SwarmSDK
4
- class ContextCompactor
5
- # Metrics tracks compression statistics
6
- #
7
- # Provides detailed information about the compression operation:
8
- # - Message counts (before/after)
9
- # - Token counts (before/after)
10
- # - Compression ratio
11
- # - Time taken
12
- # - Summary of changes
13
- #
14
- # ## Usage
15
- #
16
- # metrics = agent.compact_context
17
- # puts metrics.summary
18
- # puts "Compressed from #{metrics.original_tokens} to #{metrics.compressed_tokens} tokens"
19
- # puts "Compression ratio: #{(metrics.compression_ratio * 100).round(1)}%"
20
- #
21
- class Metrics
22
- attr_reader :original_messages, :compressed_messages, :time_taken
23
-
24
- # Initialize metrics from compression operation
25
- #
26
- # @param original_messages [Array<RubyLLM::Message>] Messages before compression
27
- # @param compressed_messages [Array<RubyLLM::Message>] Messages after compression
28
- # @param time_taken [Float] Time taken in seconds
29
- def initialize(original_messages:, compressed_messages:, time_taken:)
30
- @original_messages = original_messages
31
- @compressed_messages = compressed_messages
32
- @time_taken = time_taken
33
- end
34
-
35
- # Number of messages before compression
36
- #
37
- # @return [Integer] Original message count
38
- def original_message_count
39
- @original_messages.size
40
- end
41
-
42
- # Number of messages after compression
43
- #
44
- # @return [Integer] Compressed message count
45
- def compressed_message_count
46
- @compressed_messages.size
47
- end
48
-
49
- # Number of messages removed
50
- #
51
- # @return [Integer] Messages removed
52
- def messages_removed
53
- original_message_count - compressed_message_count
54
- end
55
-
56
- # Number of checkpoint summary messages created
57
- #
58
- # @return [Integer] Checkpoint messages
59
- def messages_summarized
60
- @compressed_messages.count do |msg|
61
- msg.role == :system && msg.content.to_s.include?("CONVERSATION CHECKPOINT")
62
- end
63
- end
64
-
65
- # Estimated tokens before compression
66
- #
67
- # @return [Integer] Original token count
68
- def original_tokens
69
- @original_tokens ||= TokenCounter.estimate_messages(@original_messages)
70
- end
71
-
72
- # Estimated tokens after compression
73
- #
74
- # @return [Integer] Compressed token count
75
- def compressed_tokens
76
- @compressed_tokens ||= TokenCounter.estimate_messages(@compressed_messages)
77
- end
78
-
79
- # Number of tokens removed
80
- #
81
- # @return [Integer] Tokens removed
82
- def tokens_removed
83
- original_tokens - compressed_tokens
84
- end
85
-
86
- # Compression ratio (compressed / original)
87
- #
88
- # @return [Float] Ratio between 0.0 and 1.0
89
- def compression_ratio
90
- return 0.0 if original_tokens.zero?
91
-
92
- compressed_tokens.to_f / original_tokens
93
- end
94
-
95
- # Compression factor (original / compressed)
96
- #
97
- # e.g., 5.0 means compressed to 1/5th of original size
98
- #
99
- # @return [Float] Compression factor
100
- def compression_factor
101
- return 0.0 if compressed_tokens.zero?
102
-
103
- original_tokens.to_f / compressed_tokens
104
- end
105
-
106
- # Compression percentage
107
- #
108
- # @return [Float] Percentage of original size (0-100)
109
- def compression_percentage
110
- (compression_ratio * 100).round(2)
111
- end
112
-
113
- # Generate a human-readable summary
114
- #
115
- # @return [String] Summary text
116
- def summary
117
- <<~SUMMARY
118
- Context Compression Results:
119
- - Messages: #{original_message_count} → #{compressed_message_count} (-#{messages_removed})
120
- - Estimated tokens: #{original_tokens} → #{compressed_tokens} (-#{tokens_removed})
121
- - Compression ratio: #{compression_factor.round(1)}:1 (#{compression_percentage}%)
122
- - Checkpoints created: #{messages_summarized}
123
- - Time taken: #{time_taken.round(3)}s
124
- SUMMARY
125
- end
126
-
127
- # Convert metrics to hash for logging
128
- #
129
- # @return [Hash] Metrics as hash
130
- def to_h
131
- {
132
- original_message_count: original_message_count,
133
- compressed_message_count: compressed_message_count,
134
- messages_removed: messages_removed,
135
- messages_summarized: messages_summarized,
136
- original_tokens: original_tokens,
137
- compressed_tokens: compressed_tokens,
138
- tokens_removed: tokens_removed,
139
- compression_ratio: compression_ratio.round(4),
140
- compression_factor: compression_factor.round(2),
141
- compression_percentage: compression_percentage,
142
- time_taken: time_taken.round(3),
143
- }
144
- end
145
- end
146
- end
147
- end
@@ -1,106 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SwarmSDK
4
- class ContextCompactor
5
- # TokenCounter provides token estimation for messages
6
- #
7
- # This uses a simple heuristic approach:
8
- # - ~4 characters per token for English prose
9
- # - ~3.5 characters per token for code
10
- #
11
- # For production use with OpenAI models, consider using the tiktoken gem
12
- # for accurate token counting. For Claude models, use Claude's token API.
13
- #
14
- # ## Usage
15
- #
16
- # tokens = TokenCounter.estimate_message(message)
17
- # total_tokens = TokenCounter.estimate_messages(messages)
18
- #
19
- class TokenCounter
20
- # Backward compatibility aliases - use Defaults module for new code
21
- CHARS_PER_TOKEN_PROSE = Defaults::TokenEstimation::CHARS_PER_TOKEN_PROSE
22
- CHARS_PER_TOKEN_CODE = Defaults::TokenEstimation::CHARS_PER_TOKEN_CODE
23
-
24
- class << self
25
- # Estimate tokens for a single message
26
- #
27
- # @param message [RubyLLM::Message] Message to estimate
28
- # @return [Integer] Estimated token count
29
- def estimate_message(message)
30
- case message.role
31
- when :user, :assistant
32
- estimate_content(message.content)
33
- when :system
34
- estimate_content(message.content)
35
- when :tool
36
- # Tool results typically have overhead
37
- base_overhead = 50
38
- content_tokens = estimate_content(message.content)
39
- base_overhead + content_tokens
40
- else
41
- # Unknown message type
42
- begin
43
- estimate_content(message.content)
44
- rescue
45
- 0
46
- end
47
- end
48
- end
49
-
50
- # Estimate tokens for multiple messages
51
- #
52
- # @param messages [Array<RubyLLM::Message>] Messages to estimate
53
- # @return [Integer] Total estimated token count
54
- def estimate_messages(messages)
55
- messages.sum { |msg| estimate_message(msg) }
56
- end
57
-
58
- # Estimate tokens for content string
59
- #
60
- # Uses heuristic to detect code vs prose and adjust accordingly.
61
- #
62
- # @param content [String, RubyLLM::Content, nil] Content to estimate
63
- # @return [Integer] Estimated token count
64
- def estimate_content(content)
65
- return 0 if content.nil?
66
-
67
- # Handle RubyLLM::Content objects
68
- text = if content.respond_to?(:to_s)
69
- content.to_s
70
- else
71
- content
72
- end
73
-
74
- return 0 if text.empty?
75
-
76
- # Detect if content is mostly code
77
- code_ratio = detect_code_ratio(text)
78
-
79
- # Choose characters per token based on content type
80
- chars_per_token = if code_ratio > 0.1
81
- CHARS_PER_TOKEN_CODE # Code
82
- else
83
- CHARS_PER_TOKEN_PROSE # Prose
84
- end
85
-
86
- (text.length / chars_per_token).ceil
87
- end
88
-
89
- private
90
-
91
- # Detect ratio of code characters to total characters
92
- #
93
- # @param text [String] Text to analyze
94
- # @return [Float] Ratio of code indicators (0.0 to 1.0)
95
- def detect_code_ratio(text)
96
- # Count code indicator characters
97
- code_chars = text.scan(/[{}()\[\];]/).length
98
-
99
- return 0.0 if text.empty?
100
-
101
- code_chars.to_f / text.length
102
- end
103
- end
104
- end
105
- end
106
- end