swarm_memory 2.1.5 → 2.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (182) hide show
  1. checksums.yaml +4 -4
  2. data/lib/swarm_memory/version.rb +1 -1
  3. metadata +5 -184
  4. data/lib/claude_swarm/base_executor.rb +0 -133
  5. data/lib/claude_swarm/claude_code_executor.rb +0 -349
  6. data/lib/claude_swarm/claude_mcp_server.rb +0 -78
  7. data/lib/claude_swarm/cli.rb +0 -697
  8. data/lib/claude_swarm/commands/ps.rb +0 -215
  9. data/lib/claude_swarm/commands/show.rb +0 -139
  10. data/lib/claude_swarm/configuration.rb +0 -373
  11. data/lib/claude_swarm/hooks/session_start_hook.rb +0 -42
  12. data/lib/claude_swarm/json_handler.rb +0 -91
  13. data/lib/claude_swarm/mcp_generator.rb +0 -230
  14. data/lib/claude_swarm/openai/chat_completion.rb +0 -256
  15. data/lib/claude_swarm/openai/executor.rb +0 -256
  16. data/lib/claude_swarm/openai/responses.rb +0 -319
  17. data/lib/claude_swarm/orchestrator.rb +0 -878
  18. data/lib/claude_swarm/process_tracker.rb +0 -78
  19. data/lib/claude_swarm/session_cost_calculator.rb +0 -209
  20. data/lib/claude_swarm/session_path.rb +0 -42
  21. data/lib/claude_swarm/settings_generator.rb +0 -77
  22. data/lib/claude_swarm/system_utils.rb +0 -46
  23. data/lib/claude_swarm/templates/generation_prompt.md.erb +0 -230
  24. data/lib/claude_swarm/tools/reset_session_tool.rb +0 -24
  25. data/lib/claude_swarm/tools/session_info_tool.rb +0 -24
  26. data/lib/claude_swarm/tools/task_tool.rb +0 -63
  27. data/lib/claude_swarm/version.rb +0 -5
  28. data/lib/claude_swarm/worktree_manager.rb +0 -475
  29. data/lib/claude_swarm/yaml_loader.rb +0 -22
  30. data/lib/claude_swarm.rb +0 -67
  31. data/lib/swarm_cli/cli.rb +0 -201
  32. data/lib/swarm_cli/command_registry.rb +0 -61
  33. data/lib/swarm_cli/commands/mcp_serve.rb +0 -130
  34. data/lib/swarm_cli/commands/mcp_tools.rb +0 -148
  35. data/lib/swarm_cli/commands/migrate.rb +0 -55
  36. data/lib/swarm_cli/commands/run.rb +0 -173
  37. data/lib/swarm_cli/config_loader.rb +0 -98
  38. data/lib/swarm_cli/formatters/human_formatter.rb +0 -781
  39. data/lib/swarm_cli/formatters/json_formatter.rb +0 -51
  40. data/lib/swarm_cli/interactive_repl.rb +0 -924
  41. data/lib/swarm_cli/mcp_serve_options.rb +0 -44
  42. data/lib/swarm_cli/mcp_tools_options.rb +0 -59
  43. data/lib/swarm_cli/migrate_options.rb +0 -54
  44. data/lib/swarm_cli/migrator.rb +0 -132
  45. data/lib/swarm_cli/options.rb +0 -151
  46. data/lib/swarm_cli/ui/components/agent_badge.rb +0 -33
  47. data/lib/swarm_cli/ui/components/content_block.rb +0 -120
  48. data/lib/swarm_cli/ui/components/divider.rb +0 -57
  49. data/lib/swarm_cli/ui/components/panel.rb +0 -62
  50. data/lib/swarm_cli/ui/components/usage_stats.rb +0 -70
  51. data/lib/swarm_cli/ui/formatters/cost.rb +0 -49
  52. data/lib/swarm_cli/ui/formatters/number.rb +0 -58
  53. data/lib/swarm_cli/ui/formatters/text.rb +0 -77
  54. data/lib/swarm_cli/ui/formatters/time.rb +0 -73
  55. data/lib/swarm_cli/ui/icons.rb +0 -36
  56. data/lib/swarm_cli/ui/renderers/event_renderer.rb +0 -188
  57. data/lib/swarm_cli/ui/state/agent_color_cache.rb +0 -45
  58. data/lib/swarm_cli/ui/state/depth_tracker.rb +0 -40
  59. data/lib/swarm_cli/ui/state/spinner_manager.rb +0 -170
  60. data/lib/swarm_cli/ui/state/usage_tracker.rb +0 -62
  61. data/lib/swarm_cli/version.rb +0 -5
  62. data/lib/swarm_cli.rb +0 -46
  63. data/lib/swarm_sdk/agent/RETRY_LOGIC.md +0 -127
  64. data/lib/swarm_sdk/agent/builder.rb +0 -552
  65. data/lib/swarm_sdk/agent/chat.rb +0 -774
  66. data/lib/swarm_sdk/agent/chat_helpers/context_tracker.rb +0 -268
  67. data/lib/swarm_sdk/agent/chat_helpers/event_emitter.rb +0 -204
  68. data/lib/swarm_sdk/agent/chat_helpers/hook_integration.rb +0 -480
  69. data/lib/swarm_sdk/agent/chat_helpers/instrumentation.rb +0 -78
  70. data/lib/swarm_sdk/agent/chat_helpers/llm_configuration.rb +0 -233
  71. data/lib/swarm_sdk/agent/chat_helpers/logging_helpers.rb +0 -116
  72. data/lib/swarm_sdk/agent/chat_helpers/serialization.rb +0 -83
  73. data/lib/swarm_sdk/agent/chat_helpers/system_reminder_injector.rb +0 -136
  74. data/lib/swarm_sdk/agent/chat_helpers/system_reminders.rb +0 -79
  75. data/lib/swarm_sdk/agent/chat_helpers/token_tracking.rb +0 -98
  76. data/lib/swarm_sdk/agent/context.rb +0 -116
  77. data/lib/swarm_sdk/agent/context_manager.rb +0 -315
  78. data/lib/swarm_sdk/agent/definition.rb +0 -477
  79. data/lib/swarm_sdk/agent/llm_instrumentation_middleware.rb +0 -182
  80. data/lib/swarm_sdk/agent/system_prompt_builder.rb +0 -161
  81. data/lib/swarm_sdk/builders/base_builder.rb +0 -409
  82. data/lib/swarm_sdk/claude_code_agent_adapter.rb +0 -205
  83. data/lib/swarm_sdk/concerns/cleanupable.rb +0 -39
  84. data/lib/swarm_sdk/concerns/snapshotable.rb +0 -67
  85. data/lib/swarm_sdk/concerns/validatable.rb +0 -55
  86. data/lib/swarm_sdk/configuration/parser.rb +0 -353
  87. data/lib/swarm_sdk/configuration/translator.rb +0 -255
  88. data/lib/swarm_sdk/configuration.rb +0 -135
  89. data/lib/swarm_sdk/context_compactor/metrics.rb +0 -147
  90. data/lib/swarm_sdk/context_compactor/token_counter.rb +0 -106
  91. data/lib/swarm_sdk/context_compactor.rb +0 -335
  92. data/lib/swarm_sdk/context_management/builder.rb +0 -128
  93. data/lib/swarm_sdk/context_management/context.rb +0 -328
  94. data/lib/swarm_sdk/defaults.rb +0 -196
  95. data/lib/swarm_sdk/events_to_messages.rb +0 -199
  96. data/lib/swarm_sdk/hooks/adapter.rb +0 -359
  97. data/lib/swarm_sdk/hooks/context.rb +0 -197
  98. data/lib/swarm_sdk/hooks/definition.rb +0 -80
  99. data/lib/swarm_sdk/hooks/error.rb +0 -29
  100. data/lib/swarm_sdk/hooks/executor.rb +0 -146
  101. data/lib/swarm_sdk/hooks/registry.rb +0 -147
  102. data/lib/swarm_sdk/hooks/result.rb +0 -150
  103. data/lib/swarm_sdk/hooks/shell_executor.rb +0 -255
  104. data/lib/swarm_sdk/hooks/tool_call.rb +0 -35
  105. data/lib/swarm_sdk/hooks/tool_result.rb +0 -62
  106. data/lib/swarm_sdk/log_collector.rb +0 -227
  107. data/lib/swarm_sdk/log_stream.rb +0 -127
  108. data/lib/swarm_sdk/markdown_parser.rb +0 -75
  109. data/lib/swarm_sdk/model_aliases.json +0 -8
  110. data/lib/swarm_sdk/models.json +0 -1
  111. data/lib/swarm_sdk/models.rb +0 -120
  112. data/lib/swarm_sdk/node_context.rb +0 -245
  113. data/lib/swarm_sdk/observer/builder.rb +0 -81
  114. data/lib/swarm_sdk/observer/config.rb +0 -45
  115. data/lib/swarm_sdk/observer/manager.rb +0 -236
  116. data/lib/swarm_sdk/patterns/agent_observer.rb +0 -160
  117. data/lib/swarm_sdk/permissions/config.rb +0 -239
  118. data/lib/swarm_sdk/permissions/error_formatter.rb +0 -121
  119. data/lib/swarm_sdk/permissions/path_matcher.rb +0 -35
  120. data/lib/swarm_sdk/permissions/validator.rb +0 -173
  121. data/lib/swarm_sdk/permissions_builder.rb +0 -122
  122. data/lib/swarm_sdk/plugin.rb +0 -309
  123. data/lib/swarm_sdk/plugin_registry.rb +0 -101
  124. data/lib/swarm_sdk/proc_helpers.rb +0 -53
  125. data/lib/swarm_sdk/prompts/base_system_prompt.md.erb +0 -117
  126. data/lib/swarm_sdk/restore_result.rb +0 -65
  127. data/lib/swarm_sdk/result.rb +0 -123
  128. data/lib/swarm_sdk/snapshot.rb +0 -156
  129. data/lib/swarm_sdk/snapshot_from_events.rb +0 -397
  130. data/lib/swarm_sdk/state_restorer.rb +0 -476
  131. data/lib/swarm_sdk/state_snapshot.rb +0 -334
  132. data/lib/swarm_sdk/swarm/agent_initializer.rb +0 -683
  133. data/lib/swarm_sdk/swarm/all_agents_builder.rb +0 -167
  134. data/lib/swarm_sdk/swarm/builder.rb +0 -249
  135. data/lib/swarm_sdk/swarm/executor.rb +0 -213
  136. data/lib/swarm_sdk/swarm/hook_triggers.rb +0 -150
  137. data/lib/swarm_sdk/swarm/logging_callbacks.rb +0 -340
  138. data/lib/swarm_sdk/swarm/mcp_configurator.rb +0 -154
  139. data/lib/swarm_sdk/swarm/swarm_registry_builder.rb +0 -67
  140. data/lib/swarm_sdk/swarm/tool_configurator.rb +0 -358
  141. data/lib/swarm_sdk/swarm.rb +0 -717
  142. data/lib/swarm_sdk/swarm_loader.rb +0 -145
  143. data/lib/swarm_sdk/swarm_registry.rb +0 -136
  144. data/lib/swarm_sdk/tools/bash.rb +0 -282
  145. data/lib/swarm_sdk/tools/clock.rb +0 -44
  146. data/lib/swarm_sdk/tools/delegate.rb +0 -267
  147. data/lib/swarm_sdk/tools/document_converters/base_converter.rb +0 -83
  148. data/lib/swarm_sdk/tools/document_converters/docx_converter.rb +0 -99
  149. data/lib/swarm_sdk/tools/document_converters/html_converter.rb +0 -101
  150. data/lib/swarm_sdk/tools/document_converters/pdf_converter.rb +0 -78
  151. data/lib/swarm_sdk/tools/document_converters/xlsx_converter.rb +0 -194
  152. data/lib/swarm_sdk/tools/edit.rb +0 -145
  153. data/lib/swarm_sdk/tools/glob.rb +0 -166
  154. data/lib/swarm_sdk/tools/grep.rb +0 -235
  155. data/lib/swarm_sdk/tools/image_extractors/docx_image_extractor.rb +0 -43
  156. data/lib/swarm_sdk/tools/image_extractors/pdf_image_extractor.rb +0 -163
  157. data/lib/swarm_sdk/tools/image_formats/tiff_builder.rb +0 -65
  158. data/lib/swarm_sdk/tools/multi_edit.rb +0 -236
  159. data/lib/swarm_sdk/tools/path_resolver.rb +0 -92
  160. data/lib/swarm_sdk/tools/read.rb +0 -261
  161. data/lib/swarm_sdk/tools/registry.rb +0 -205
  162. data/lib/swarm_sdk/tools/scratchpad/scratchpad_list.rb +0 -117
  163. data/lib/swarm_sdk/tools/scratchpad/scratchpad_read.rb +0 -97
  164. data/lib/swarm_sdk/tools/scratchpad/scratchpad_write.rb +0 -108
  165. data/lib/swarm_sdk/tools/stores/read_tracker.rb +0 -96
  166. data/lib/swarm_sdk/tools/stores/scratchpad_storage.rb +0 -272
  167. data/lib/swarm_sdk/tools/stores/storage.rb +0 -142
  168. data/lib/swarm_sdk/tools/stores/todo_manager.rb +0 -65
  169. data/lib/swarm_sdk/tools/think.rb +0 -98
  170. data/lib/swarm_sdk/tools/todo_write.rb +0 -235
  171. data/lib/swarm_sdk/tools/web_fetch.rb +0 -262
  172. data/lib/swarm_sdk/tools/write.rb +0 -112
  173. data/lib/swarm_sdk/utils.rb +0 -68
  174. data/lib/swarm_sdk/validation_result.rb +0 -33
  175. data/lib/swarm_sdk/version.rb +0 -5
  176. data/lib/swarm_sdk/workflow/agent_config.rb +0 -79
  177. data/lib/swarm_sdk/workflow/builder.rb +0 -143
  178. data/lib/swarm_sdk/workflow/executor.rb +0 -497
  179. data/lib/swarm_sdk/workflow/node_builder.rb +0 -555
  180. data/lib/swarm_sdk/workflow/transformer_executor.rb +0 -249
  181. data/lib/swarm_sdk/workflow.rb +0 -554
  182. data/lib/swarm_sdk.rb +0 -524
@@ -1,135 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SwarmSDK
4
- # Configuration facade that delegates to Parser and Translator
5
- #
6
- # This class maintains the public API while internally delegating to:
7
- # - Configuration::Parser - YAML parsing, validation, and normalization
8
- # - Configuration::Translator - Translation to Swarm/Workflow DSL builders
9
- #
10
- # ## Public API (unchanged)
11
- # - Configuration.load_file(path) - Load from file
12
- # - Configuration.new(yaml_content, base_dir:) - Load from string
13
- # - config.load_and_validate - Parse and validate
14
- # - config.to_swarm(allow_filesystem_tools:) - Convert to Swarm/Workflow
15
- # - config.agent_names - Get list of agent names
16
- # - config.connections_for(agent_name) - Get delegation targets
17
- #
18
- # ## Architecture
19
- # The facade pattern keeps backward compatibility while separating concerns:
20
- # - Parser handles all YAML parsing and validation logic
21
- # - Translator handles all DSL builder translation logic
22
- # - Configuration delegates to both, exposing parsed data via attr_readers
23
- class Configuration
24
- attr_reader :config_type,
25
- :swarm_name,
26
- :swarm_id,
27
- :lead_agent,
28
- :start_node,
29
- :agents,
30
- :all_agents_config,
31
- :swarm_hooks,
32
- :all_agents_hooks,
33
- :scratchpad_enabled,
34
- :nodes,
35
- :external_swarms
36
-
37
- class << self
38
- # Load configuration from YAML file
39
- #
40
- # @param path [String, Pathname] Path to YAML configuration file
41
- # @return [Configuration] Validated configuration instance
42
- # @raise [ConfigurationError] If file not found or invalid
43
- def load_file(path)
44
- path = Pathname.new(path).expand_path
45
-
46
- unless path.exist?
47
- raise ConfigurationError, "Configuration file not found: #{path}"
48
- end
49
-
50
- yaml_content = File.read(path)
51
- base_dir = path.dirname
52
-
53
- new(yaml_content, base_dir: base_dir).tap(&:load_and_validate)
54
- rescue Errno::ENOENT
55
- raise ConfigurationError, "Configuration file not found: #{path}"
56
- end
57
- end
58
-
59
- # Initialize configuration from YAML string
60
- #
61
- # @param yaml_content [String] YAML configuration content
62
- # @param base_dir [String, Pathname] Base directory for resolving agent file paths (default: Dir.pwd)
63
- def initialize(yaml_content, base_dir: Dir.pwd)
64
- raise ArgumentError, "yaml_content cannot be nil" if yaml_content.nil?
65
- raise ArgumentError, "base_dir cannot be nil" if base_dir.nil?
66
-
67
- @yaml_content = yaml_content
68
- @base_dir = Pathname.new(base_dir).expand_path
69
- @parser = nil
70
- @translator = nil
71
- end
72
-
73
- # Parse and validate YAML configuration
74
- #
75
- # Delegates to Parser for all parsing logic, then syncs parsed data
76
- # to instance variables for backward compatibility.
77
- #
78
- # @return [self]
79
- def load_and_validate
80
- @parser = Parser.new(@yaml_content, base_dir: @base_dir)
81
- @parser.parse
82
-
83
- # Sync parsed data to instance variables for backward compatibility
84
- sync_from_parser
85
-
86
- self
87
- end
88
-
89
- def agent_names
90
- @agents.keys
91
- end
92
-
93
- def connections_for(agent_name)
94
- agent_config = @agents[agent_name]
95
- return [] unless agent_config
96
-
97
- delegates = agent_config[:delegates_to] || []
98
- Array(delegates).map(&:to_sym)
99
- end
100
-
101
- # Convert configuration to Swarm or Workflow using appropriate builder
102
- #
103
- # Delegates to Translator for all DSL translation logic.
104
- #
105
- # @param allow_filesystem_tools [Boolean, nil] Whether to allow filesystem tools (nil uses global setting)
106
- # @return [Swarm, Workflow] Configured swarm or workflow
107
- def to_swarm(allow_filesystem_tools: nil)
108
- raise ConfigurationError, "Configuration not loaded. Call load_and_validate first." unless @parser
109
-
110
- @translator = Translator.new(@parser)
111
- @translator.to_swarm(allow_filesystem_tools: allow_filesystem_tools)
112
- end
113
-
114
- private
115
-
116
- # Sync parsed data from Parser to instance variables
117
- #
118
- # This maintains backward compatibility with code that accesses
119
- # @config_type, @agents, etc. directly via attr_readers.
120
- def sync_from_parser
121
- @config_type = @parser.config_type
122
- @swarm_name = @parser.swarm_name
123
- @swarm_id = @parser.swarm_id
124
- @lead_agent = @parser.lead_agent
125
- @start_node = @parser.start_node
126
- @agents = @parser.agents
127
- @all_agents_config = @parser.all_agents_config
128
- @swarm_hooks = @parser.swarm_hooks
129
- @all_agents_hooks = @parser.all_agents_hooks
130
- @external_swarms = @parser.external_swarms
131
- @nodes = @parser.nodes
132
- @scratchpad_enabled = @parser.scratchpad_mode # NOTE: attr_reader says scratchpad_enabled
133
- end
134
- end
135
- end
@@ -1,147 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SwarmSDK
4
- class ContextCompactor
5
- # Metrics tracks compression statistics
6
- #
7
- # Provides detailed information about the compression operation:
8
- # - Message counts (before/after)
9
- # - Token counts (before/after)
10
- # - Compression ratio
11
- # - Time taken
12
- # - Summary of changes
13
- #
14
- # ## Usage
15
- #
16
- # metrics = agent.compact_context
17
- # puts metrics.summary
18
- # puts "Compressed from #{metrics.original_tokens} to #{metrics.compressed_tokens} tokens"
19
- # puts "Compression ratio: #{(metrics.compression_ratio * 100).round(1)}%"
20
- #
21
- class Metrics
22
- attr_reader :original_messages, :compressed_messages, :time_taken
23
-
24
- # Initialize metrics from compression operation
25
- #
26
- # @param original_messages [Array<RubyLLM::Message>] Messages before compression
27
- # @param compressed_messages [Array<RubyLLM::Message>] Messages after compression
28
- # @param time_taken [Float] Time taken in seconds
29
- def initialize(original_messages:, compressed_messages:, time_taken:)
30
- @original_messages = original_messages
31
- @compressed_messages = compressed_messages
32
- @time_taken = time_taken
33
- end
34
-
35
- # Number of messages before compression
36
- #
37
- # @return [Integer] Original message count
38
- def original_message_count
39
- @original_messages.size
40
- end
41
-
42
- # Number of messages after compression
43
- #
44
- # @return [Integer] Compressed message count
45
- def compressed_message_count
46
- @compressed_messages.size
47
- end
48
-
49
- # Number of messages removed
50
- #
51
- # @return [Integer] Messages removed
52
- def messages_removed
53
- original_message_count - compressed_message_count
54
- end
55
-
56
- # Number of checkpoint summary messages created
57
- #
58
- # @return [Integer] Checkpoint messages
59
- def messages_summarized
60
- @compressed_messages.count do |msg|
61
- msg.role == :system && msg.content.to_s.include?("CONVERSATION CHECKPOINT")
62
- end
63
- end
64
-
65
- # Estimated tokens before compression
66
- #
67
- # @return [Integer] Original token count
68
- def original_tokens
69
- @original_tokens ||= TokenCounter.estimate_messages(@original_messages)
70
- end
71
-
72
- # Estimated tokens after compression
73
- #
74
- # @return [Integer] Compressed token count
75
- def compressed_tokens
76
- @compressed_tokens ||= TokenCounter.estimate_messages(@compressed_messages)
77
- end
78
-
79
- # Number of tokens removed
80
- #
81
- # @return [Integer] Tokens removed
82
- def tokens_removed
83
- original_tokens - compressed_tokens
84
- end
85
-
86
- # Compression ratio (compressed / original)
87
- #
88
- # @return [Float] Ratio between 0.0 and 1.0
89
- def compression_ratio
90
- return 0.0 if original_tokens.zero?
91
-
92
- compressed_tokens.to_f / original_tokens
93
- end
94
-
95
- # Compression factor (original / compressed)
96
- #
97
- # e.g., 5.0 means compressed to 1/5th of original size
98
- #
99
- # @return [Float] Compression factor
100
- def compression_factor
101
- return 0.0 if compressed_tokens.zero?
102
-
103
- original_tokens.to_f / compressed_tokens
104
- end
105
-
106
- # Compression percentage
107
- #
108
- # @return [Float] Percentage of original size (0-100)
109
- def compression_percentage
110
- (compression_ratio * 100).round(2)
111
- end
112
-
113
- # Generate a human-readable summary
114
- #
115
- # @return [String] Summary text
116
- def summary
117
- <<~SUMMARY
118
- Context Compression Results:
119
- - Messages: #{original_message_count} → #{compressed_message_count} (-#{messages_removed})
120
- - Estimated tokens: #{original_tokens} → #{compressed_tokens} (-#{tokens_removed})
121
- - Compression ratio: #{compression_factor.round(1)}:1 (#{compression_percentage}%)
122
- - Checkpoints created: #{messages_summarized}
123
- - Time taken: #{time_taken.round(3)}s
124
- SUMMARY
125
- end
126
-
127
- # Convert metrics to hash for logging
128
- #
129
- # @return [Hash] Metrics as hash
130
- def to_h
131
- {
132
- original_message_count: original_message_count,
133
- compressed_message_count: compressed_message_count,
134
- messages_removed: messages_removed,
135
- messages_summarized: messages_summarized,
136
- original_tokens: original_tokens,
137
- compressed_tokens: compressed_tokens,
138
- tokens_removed: tokens_removed,
139
- compression_ratio: compression_ratio.round(4),
140
- compression_factor: compression_factor.round(2),
141
- compression_percentage: compression_percentage,
142
- time_taken: time_taken.round(3),
143
- }
144
- end
145
- end
146
- end
147
- end
@@ -1,106 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module SwarmSDK
4
- class ContextCompactor
5
- # TokenCounter provides token estimation for messages
6
- #
7
- # This uses a simple heuristic approach:
8
- # - ~4 characters per token for English prose
9
- # - ~3.5 characters per token for code
10
- #
11
- # For production use with OpenAI models, consider using the tiktoken gem
12
- # for accurate token counting. For Claude models, use Claude's token API.
13
- #
14
- # ## Usage
15
- #
16
- # tokens = TokenCounter.estimate_message(message)
17
- # total_tokens = TokenCounter.estimate_messages(messages)
18
- #
19
- class TokenCounter
20
- # Backward compatibility aliases - use Defaults module for new code
21
- CHARS_PER_TOKEN_PROSE = Defaults::TokenEstimation::CHARS_PER_TOKEN_PROSE
22
- CHARS_PER_TOKEN_CODE = Defaults::TokenEstimation::CHARS_PER_TOKEN_CODE
23
-
24
- class << self
25
- # Estimate tokens for a single message
26
- #
27
- # @param message [RubyLLM::Message] Message to estimate
28
- # @return [Integer] Estimated token count
29
- def estimate_message(message)
30
- case message.role
31
- when :user, :assistant
32
- estimate_content(message.content)
33
- when :system
34
- estimate_content(message.content)
35
- when :tool
36
- # Tool results typically have overhead
37
- base_overhead = 50
38
- content_tokens = estimate_content(message.content)
39
- base_overhead + content_tokens
40
- else
41
- # Unknown message type
42
- begin
43
- estimate_content(message.content)
44
- rescue
45
- 0
46
- end
47
- end
48
- end
49
-
50
- # Estimate tokens for multiple messages
51
- #
52
- # @param messages [Array<RubyLLM::Message>] Messages to estimate
53
- # @return [Integer] Total estimated token count
54
- def estimate_messages(messages)
55
- messages.sum { |msg| estimate_message(msg) }
56
- end
57
-
58
- # Estimate tokens for content string
59
- #
60
- # Uses heuristic to detect code vs prose and adjust accordingly.
61
- #
62
- # @param content [String, RubyLLM::Content, nil] Content to estimate
63
- # @return [Integer] Estimated token count
64
- def estimate_content(content)
65
- return 0 if content.nil?
66
-
67
- # Handle RubyLLM::Content objects
68
- text = if content.respond_to?(:to_s)
69
- content.to_s
70
- else
71
- content
72
- end
73
-
74
- return 0 if text.empty?
75
-
76
- # Detect if content is mostly code
77
- code_ratio = detect_code_ratio(text)
78
-
79
- # Choose characters per token based on content type
80
- chars_per_token = if code_ratio > 0.1
81
- CHARS_PER_TOKEN_CODE # Code
82
- else
83
- CHARS_PER_TOKEN_PROSE # Prose
84
- end
85
-
86
- (text.length / chars_per_token).ceil
87
- end
88
-
89
- private
90
-
91
- # Detect ratio of code characters to total characters
92
- #
93
- # @param text [String] Text to analyze
94
- # @return [Float] Ratio of code indicators (0.0 to 1.0)
95
- def detect_code_ratio(text)
96
- # Count code indicator characters
97
- code_chars = text.scan(/[{}()\[\];]/).length
98
-
99
- return 0.0 if text.empty?
100
-
101
- code_chars.to_f / text.length
102
- end
103
- end
104
- end
105
- end
106
- end