scout-ai 1.2.5 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +111 -31
  3. data/README.md +171 -320
  4. data/Rakefile +17 -1
  5. data/VERSION +1 -1
  6. data/doc/Improvements.md +325 -0
  7. data/doc/StartHere.md +110 -0
  8. data/doc/developer/Architecture.md +126 -0
  9. data/doc/developer/Backends.md +199 -0
  10. data/doc/developer/ChatLifecycle.md +183 -0
  11. data/doc/developer/DelegationInternals.md +295 -0
  12. data/doc/developer/DesignPrinciples.md +245 -0
  13. data/doc/developer/PromptProcessing.md +292 -0
  14. data/doc/developer/Provenance.md +317 -0
  15. data/doc/user/BuildingAgents.md +345 -0
  16. data/doc/user/Cookbook.md +333 -0
  17. data/doc/user/CoreConcepts.md +181 -0
  18. data/doc/user/Delegation.md +191 -0
  19. data/doc/user/GettingStarted.md +159 -0
  20. data/doc/user/ManagingContext.md +163 -0
  21. data/doc/user/MultiAgentWorkflows.md +256 -0
  22. data/doc/user/Python.md +159 -0
  23. data/doc/user/RunningInference.md +200 -0
  24. data/doc/user/ToolCalling.md +193 -0
  25. data/doc/user/WritingChats.md +197 -0
  26. data/lib/scout/llm/agent/chat.rb +62 -12
  27. data/lib/scout/llm/agent/delegate.rb +274 -65
  28. data/lib/scout/llm/agent/iterate.rb +2 -2
  29. data/lib/scout/llm/agent/save.rb +273 -0
  30. data/lib/scout/llm/agent/workflow.rb +164 -0
  31. data/lib/scout/llm/agent.rb +86 -61
  32. data/lib/scout/llm/ask.rb +36 -6
  33. data/lib/scout/llm/backends/anthropic.rb +9 -1
  34. data/lib/scout/llm/backends/bedrock.rb +15 -3
  35. data/lib/scout/llm/backends/default.rb +129 -89
  36. data/lib/scout/llm/backends/glm.rb +58 -0
  37. data/lib/scout/llm/backends/huggingface.rb +13 -1
  38. data/lib/scout/llm/backends/ollama.rb +13 -0
  39. data/lib/scout/llm/backends/openwebui.rb +8 -3
  40. data/lib/scout/llm/chat/agent_meta.rb +264 -0
  41. data/lib/scout/llm/chat/annotation.rb +37 -14
  42. data/lib/scout/llm/chat/parse.rb +28 -6
  43. data/lib/scout/llm/chat/persist.rb +25 -0
  44. data/lib/scout/llm/chat/process/clear.rb +19 -1
  45. data/lib/scout/llm/chat/process/files.rb +16 -1
  46. data/lib/scout/llm/chat/process/meta.rb +422 -32
  47. data/lib/scout/llm/chat/process/options.rb +21 -1
  48. data/lib/scout/llm/chat/process/tools.rb +49 -12
  49. data/lib/scout/llm/chat/process.rb +4 -0
  50. data/lib/scout/llm/chat/prompt/shorten_tools.rb +125 -0
  51. data/lib/scout/llm/chat/prompt/shorten_tools_epoch.rb +365 -0
  52. data/lib/scout/llm/chat/prompt.rb +48 -0
  53. data/lib/scout/llm/chat/provenance.rb +775 -0
  54. data/lib/scout/llm/chat/tool_calls.rb +76 -0
  55. data/lib/scout/llm/chat.rb +18 -2
  56. data/lib/scout/llm/embed.rb +8 -3
  57. data/lib/scout/llm/image.rb +86 -0
  58. data/lib/scout/llm/rag.rb +3 -3
  59. data/lib/scout/llm/tools/call.rb +159 -10
  60. data/lib/scout/llm/tools/knowledge_base.rb +1 -1
  61. data/lib/scout/llm/tools/workflow.rb +13 -5
  62. data/lib/scout-ai.rb +1 -0
  63. data/research/ChatAnalyst-required-changes.md +167 -0
  64. data/research/agent-delegation-analysis.md +810 -0
  65. data/research/agent-meta-provenance-integration-plan.md +622 -0
  66. data/research/agent-workflow-analysis.md +1120 -0
  67. data/research/backends-analysis.md +836 -0
  68. data/research/chat-core-analysis.md +946 -0
  69. data/research/chatanalyst-provenance/00-baseline.md +30 -0
  70. data/research/chatanalyst-provenance/01-repo-map.md +60 -0
  71. data/research/chatanalyst-provenance/02-event-reconstruction.md +55 -0
  72. data/research/chatanalyst-provenance/03-duplication-evidence.md +45 -0
  73. data/research/chatanalyst-provenance/04-tooling-root-cause.md +57 -0
  74. data/research/chatanalyst-provenance/05-fix-plan.md +46 -0
  75. data/research/chatanalyst-provenance/07-critic-review.md +25 -0
  76. data/research/chatanalyst-provenance/final-report.md +45 -0
  77. data/research/chatanalyst-provenance/resumption.md +37 -0
  78. data/research/coding-philosophy-analysis.md +928 -0
  79. data/research/commands-analysis.md +947 -0
  80. data/research/multi-agent-patterns-analysis.md +853 -0
  81. data/research/prompt-strategies-analysis.md +630 -0
  82. data/research/prov-verbosity-fix-notes.md +77 -0
  83. data/research/provenance-analysis.md +469 -0
  84. data/research/provenance-navigation-design.md +640 -0
  85. data/research/synthesis-report.md +487 -0
  86. data/research/tools-system-analysis.md +779 -0
  87. data/scout-ai.gemspec +97 -13
  88. data/scout_commands/agent/ask +13 -3
  89. data/scout_commands/agent/kb +2 -0
  90. data/scout_commands/llm/ask +11 -4
  91. data/scout_commands/llm/md +76 -0
  92. data/scout_commands/llm/prov +602 -0
  93. data/scout_commands/llm/word +71 -0
  94. data/share/word/reference.docx +0 -0
  95. data/test/etc/AI/mock.yaml +11 -0
  96. data/test/fixtures/backends/anthropic.json +19 -0
  97. data/test/fixtures/backends/anthropic_tool_use.json +24 -0
  98. data/test/fixtures/backends/bedrock.json +8 -0
  99. data/test/fixtures/backends/bedrock_embedding.json +3 -0
  100. data/test/fixtures/backends/bedrock_tool_use.json +17 -0
  101. data/test/fixtures/backends/ollama.json +16 -0
  102. data/test/fixtures/backends/ollama_tool_call.json +27 -0
  103. data/test/fixtures/backends/openai_chat.json +21 -0
  104. data/test/fixtures/backends/openai_chat_tool_call.json +31 -0
  105. data/test/fixtures/backends/responses.json +33 -0
  106. data/test/fixtures/backends/responses_tool_call.json +28 -0
  107. data/test/integration/README.md +32 -0
  108. data/test/integration/scout/llm/backends/test_endpoints.rb +34 -0
  109. data/test/integration/scout/llm/backends/test_openwebui.rb +61 -0
  110. data/test/integration/scout/llm/backends/test_relay.rb +52 -0
  111. data/test/integration/scout/llm/test_infrastructure.rb +74 -0
  112. data/test/{scout → integration/scout}/llm/test_mcp.rb +1 -1
  113. data/test/integration/scout/llm/tools/test_mcp.rb +42 -0
  114. data/test/integration/scout/model/test_base.rb +91 -0
  115. data/test/scout/llm/agent/test_chat.rb +8 -2
  116. data/test/scout/llm/agent/test_save.rb +413 -0
  117. data/test/scout/llm/agent/test_workflow.rb +110 -0
  118. data/test/scout/llm/backends/test_anthropic.rb +93 -10
  119. data/test/scout/llm/backends/test_bedrock.rb +118 -2
  120. data/test/scout/llm/backends/test_ollama.rb +70 -20
  121. data/test/scout/llm/backends/test_openwebui.rb +42 -40
  122. data/test/scout/llm/backends/test_relay.rb +4 -2
  123. data/test/scout/llm/chat/agent_meta_fixtures.rb +131 -0
  124. data/test/scout/llm/chat/process/test_meta.rb +518 -0
  125. data/test/scout/llm/chat/process/test_normalize_usage.rb +183 -0
  126. data/test/scout/llm/chat/test_agent_meta.rb +357 -0
  127. data/test/scout/llm/chat/test_agent_meta_provenance.rb +467 -0
  128. data/test/scout/llm/chat/test_agent_meta_tokens.rb +594 -0
  129. data/test/scout/llm/chat/test_parse.rb +70 -15
  130. data/test/scout/llm/chat/test_prov_cli.rb +274 -0
  131. data/test/scout/llm/chat/test_provenance.rb +240 -0
  132. data/test/scout/llm/chat/test_tool_calls.rb +38 -0
  133. data/test/scout/llm/test_agent.rb +13 -36
  134. data/test/scout/llm/test_ask.rb +75 -52
  135. data/test/scout/llm/test_chat.rb +107 -13
  136. data/test/scout/llm/test_embed.rb +48 -0
  137. data/test/scout/llm/test_rag.rb +23 -16
  138. data/test/scout/llm/test_tools.rb +12 -1
  139. data/test/scout/llm/tools/test_knowledge_base.rb +0 -1
  140. data/test/scout/llm/tools/test_mcp.rb +5 -3
  141. data/test/scout/llm/tools/test_workflow.rb +23 -2
  142. data/test/scout/model/python/huggingface/causal/test_next_token.rb +11 -5
  143. data/test/scout/model/python/huggingface/test_causal.rb +9 -3
  144. data/test/scout/model/python/huggingface/test_classification.rb +11 -2
  145. data/test/scout/model/python/test_torch.rb +2 -0
  146. data/test/scout/model/python/torch/test_helpers.rb +4 -0
  147. data/test/scout/model/test_base.rb +4 -2
  148. data/test/support/availability.rb +231 -0
  149. data/test/support/fake_clients.rb +138 -0
  150. data/test/support/fixtures.rb +21 -0
  151. data/test/support/infrastructure_probes.rb +136 -0
  152. data/test/support/mock_backend.rb +215 -0
  153. data/test/test_helper.rb +32 -2
  154. metadata +96 -12
  155. data/doc/Agent.md +0 -354
  156. data/doc/Chat.md +0 -481
  157. data/doc/LLM.md +0 -356
  158. data/doc/PythonAgentTasks.md +0 -333
  159. data/doc/RAG.md +0 -129
  160. data/doc/USER_GUIDE.md +0 -572
  161. data/scout_commands/documenter +0 -148
  162. data/test/scout/llm/backends/test_openai.rb +0 -192
  163. data/test/scout/llm/backends/test_responses.rb +0 -238
  164. data/test/scout/llm/test_parse.rb +0 -98
@@ -0,0 +1,264 @@
1
+ module Chat
2
+ # Render the human readable message for a malformed or unusable agent_meta
3
+ # receipt. This is a plain module function, not a dedicated error class: the
4
+ # raised exception is ScoutException and the structured facts (chat path,
5
+ # output address, call id, tool name, raw entry, reference) always travel in
6
+ # the on_error callback `reference` Hash built by Chat.agent_meta_error_reference.
7
+ def self.agent_meta_error_message(reason, chat_path: nil, output_address: nil, call_id: nil, tool_name: nil, reference: nil)
8
+ message = +"agent_meta receipt #{reason}"
9
+ message << " in chat #{chat_path}" if chat_path
10
+ message << " at output #{Array(output_address).inspect}" unless output_address.nil?
11
+ details = []
12
+ details << "call #{call_id}" if call_id
13
+ details << "tool #{tool_name}" if tool_name
14
+ message << " (#{details * ', '})" unless details.empty?
15
+ message << " reference #{reference}" if reference
16
+ message
17
+ end
18
+
19
+ # Delegated-agent inference evidence embedded in function_call_output
20
+ # envelopes.
21
+ #
22
+ # When a tool returns an LLM::Agent, LLM.process_calls embeds the child
23
+ # agent's inference evidence in the parent function_call_output JSON under
24
+ # the `meta` key. These records are portable evidence of child inference
25
+ # events and child producer jobs; they are NOT ordinary parent-chat meta
26
+ # messages and must never be appended to the parent Chat (that would
27
+ # corrupt checkpoint bookkeeping). Everything in this module is read-only
28
+ # provenance analysis: plain Arrays and Hashes in and out, no wrapper
29
+ # classes, no mutation of the inspected chat.
30
+ #
31
+ # Two receipt formats are accepted (the envelope is generic: any tool
32
+ # output may carry one, so nothing here special-cases a tool name such as
33
+ # `ask`):
34
+ #
35
+ # meta (current): {"meta": [{"pt": 10, "inference_id": "w1"}, ...]}
36
+ # Entries are ALREADY-DESERIALIZED field Hashes (no role/content
37
+ # wrapper, no re-parsing). Evidence addresses use the [:meta, index]
38
+ # suffix, mirroring the persisted key. raw_message is nil.
39
+ #
40
+ # agent_meta (legacy): {"agent_meta": [{"role": "meta", "content": "pt=10 ..."}, ...]}
41
+ # Serialized meta messages; each content String is parsed with
42
+ # Chat.parse_meta. Evidence addresses keep the [:agent_meta, index]
43
+ # suffix and raw_message is {role: 'meta', content: <String>} so old
44
+ # data stays addressable exactly as it always was.
45
+ #
46
+ # When both keys are present in one envelope the `meta` key wins (current
47
+ # writer emits exactly one of the two).
48
+
49
+ # One Hash per valid agent_meta entry found across the paired tool outputs
50
+ # of the chat. Pairing is delegated to Chat.tool_calls; raw text is never
51
+ # scanned with regular expressions.
52
+ #
53
+ # Record shape:
54
+ #
55
+ # {
56
+ # origin: :agent_meta,
57
+ # meta: <IndiferentHash of meta fields>,
58
+ # source: <String path or nil>,
59
+ # output_address: <as returned by Chat.tool_calls>,
60
+ # evidence_address: <output_address + [:agent_meta, index]> (legacy
61
+ # entries; current-format entries use [:meta, index]),
62
+ # call_id: ...,
63
+ # tool_name: ...,
64
+ # agent_meta_index: ...,
65
+ # raw_message: {role: "meta", content: "..."} (legacy; nil for
66
+ # current-format entries, which are born deserialized)
67
+ # }
68
+ #
69
+ # Malformed data is skipped. When the caller supplies an Array through the
70
+ # `warnings` keyword, each malformed item appends one warning Hash:
71
+ #
72
+ # {
73
+ # origin: :agent_meta,
74
+ # reason: :not_an_array | :not_a_hash | :invalid_role |
75
+ # :invalid_content | :unparseable_meta | :empty_meta,
76
+ # source:, output_address:, evidence_address:,
77
+ # call_id:, tool_name:,
78
+ # agent_meta_index: (nil when the whole receipt value is malformed),
79
+ # evidence_address: (nil when the index is unknown),
80
+ # raw_entry: <the malformed item as found>
81
+ # }
82
+ #
83
+ # :empty_meta applies to current-format entries (deserialized field Hash
84
+ # with no fields). Malformed receipt data is never silently reinterpreted
85
+ # as provenance.
86
+ def self.agent_meta_evidence(chat, source: nil, warnings: nil)
87
+ tool_calls(chat, source: source).flat_map do |call|
88
+ output_info = call[:output_info]
89
+ next [] unless Hash === output_info
90
+
91
+ # Key presence, not truthiness: an explicit `meta: false` or
92
+ # `agent_meta: nil` is a present-but-malformed receipt and must warn,
93
+ # while an absent key simply has no receipt at all.
94
+ has_meta = output_info.key?('meta') || output_info.key?(:meta)
95
+ has_agent_meta = output_info.key?('agent_meta') || output_info.key?(:agent_meta)
96
+ next [] unless has_meta || has_agent_meta
97
+
98
+ # Current writer emits exactly one of the two; when both are present
99
+ # the current `meta` key wins and the legacy one is ignored.
100
+ format = has_meta ? :meta : :agent_meta
101
+ agent_meta = output_info['meta']
102
+ agent_meta = output_info[:meta] if agent_meta.nil? && output_info.key?(:meta)
103
+ agent_meta = output_info['agent_meta'] if format == :agent_meta && agent_meta.nil?
104
+ agent_meta = output_info[:agent_meta] if format == :agent_meta && agent_meta.nil?
105
+
106
+ add_warning = lambda do |reason, index, raw_entry|
107
+ return unless Array === warnings
108
+ output_address = call[:output_address]
109
+ evidence_address = if index.nil?
110
+ nil
111
+ elsif Array === output_address
112
+ output_address + [format, index]
113
+ else
114
+ [output_address, format, index]
115
+ end
116
+ warnings << {
117
+ origin: :agent_meta,
118
+ reason: reason,
119
+ source: source ? source.to_s : nil,
120
+ output_address: output_address,
121
+ evidence_address: evidence_address,
122
+ call_id: call[:call_id],
123
+ tool_name: call[:name],
124
+ agent_meta_index: index,
125
+ raw_entry: raw_entry
126
+ }
127
+ nil
128
+ end
129
+
130
+ unless Array === agent_meta
131
+ add_warning.call(:not_an_array, nil, agent_meta)
132
+ next []
133
+ end
134
+
135
+ agent_meta.each_with_index.collect do |entry, index|
136
+ unless Hash === entry
137
+ add_warning.call(:not_a_hash, index, entry)
138
+ next nil
139
+ end
140
+
141
+ meta, raw_message =
142
+ if format == :meta
143
+ # Current format: entries are already-deserialized field Hashes.
144
+ # No role/content wrapper and no re-parse; an entry with no
145
+ # fields carries no evidence and is warned about.
146
+ fields = IndiferentHash.setup(entry.dup)
147
+ [fields, nil]
148
+ else
149
+ # Legacy format: serialized {role: 'meta', content: 'k=v ...'}
150
+ # meta messages.
151
+ role = entry['role'] || entry[:role]
152
+ content = entry['content'] || entry[:content]
153
+
154
+ if role.to_s != 'meta'
155
+ add_warning.call(:invalid_role, index, entry)
156
+ next nil
157
+ end
158
+
159
+ unless String === content
160
+ add_warning.call(:invalid_content, index, entry)
161
+ next nil
162
+ end
163
+
164
+ [parse_meta(content), { role: role, content: content }]
165
+ end
166
+
167
+ if meta.empty?
168
+ reason = format == :meta ? :empty_meta : :unparseable_meta
169
+ add_warning.call(reason, index, entry)
170
+ next nil
171
+ end
172
+
173
+ output_address = call[:output_address]
174
+ # Without a source, Chat.tool_calls reports a bare message index; the
175
+ # receipt address keeps the key-mirroring suffix so it stays truthful
176
+ # to the persisted envelope ([:meta, index] for current data,
177
+ # [:agent_meta, index] for legacy data).
178
+ evidence_address = if Array === output_address
179
+ output_address + [format, index]
180
+ else
181
+ [output_address, format, index]
182
+ end
183
+ {
184
+ origin: :agent_meta,
185
+ meta: meta,
186
+ source: source ? source.to_s : nil,
187
+ output_address: output_address,
188
+ evidence_address: evidence_address,
189
+ call_id: call[:call_id],
190
+ tool_name: call[:name],
191
+ agent_meta_index: index,
192
+ raw_message: raw_message
193
+ }
194
+ end.compact
195
+ end
196
+ end
197
+
198
+ # All meta evidence for a chat, with a shared `origin` field:
199
+ #
200
+ # * :chat_meta - normal persisted `meta:` messages of this chat
201
+ # (meta, source, meta_address in [source, index] form,
202
+ # message: the raw message);
203
+ # * :agent_meta - receipt entries from function_call_output envelopes,
204
+ # appended after the local records (see
205
+ # Chat.agent_meta_evidence).
206
+ #
207
+ # This is an analysis view only. Chat#meta, chat.role_messages(:meta), and
208
+ # Chat.token_totals([chat]) keep describing the local Chat and do not absorb
209
+ # delegated receipts.
210
+ #
211
+ # Validation asymmetry, intentional: ordinary persisted `meta:` messages are
212
+ # historical data with no enforced schema, so they stay permissive here (a
213
+ # record is included even when Chat.parse_meta yields an empty Hash) exactly
214
+ # as Chat.trace_chats and Chat.token_totals have always treated them. A
215
+ # receipt instead has a narrow, generated schema, so malformed receipt
216
+ # entries are skipped with a warning instead of being silently trusted.
217
+ def self.meta_evidence(chat, source: nil, warnings: nil)
218
+ local = chat.each_with_index.select do |message, _index|
219
+ message[:role].to_s == 'meta'
220
+ end.collect do |message, index|
221
+ {
222
+ origin: :chat_meta,
223
+ meta: parse_meta(message[:content].to_s),
224
+ source: source ? source.to_s : nil,
225
+ meta_address: source ? [source.to_s, index] : index,
226
+ message: message
227
+ }
228
+ end
229
+
230
+ local + agent_meta_evidence(chat, source: source, warnings: warnings)
231
+ end
232
+
233
+ # agent_meta entries that carry a producer job reference (`job=`). Each
234
+ # returned record is the full evidence record with the reference added at
235
+ # the top level as `job:`, so both the reference value and its evidence
236
+ # location stay together. Job projection metas are producer references,
237
+ # not token events.
238
+ def self.agent_meta_job_references(chat, source: nil, warnings: nil)
239
+ agent_meta_evidence(chat, source: source, warnings: warnings)
240
+ .select { |record| record[:meta][:job] }
241
+ .collect { |record| record.merge(job: record[:meta][:job]) }
242
+ end
243
+
244
+ # Uniform `reference` Hash handed to on_error for agent_meta provenance
245
+ # problems. Keys are always present; unavailable facts are nil. It retains
246
+ # the original reference/raw entry, the reason, the output and evidence
247
+ # addresses, the receipt entry index, the call id and tool name when known,
248
+ # and the chat path.
249
+ def self.agent_meta_error_reference(reason:, source:, output_address: nil, evidence_address: nil,
250
+ call_id: nil, tool_name: nil, agent_meta_index: nil,
251
+ raw_entry: nil, reference: nil)
252
+ {
253
+ reason: reason,
254
+ source: source ? source.to_s : nil,
255
+ output_address: output_address,
256
+ evidence_address: evidence_address,
257
+ call_id: call_id,
258
+ tool_name: tool_name,
259
+ agent_meta_index: agent_meta_index,
260
+ raw_entry: raw_entry,
261
+ reference: reference
262
+ }
263
+ end
264
+ end
@@ -1,4 +1,5 @@
1
1
  require 'scout/annotation'
2
+
2
3
  module Chat
3
4
  extend Annotation
4
5
 
@@ -62,6 +63,13 @@ module Chat
62
63
  message(:task, content)
63
64
  end
64
65
 
66
+ def exec_task(workflow, task_name, inputs = {})
67
+ input_str = IndiferentHash.print_options inputs
68
+ content = [workflow, task_name, input_str]*" "
69
+ message(:exec_task, content)
70
+ end
71
+
72
+
65
73
  def inline_task(workflow, task_name, inputs = {})
66
74
  input_str = IndiferentHash.print_options inputs
67
75
  content = [workflow, task_name, input_str]*" "
@@ -84,7 +92,7 @@ module Chat
84
92
  end
85
93
 
86
94
  def tag(content, name=nil, tag=:file, role=:user)
87
- self.message role, LLM.tag(tag, content, name)
95
+ self.message role, Chat.tag(tag, content, name)
88
96
  end
89
97
 
90
98
 
@@ -93,10 +101,11 @@ module Chat
93
101
  end
94
102
 
95
103
  def chat(options = {})
96
- response = ask(options.merge(return_messages: true))
104
+ options = IndiferentHash.add_defaults options, return_messages: true
105
+ response = ask(options)
97
106
  if Array === response
98
107
  self.concat(response)
99
- final
108
+ answer
100
109
  else
101
110
  self.push({role: :assistant, content: response})
102
111
  response
@@ -121,17 +130,21 @@ module Chat
121
130
  coda = [coda] if Hash === coda
122
131
  new = Chat.follow(self.dup, coda)
123
132
  self.replace new
133
+ self
124
134
  end
125
135
 
126
136
  def follow(coda)
137
+ coda = [coda] if Hash === coda
127
138
  new = Chat.follow(self.dup, coda)
128
139
  self.replace new
140
+ self
129
141
  end
130
142
 
131
143
  def prepend(intro)
132
144
  into = [intro] if Hash === intro
133
145
  new = Chat.follow(intro, self.dup)
134
146
  self.replace new
147
+ self
135
148
  end
136
149
 
137
150
  def remove_role(role=:clear)
@@ -140,10 +153,10 @@ module Chat
140
153
  messages
141
154
  end
142
155
 
143
- def json(...)
156
+ def json(*args, only_ask: false, **kwargs)
144
157
  self.format :json
145
- output = ask(...)
146
- obj = JSON.parse output
158
+ output = only_ask ? ask(*args, **kwargs) : chat(*args, **kwargs)
159
+ obj = Chat.parse_json output
147
160
  if (Hash === obj) and obj.keys == ['content']
148
161
  obj['content']
149
162
  else
@@ -151,10 +164,10 @@ module Chat
151
164
  end
152
165
  end
153
166
 
154
- def json_format(format, ...)
167
+ def json_format(fornat, *args, only_ask: false, **kwargs)
155
168
  self.format format
156
- output = ask(...)
157
- obj = JSON.parse output
169
+ output = only_ask ? ask(*args, **kwargs) : chat(*args, **kwargs)
170
+ obj = Chat.parse_json output
158
171
  if (Hash === obj) and obj.keys == ['content']
159
172
  obj['content']
160
173
  else
@@ -236,12 +249,22 @@ module Chat
236
249
  # Image
237
250
  def create_image(file, ...)
238
251
  base64_image = LLM.image(LLM.chat(self), ...)
239
- Open.write(file, Base64.decode(file_content), mode: 'wb')
252
+ Open.write(file, Base64.decode64(base64_image), mode: 'wb')
253
+ end
254
+
255
+ def role_messages(role, &block)
256
+ if Symbol === role
257
+ self.select{|msg| msg[:role].to_sym == role }
258
+ else
259
+ self.select{|msg| msg[:role].to_s == role }
260
+ end
240
261
  end
241
262
 
242
- def meta()
243
- meta_msg = self.select{|info| info[:role].to_s == "meta" }.last
244
- return {} if meta_msg.nil?
245
- Chat.parse_meta meta_msg[:content]
263
+ # Return the last job reference from the meta messages in this chat.
264
+ def last_job
265
+ meta_msg = self.reverse.find{|info| info[:role].to_s == "meta" }
266
+ return nil if meta_msg.nil?
267
+ meta = Chat.parse_meta(meta_msg[:content])
268
+ meta[:job]
246
269
  end
247
270
  end
@@ -1,4 +1,12 @@
1
1
  module Chat
2
+ def self.parse_json(text)
3
+ return nil if text.nil? || text.empty?
4
+ re = /.*\`\`\`json\n(.*)\`\`\`\n?.*/sm
5
+ re = Regexp.new(re.source.encode(text.encoding), re.options)
6
+ text = text.gsub(re, '\1') if text.include?('```json')
7
+ JSON.parse text
8
+ end
9
+
2
10
  def self.parse(text, role = nil)
3
11
  default_role = "user"
4
12
 
@@ -48,16 +56,16 @@ module Chat
48
56
  line = line.sub("]]", "")
49
57
  current_content << "\n" << line unless line.strip.empty?
50
58
  next
51
- elsif stripped.match(/^[^\s]*:-- .* {{{/)
59
+ elsif stripped.match(/^[^\s]* :-- .* {{{/)
52
60
  in_protected_block = true
53
61
  protected_block_type = :square
54
- line = line.sub(/^[^\s]*:-- (.*) {{{.*/, '<cmd_output cmd="\1">')
62
+ line = line.sub(/^[^\s]* :-- (.*) {{{.*/, '<cmd_output cmd="\1">')
55
63
  current_content << "\n" << line unless line.strip.empty?
56
64
  next
57
- elsif stripped.match(/^.*:--.* }}}/) && in_protected_block && protected_block_type == :square
65
+ elsif stripped.match(/^.* :--.* }}}/) && in_protected_block && protected_block_type == :square
58
66
  in_protected_block = false
59
67
  protected_block_type = nil
60
- line = line.sub(/^.*:-- .* }}}.*/, "</cmd_output>")
68
+ line = line.sub(/^.* :-- .* }}}.*/, "</cmd_output>")
61
69
  current_content << "\n" << line unless line.strip.empty?
62
70
  next
63
71
  elsif in_protected_block
@@ -82,6 +90,7 @@ module Chat
82
90
  if stripped =~ /^<(\w+)(\s+[^>]*)?>/ && (tag = $1) && text =~ %r{</#{$1}>}
83
91
  protected_stack.push(tag)
84
92
  in_protected_block = true
93
+ current_content << "\n" << line
85
94
  protected_block_type = :xml
86
95
  next
87
96
  end
@@ -93,7 +102,11 @@ module Chat
93
102
 
94
103
  current_content = current_content.strip if current_content
95
104
  # Save current message if any
96
- messages << { role: current_role, content: current_content } if current_content && ! current_content.empty?
105
+ if current_content && ! current_content.empty?
106
+ messages << { role: current_role, content: current_content }
107
+ elsif messages.empty?
108
+ #messages << { role: current_role, content: '' }
109
+ end
97
110
 
98
111
  if inline_content.empty?
99
112
  # Block message
@@ -106,6 +119,12 @@ module Chat
106
119
  current_role = 'user' if role == 'agent'
107
120
  current_content = ""
108
121
  end
122
+ elsif line =~ /^\\([a-z0-9_]+):(.*)$/
123
+ if current_content.nil?
124
+ current_content = line[1..-1]
125
+ else
126
+ current_content += "\n" + line[1..-1]
127
+ end
109
128
  else
110
129
  if current_content.nil?
111
130
  current_content = line
@@ -134,7 +153,10 @@ module Chat
134
153
  if %w(option previous_response_id function_call function_call_output meta).include? message[:role].to_s
135
154
  message[:role].to_s + ": " + message[:content].to_s
136
155
  else
137
- message[:role].to_s + ":\n\n" + message[:content].to_s
156
+ re = Regexp.new(/^([a-z]+:)(\s)/ms)
157
+ re = Regexp.new(re.source.encode(message[:content].to_s.encoding), re.options)
158
+ message[:role].to_s + ":\n\n" +
159
+ message[:content].to_s.gsub(re, '\\\\\1\2')
138
160
  end
139
161
  end
140
162
  end * "\n\n"
@@ -0,0 +1,25 @@
1
+ require 'scout/persist'
2
+ require 'scout/workflow'
3
+
4
+ Persist.save_drivers[:chat] = proc do |file,content|
5
+ case content
6
+ when LLM::Agent
7
+ new_chat = content.current_chat - content.start_chat
8
+ Open.sensible_write(file, LLM.print(new_chat))
9
+ when Array
10
+ Open.sensible_write(file, LLM.print(content))
11
+ else
12
+ stream = if content.respond_to?(:stream)
13
+ content.stream
14
+ elsif content.respond_to?(:dumper_stream)
15
+ content.dumper_stream
16
+ else
17
+ content
18
+ end
19
+ Open.sensible_write(file, stream)
20
+ end
21
+ end
22
+
23
+ Persist.load_drivers[:chat] = proc do |file| String === file ? LLM.chat(file) : file end
24
+
25
+ Workflow::TYPE_EXTENSIONS[:chat] = :chat
@@ -2,21 +2,39 @@ module Chat
2
2
  def self.clear(messages, role = 'clear')
3
3
  new = []
4
4
 
5
+ clear_tools = false
6
+ clean_roles = []
5
7
  messages.reverse.each do |message|
6
8
  if message[:role].to_s == role.to_s
7
9
  break
10
+ elsif message[:role].to_s == 'clear_tools'
11
+ clear_tools = message['content'].to_s != 'false'
12
+ elsif message[:role].to_s == 'function_call' ||
13
+ message[:role].to_s == 'function_call_output'
14
+ new << message unless clear_tools
15
+ elsif message[:role].to_s == 'clean_role' ||
16
+ message[:role].to_s == 'clear_role'
17
+ clean_roles << message[:content].strip
8
18
  else
9
19
  new << message
10
20
  end
11
21
  end
12
22
 
13
- Chat.setup new.reverse
23
+ new = Chat.setup new.reverse
24
+
25
+ clean_roles.each do |role|
26
+ new = self.clean(new, role)
27
+ end
28
+
29
+ new
14
30
  end
15
31
 
16
32
  def self.clean(messages, role = ['skip', 'previous_response_id'])
33
+ role = role.collect{|r| r.to_s } if Array == role
17
34
  messages.reject do |message|
18
35
  ((String === message[:content]) && message[:content].empty?) ||
19
36
  if Array === role
37
+ role.include?(message[:role].to_s)
20
38
  else
21
39
  message[:role].to_s == role.to_s
22
40
  end
@@ -87,10 +87,25 @@ module Chat
87
87
 
88
88
  message[:content] = found_file
89
89
  message
90
+ elsif message[:role] == 'step'
91
+ step = message[:content].to_s.strip
92
+ meta = Chat.meta(messages.dup)
93
+ job_path = meta['job']
94
+ job = Step.load job_path
95
+ job = job.step(step) unless job.task_name == step
96
+ {role: 'assistant', content: job.load.answer }
97
+ elsif message[:role] == 'allow_path'
98
+ path = message[:content].to_s.strip
99
+ Chat.allow_path(path)
100
+ nil
101
+ elsif message[:role] == 'allow_read_path'
102
+ path = message[:content].to_s.strip
103
+ Chat.allow_read_path(path)
104
+ nil
90
105
  else
91
106
  message
92
107
  end
93
- end.flatten
108
+ end.flatten.compact
94
109
  end
95
110
 
96
111
  end