ruby_llm 1.16.0 → 2.0.0.rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (474) hide show
  1. checksums.yaml +4 -4
  2. data/.rdoc_options +25 -0
  3. data/README.md +85 -32
  4. data/exe/ruby_llm +8 -0
  5. data/lib/generators/ruby_llm/agent/templates/agent.rb.tt +0 -1
  6. data/lib/generators/ruby_llm/chat_ui/chat_ui_generator.rb +3 -43
  7. data/lib/generators/ruby_llm/chat_ui/templates/controllers/chats_controller.rb.tt +11 -3
  8. data/lib/generators/ruby_llm/chat_ui/templates/controllers/messages_controller.rb.tt +8 -0
  9. data/lib/generators/ruby_llm/chat_ui/templates/controllers/models_controller.rb.tt +4 -4
  10. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/chats/_chat.html.erb.tt +1 -1
  11. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/chats/_form.html.erb.tt +1 -1
  12. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/chats/index.html.erb.tt +1 -1
  13. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/chats/show.html.erb.tt +2 -2
  14. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/messages/_assistant.html.erb.tt +1 -1
  15. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/messages/_system.html.erb.tt +1 -1
  16. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/messages/_tool.html.erb.tt +1 -1
  17. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/messages/_tool_calls.html.erb.tt +6 -4
  18. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/messages/_user.html.erb.tt +1 -1
  19. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/messages/tool_calls/_default.html.erb.tt +2 -2
  20. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/models/_model.html.erb.tt +5 -6
  21. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/models/index.html.erb.tt +2 -2
  22. data/lib/generators/ruby_llm/chat_ui/templates/tailwind/views/models/show.html.erb.tt +5 -5
  23. data/lib/generators/ruby_llm/chat_ui/templates/views/chats/_chat.html.erb.tt +1 -1
  24. data/lib/generators/ruby_llm/chat_ui/templates/views/chats/_form.html.erb.tt +1 -1
  25. data/lib/generators/ruby_llm/chat_ui/templates/views/chats/index.html.erb.tt +1 -1
  26. data/lib/generators/ruby_llm/chat_ui/templates/views/chats/show.html.erb.tt +2 -2
  27. data/lib/generators/ruby_llm/chat_ui/templates/views/messages/_assistant.html.erb.tt +1 -1
  28. data/lib/generators/ruby_llm/chat_ui/templates/views/messages/_system.html.erb.tt +1 -1
  29. data/lib/generators/ruby_llm/chat_ui/templates/views/messages/_tool.html.erb.tt +1 -1
  30. data/lib/generators/ruby_llm/chat_ui/templates/views/messages/_tool_calls.html.erb.tt +6 -4
  31. data/lib/generators/ruby_llm/chat_ui/templates/views/messages/_user.html.erb.tt +1 -1
  32. data/lib/generators/ruby_llm/chat_ui/templates/views/messages/create.turbo_stream.erb.tt +4 -6
  33. data/lib/generators/ruby_llm/chat_ui/templates/views/messages/tool_calls/_default.html.erb.tt +2 -2
  34. data/lib/generators/ruby_llm/chat_ui/templates/views/models/_model.html.erb.tt +5 -6
  35. data/lib/generators/ruby_llm/chat_ui/templates/views/models/index.html.erb.tt +2 -2
  36. data/lib/generators/ruby_llm/chat_ui/templates/views/models/show.html.erb.tt +3 -3
  37. data/lib/generators/ruby_llm/generator_helpers.rb +106 -62
  38. data/lib/generators/ruby_llm/install/install_generator.rb +3 -11
  39. data/lib/generators/ruby_llm/install/templates/create_chats_migration.rb.tt +2 -0
  40. data/lib/generators/ruby_llm/install/templates/create_messages_migration.rb.tt +14 -6
  41. data/lib/generators/ruby_llm/install/templates/create_ruby_llm_records_migration.rb.tt +117 -0
  42. data/lib/generators/ruby_llm/install/templates/initializer.rb.tt +0 -8
  43. data/lib/generators/ruby_llm/provider/cli.rb +175 -0
  44. data/lib/generators/ruby_llm/provider/scaffold.rb +323 -0
  45. data/lib/generators/ruby_llm/provider/templates/core/provider.rb.erb +37 -0
  46. data/lib/generators/ruby_llm/provider/templates/core/provider_spec.rb.erb +34 -0
  47. data/lib/generators/ruby_llm/provider/templates/gem/archspec.rb.erb +14 -0
  48. data/lib/generators/ruby_llm/provider/templates/gem/bin/console.erb +15 -0
  49. data/lib/generators/ruby_llm/provider/templates/gem/bin/setup.erb +5 -0
  50. data/lib/generators/ruby_llm/provider/templates/gem/chat_schema_spec.rb.erb +30 -0
  51. data/lib/generators/ruby_llm/provider/templates/gem/chat_spec.rb.erb +35 -0
  52. data/lib/generators/ruby_llm/provider/templates/gem/chat_streaming_spec.rb.erb +22 -0
  53. data/lib/generators/ruby_llm/provider/templates/gem/chat_tools_spec.rb.erb +30 -0
  54. data/lib/generators/ruby_llm/provider/templates/gem/ci.yml.erb +32 -0
  55. data/lib/generators/ruby_llm/provider/templates/gem/embedding_spec.rb.erb +49 -0
  56. data/lib/generators/ruby_llm/provider/templates/gem/env.erb +2 -0
  57. data/lib/generators/ruby_llm/provider/templates/gem/fixtures_gitkeep.erb +1 -0
  58. data/lib/generators/ruby_llm/provider/templates/gem/flayignore.erb +1 -0
  59. data/lib/generators/ruby_llm/provider/templates/gem/gemfile.erb +25 -0
  60. data/lib/generators/ruby_llm/provider/templates/gem/gemspec.erb +28 -0
  61. data/lib/generators/ruby_llm/provider/templates/gem/gitignore.erb +7 -0
  62. data/lib/generators/ruby_llm/provider/templates/gem/gitleaks.yml.erb +22 -0
  63. data/lib/generators/ruby_llm/provider/templates/gem/image_spec.rb.erb +23 -0
  64. data/lib/generators/ruby_llm/provider/templates/gem/license.erb +21 -0
  65. data/lib/generators/ruby_llm/provider/templates/gem/models.rb.erb +19 -0
  66. data/lib/generators/ruby_llm/provider/templates/gem/models_spec.rb.erb +17 -0
  67. data/lib/generators/ruby_llm/provider/templates/gem/moderation_spec.rb.erb +22 -0
  68. data/lib/generators/ruby_llm/provider/templates/gem/overcommit.yml.erb +31 -0
  69. data/lib/generators/ruby_llm/provider/templates/gem/provider.rb.erb +52 -0
  70. data/lib/generators/ruby_llm/provider/templates/gem/provider_spec.rb.erb +38 -0
  71. data/lib/generators/ruby_llm/provider/templates/gem/rakefile.erb +38 -0
  72. data/lib/generators/ruby_llm/provider/templates/gem/readme.md.erb +50 -0
  73. data/lib/generators/ruby_llm/provider/templates/gem/release.yml.erb +36 -0
  74. data/lib/generators/ruby_llm/provider/templates/gem/rerank_spec.rb.erb +23 -0
  75. data/lib/generators/ruby_llm/provider/templates/gem/rspec.erb +2 -0
  76. data/lib/generators/ruby_llm/provider/templates/gem/rubocop.yml.erb +29 -0
  77. data/lib/generators/ruby_llm/provider/templates/gem/rubyllm_configuration.rb.erb +14 -0
  78. data/lib/generators/ruby_llm/provider/templates/gem/spec_helper.rb.erb +26 -0
  79. data/lib/generators/ruby_llm/provider/templates/gem/speech_spec.rb.erb +25 -0
  80. data/lib/generators/ruby_llm/provider/templates/gem/vcr_configuration.rb.erb +16 -0
  81. data/lib/generators/ruby_llm/provider/templates/gem/video_spec.rb.erb +27 -0
  82. data/lib/generators/ruby_llm/schema/schema_generator.rb +5 -1
  83. data/lib/generators/ruby_llm/schema/templates/schema.rb.tt +1 -1
  84. data/lib/generators/ruby_llm/tool/templates/tailwind/tool_call.html.erb.tt +13 -0
  85. data/lib/generators/ruby_llm/tool/templates/tailwind/tool_result.html.erb.tt +21 -0
  86. data/lib/generators/ruby_llm/tool/templates/tool.rb.tt +3 -3
  87. data/lib/generators/ruby_llm/tool/templates/tool_call.html.erb.tt +7 -12
  88. data/lib/generators/ruby_llm/tool/templates/tool_result.html.erb.tt +5 -2
  89. data/lib/generators/ruby_llm/tool/tool_generator.rb +25 -59
  90. data/lib/generators/ruby_llm/upgrade/templates/backfill_v2_data.rb.tt +461 -0
  91. data/lib/generators/ruby_llm/upgrade/templates/cleanup_v2_upgrade.rb.tt +100 -0
  92. data/lib/generators/ruby_llm/upgrade/templates/finish_v2_upgrade.rb.tt +215 -0
  93. data/lib/generators/ruby_llm/upgrade/templates/prepare_v2_upgrade.rb.tt +660 -0
  94. data/lib/generators/ruby_llm/upgrade/templates/ruby_llm_upgrade.rb.tt +222 -0
  95. data/lib/generators/ruby_llm/upgrade/templates/upgrade_initializer.rb.tt +13 -0
  96. data/lib/generators/ruby_llm/upgrade/upgrade_generator.rb +167 -0
  97. data/lib/generators/ruby_llm/upgrade/upgrade_migration.rb +344 -0
  98. data/lib/ruby_llm/accounting/usage.rb +245 -0
  99. data/lib/ruby_llm/active_record/acts_as.rb +94 -111
  100. data/lib/ruby_llm/active_record/attachment_helpers.rb +180 -0
  101. data/lib/ruby_llm/active_record/batch.rb +97 -0
  102. data/lib/ruby_llm/active_record/chat_methods.rb +823 -305
  103. data/lib/ruby_llm/active_record/message_methods.rb +113 -136
  104. data/lib/ruby_llm/active_record/model.rb +135 -0
  105. data/lib/ruby_llm/active_record/payload_helpers.rb +1 -2
  106. data/lib/ruby_llm/active_record/tool_call.rb +33 -0
  107. data/lib/ruby_llm/active_record/usage.rb +61 -0
  108. data/lib/ruby_llm/agent.rb +1065 -151
  109. data/lib/ruby_llm/aliases.json +268 -100
  110. data/lib/ruby_llm/attachment.rb +187 -48
  111. data/lib/ruby_llm/batch.rb +432 -0
  112. data/lib/ruby_llm/cached_content.rb +112 -0
  113. data/lib/ruby_llm/chat/tool_concurrency.rb +111 -0
  114. data/lib/ruby_llm/chat.rb +1127 -198
  115. data/lib/ruby_llm/chunk.rb +10 -0
  116. data/lib/ruby_llm/citation.rb +105 -0
  117. data/lib/ruby_llm/configuration.rb +261 -24
  118. data/lib/ruby_llm/context.rb +128 -6
  119. data/lib/ruby_llm/cost.rb +217 -80
  120. data/lib/ruby_llm/downloaded_file.rb +33 -0
  121. data/lib/ruby_llm/embedding.rb +121 -17
  122. data/lib/ruby_llm/embedding_request.rb +53 -0
  123. data/lib/ruby_llm/error.rb +159 -23
  124. data/lib/ruby_llm/fallback.rb +133 -0
  125. data/lib/ruby_llm/files/mime_type.rb +97 -0
  126. data/lib/ruby_llm/image.rb +153 -32
  127. data/lib/ruby_llm/message.rb +233 -54
  128. data/lib/ruby_llm/model/modalities.rb +17 -4
  129. data/lib/ruby_llm/model/pricing.rb +24 -5
  130. data/lib/ruby_llm/model/pricing_category.rb +103 -14
  131. data/lib/ruby_llm/model/pricing_tier.rb +55 -15
  132. data/lib/ruby_llm/model.rb +244 -2
  133. data/lib/ruby_llm/models/aliases.rb +41 -0
  134. data/lib/ruby_llm/models/registry.rb +165 -0
  135. data/lib/ruby_llm/models/schema.rb +99 -0
  136. data/lib/ruby_llm/models.json +71172 -33253
  137. data/lib/ruby_llm/models.rb +477 -215
  138. data/lib/ruby_llm/moderation.rb +139 -26
  139. data/lib/ruby_llm/ocr.rb +112 -0
  140. data/lib/ruby_llm/prompt.rb +79 -0
  141. data/lib/ruby_llm/protocol/binary_streaming.rb +65 -0
  142. data/lib/ruby_llm/protocol/stream_accumulator.rb +214 -0
  143. data/lib/ruby_llm/protocol/streaming.rb +230 -0
  144. data/lib/ruby_llm/protocol.rb +662 -0
  145. data/lib/ruby_llm/protocols/anthropic/batches.rb +73 -0
  146. data/lib/ruby_llm/protocols/anthropic/chat.rb +540 -0
  147. data/lib/ruby_llm/protocols/anthropic/embeddings.rb +14 -0
  148. data/lib/ruby_llm/protocols/anthropic/files.rb +38 -0
  149. data/lib/ruby_llm/protocols/anthropic/media.rb +141 -0
  150. data/lib/ruby_llm/protocols/anthropic/models.rb +129 -0
  151. data/lib/ruby_llm/protocols/anthropic/streaming.rb +166 -0
  152. data/lib/ruby_llm/{providers → protocols}/anthropic/tools.rb +24 -41
  153. data/lib/ruby_llm/protocols/anthropic.rb +100 -0
  154. data/lib/ruby_llm/protocols/azure/files.rb +16 -0
  155. data/lib/ruby_llm/protocols/bedrock/async_videos.rb +109 -0
  156. data/lib/ruby_llm/protocols/bedrock/batches.rb +129 -0
  157. data/lib/ruby_llm/protocols/bedrock/files.rb +110 -0
  158. data/lib/ruby_llm/protocols/bedrock/guardrails.rb +122 -0
  159. data/lib/ruby_llm/protocols/bedrock/rerank.rb +95 -0
  160. data/lib/ruby_llm/protocols/chat_completions/batches.rb +32 -0
  161. data/lib/ruby_llm/protocols/chat_completions/chat.rb +493 -0
  162. data/lib/ruby_llm/protocols/chat_completions/embedding_batches.rb +35 -0
  163. data/lib/ruby_llm/protocols/chat_completions/embeddings.rb +60 -0
  164. data/lib/ruby_llm/protocols/chat_completions/images.rb +127 -0
  165. data/lib/ruby_llm/{providers/openai → protocols/chat_completions}/media.rb +30 -17
  166. data/lib/ruby_llm/protocols/chat_completions/models.rb +39 -0
  167. data/lib/ruby_llm/protocols/chat_completions/moderation.rb +52 -0
  168. data/lib/ruby_llm/protocols/chat_completions/rerank.rb +56 -0
  169. data/lib/ruby_llm/protocols/chat_completions/speech.rb +40 -0
  170. data/lib/ruby_llm/protocols/chat_completions/streaming.rb +69 -0
  171. data/lib/ruby_llm/{providers/openai → protocols/chat_completions}/tools.rb +15 -17
  172. data/lib/ruby_llm/protocols/chat_completions/transcription.rb +150 -0
  173. data/lib/ruby_llm/protocols/chat_completions.rb +21 -0
  174. data/lib/ruby_llm/protocols/cohere/batch_requests.rb +75 -0
  175. data/lib/ruby_llm/protocols/cohere/batches.rb +98 -0
  176. data/lib/ruby_llm/protocols/cohere/chat.rb +227 -0
  177. data/lib/ruby_llm/protocols/cohere/datasets.rb +102 -0
  178. data/lib/ruby_llm/protocols/cohere/embeddings.rb +68 -0
  179. data/lib/ruby_llm/protocols/cohere/media.rb +77 -0
  180. data/lib/ruby_llm/protocols/cohere/models.rb +92 -0
  181. data/lib/ruby_llm/protocols/cohere/ocr.rb +63 -0
  182. data/lib/ruby_llm/protocols/cohere/rerank.rb +52 -0
  183. data/lib/ruby_llm/protocols/cohere/streaming.rb +105 -0
  184. data/lib/ruby_llm/protocols/cohere/tokenization.rb +22 -0
  185. data/lib/ruby_llm/protocols/cohere/tools.rb +132 -0
  186. data/lib/ruby_llm/protocols/cohere/transcription.rb +41 -0
  187. data/lib/ruby_llm/protocols/cohere.rb +21 -0
  188. data/lib/ruby_llm/protocols/converse/batches.rb +55 -0
  189. data/lib/ruby_llm/protocols/converse/chat.rb +685 -0
  190. data/lib/ruby_llm/protocols/converse/media.rb +178 -0
  191. data/lib/ruby_llm/protocols/converse/streaming.rb +424 -0
  192. data/lib/ruby_llm/protocols/converse.rb +54 -0
  193. data/lib/ruby_llm/protocols/deepgram/models.rb +74 -0
  194. data/lib/ruby_llm/protocols/deepgram/speech.rb +93 -0
  195. data/lib/ruby_llm/protocols/deepgram/streaming_transcription.rb +89 -0
  196. data/lib/ruby_llm/protocols/deepgram/transcription.rb +96 -0
  197. data/lib/ruby_llm/protocols/deepgram.rb +19 -0
  198. data/lib/ruby_llm/protocols/deepseek/files.rb +31 -0
  199. data/lib/ruby_llm/protocols/elevenlabs/assets.rb +36 -0
  200. data/lib/ruby_llm/protocols/elevenlabs/flows/images.rb +74 -0
  201. data/lib/ruby_llm/protocols/elevenlabs/flows/media.rb +42 -0
  202. data/lib/ruby_llm/protocols/elevenlabs/flows/videos.rb +93 -0
  203. data/lib/ruby_llm/protocols/elevenlabs/flows.rb +14 -0
  204. data/lib/ruby_llm/protocols/elevenlabs/models.rb +64 -0
  205. data/lib/ruby_llm/protocols/elevenlabs/speech.rb +67 -0
  206. data/lib/ruby_llm/protocols/elevenlabs/streaming_transcription.rb +127 -0
  207. data/lib/ruby_llm/protocols/elevenlabs/transcription.rb +61 -0
  208. data/lib/ruby_llm/protocols/elevenlabs.rb +15 -0
  209. data/lib/ruby_llm/protocols/files.rb +119 -0
  210. data/lib/ruby_llm/protocols/gemini/batches.rb +162 -0
  211. data/lib/ruby_llm/protocols/gemini/caches.rb +59 -0
  212. data/lib/ruby_llm/protocols/gemini/chat.rb +453 -0
  213. data/lib/ruby_llm/protocols/gemini/embedding_batches.rb +86 -0
  214. data/lib/ruby_llm/protocols/gemini/embeddings.rb +70 -0
  215. data/lib/ruby_llm/protocols/gemini/file_transcription.rb +32 -0
  216. data/lib/ruby_llm/protocols/gemini/files.rb +115 -0
  217. data/lib/ruby_llm/protocols/gemini/images.rb +183 -0
  218. data/lib/ruby_llm/protocols/gemini/live_transcription.rb +140 -0
  219. data/lib/ruby_llm/{providers → protocols}/gemini/media.rb +17 -11
  220. data/lib/ruby_llm/protocols/gemini/models.rb +71 -0
  221. data/lib/ruby_llm/protocols/gemini/speech.rb +56 -0
  222. data/lib/ruby_llm/protocols/gemini/streaming.rb +96 -0
  223. data/lib/ruby_llm/protocols/gemini/tools.rb +157 -0
  224. data/lib/ruby_llm/{providers → protocols}/gemini/transcription.rb +22 -22
  225. data/lib/ruby_llm/protocols/gemini/videos.rb +103 -0
  226. data/lib/ruby_llm/protocols/gemini.rb +35 -0
  227. data/lib/ruby_llm/protocols/gpustack/responses.rb +111 -0
  228. data/lib/ruby_llm/protocols/gpustack/tokenization.rb +22 -0
  229. data/lib/ruby_llm/protocols/gpustack/videos.rb +96 -0
  230. data/lib/ruby_llm/protocols/interactions/chat.rb +145 -0
  231. data/lib/ruby_llm/protocols/interactions/content.rb +90 -0
  232. data/lib/ruby_llm/protocols/interactions/streaming.rb +91 -0
  233. data/lib/ruby_llm/protocols/interactions/tools.rb +51 -0
  234. data/lib/ruby_llm/protocols/interactions/transcription.rb +58 -0
  235. data/lib/ruby_llm/protocols/interactions.rb +29 -0
  236. data/lib/ruby_llm/protocols/invoke_model/cohere_embeddings.rb +51 -0
  237. data/lib/ruby_llm/protocols/invoke_model/embedding_batches.rb +111 -0
  238. data/lib/ruby_llm/protocols/invoke_model/nova_embeddings.rb +50 -0
  239. data/lib/ruby_llm/protocols/invoke_model/stability_images.rb +103 -0
  240. data/lib/ruby_llm/protocols/invoke_model/titan_multimodal_embeddings.rb +33 -0
  241. data/lib/ruby_llm/protocols/invoke_model/titan_text_embeddings.rb +44 -0
  242. data/lib/ruby_llm/protocols/invoke_model.rb +57 -0
  243. data/lib/ruby_llm/protocols/mistral/content.rb +49 -0
  244. data/lib/ruby_llm/protocols/mistral/conversations/chat.rb +160 -0
  245. data/lib/ruby_llm/protocols/mistral/conversations/images.rb +43 -0
  246. data/lib/ruby_llm/protocols/mistral/conversations/streaming.rb +83 -0
  247. data/lib/ruby_llm/protocols/mistral/conversations.rb +30 -0
  248. data/lib/ruby_llm/protocols/mistral/files.rb +36 -0
  249. data/lib/ruby_llm/protocols/mistral/multi_completion.rb +160 -0
  250. data/lib/ruby_llm/protocols/openai/batches.rb +126 -0
  251. data/lib/ruby_llm/protocols/openai/files.rb +42 -0
  252. data/lib/ruby_llm/protocols/openrouter/batches.rb +147 -0
  253. data/lib/ruby_llm/protocols/openrouter/files.rb +24 -0
  254. data/lib/ruby_llm/protocols/openrouter/responses.rb +53 -0
  255. data/lib/ruby_llm/protocols/openrouter/transcription.rb +51 -0
  256. data/lib/ruby_llm/protocols/perplexity/files.rb +48 -0
  257. data/lib/ruby_llm/protocols/perplexity/router.rb +59 -0
  258. data/lib/ruby_llm/protocols/responses/approvals.rb +32 -0
  259. data/lib/ruby_llm/protocols/responses/batches.rb +32 -0
  260. data/lib/ruby_llm/protocols/responses/chat.rb +476 -0
  261. data/lib/ruby_llm/protocols/responses/compaction.rb +29 -0
  262. data/lib/ruby_llm/protocols/responses/media.rb +61 -0
  263. data/lib/ruby_llm/protocols/responses/streaming.rb +117 -0
  264. data/lib/ruby_llm/protocols/responses/token_counting.rb +26 -0
  265. data/lib/ruby_llm/protocols/responses/tools.rb +39 -0
  266. data/lib/ruby_llm/protocols/responses.rb +35 -0
  267. data/lib/ruby_llm/protocols/vertexai/batch_prediction.rb +155 -0
  268. data/lib/ruby_llm/protocols/vertexai/embedding_prediction/requests.rb +85 -0
  269. data/lib/ruby_llm/protocols/vertexai/embedding_prediction/results.rb +74 -0
  270. data/lib/ruby_llm/protocols/vertexai/embedding_prediction.rb +56 -0
  271. data/lib/ruby_llm/protocols/vertexai/files.rb +101 -0
  272. data/lib/ruby_llm/protocols/vertexai/ranking.rb +69 -0
  273. data/lib/ruby_llm/protocols/vertexai/research.rb +193 -0
  274. data/lib/ruby_llm/protocols/xai/files.rb +30 -0
  275. data/lib/ruby_llm/protocols/xai/streaming_transcription.rb +120 -0
  276. data/lib/ruby_llm/protocols/xai/tokenization.rb +23 -0
  277. data/lib/ruby_llm/provider.rb +560 -128
  278. data/lib/ruby_llm/providers/anthropic/capabilities.rb +5 -7
  279. data/lib/ruby_llm/providers/anthropic.rb +4 -6
  280. data/lib/ruby_llm/providers/azure/audio.rb +18 -0
  281. data/lib/ruby_llm/providers/azure/capabilities.rb +16 -0
  282. data/lib/ruby_llm/providers/azure/chat.rb +2 -9
  283. data/lib/ruby_llm/providers/azure/chat_completions/batches.rb +29 -0
  284. data/lib/ruby_llm/providers/azure/chat_completions.rb +80 -0
  285. data/lib/ruby_llm/providers/azure/cohere.rb +33 -0
  286. data/lib/ruby_llm/providers/azure/embeddings.rb +3 -2
  287. data/lib/ruby_llm/providers/azure/images.rb +22 -0
  288. data/lib/ruby_llm/providers/azure/media.rb +5 -14
  289. data/lib/ruby_llm/providers/azure/models.rb +35 -0
  290. data/lib/ruby_llm/providers/azure/responses.rb +26 -0
  291. data/lib/ruby_llm/providers/azure/videos.rb +64 -0
  292. data/lib/ruby_llm/providers/azure.rb +77 -78
  293. data/lib/ruby_llm/providers/bedrock/auth.rb +60 -41
  294. data/lib/ruby_llm/providers/bedrock/capabilities.rb +18 -0
  295. data/lib/ruby_llm/providers/bedrock/mantle/anthropic.rb +39 -0
  296. data/lib/ruby_llm/providers/bedrock/mantle/chat_completions.rb +23 -0
  297. data/lib/ruby_llm/providers/bedrock/mantle/responses.rb +24 -0
  298. data/lib/ruby_llm/providers/bedrock/mantle/voxtral.rb +96 -0
  299. data/lib/ruby_llm/providers/bedrock/mantle.rb +57 -0
  300. data/lib/ruby_llm/providers/bedrock/models.rb +193 -41
  301. data/lib/ruby_llm/providers/bedrock.rb +216 -45
  302. data/lib/ruby_llm/providers/cohere.rb +31 -0
  303. data/lib/ruby_llm/providers/deepgram.rb +37 -0
  304. data/lib/ruby_llm/providers/deepseek/capabilities.rb +4 -51
  305. data/lib/ruby_llm/providers/deepseek/chat.rb +49 -2
  306. data/lib/ruby_llm/providers/deepseek/responses.rb +68 -0
  307. data/lib/ruby_llm/providers/deepseek.rb +9 -2
  308. data/lib/ruby_llm/providers/elevenlabs.rb +35 -0
  309. data/lib/ruby_llm/providers/gemini/capabilities.rb +8 -107
  310. data/lib/ruby_llm/providers/gemini.rb +15 -8
  311. data/lib/ruby_llm/providers/gpustack/chat.rb +2 -16
  312. data/lib/ruby_llm/providers/gpustack/embeddings.rb +28 -0
  313. data/lib/ruby_llm/providers/gpustack/media.rb +17 -16
  314. data/lib/ruby_llm/providers/gpustack/models.rb +72 -62
  315. data/lib/ruby_llm/providers/gpustack/speech.rb +15 -0
  316. data/lib/ruby_llm/providers/gpustack/transcription.rb +29 -0
  317. data/lib/ruby_llm/providers/gpustack.rb +35 -11
  318. data/lib/ruby_llm/providers/mistral/capabilities.rb +7 -155
  319. data/lib/ruby_llm/providers/mistral/chat.rb +37 -61
  320. data/lib/ruby_llm/providers/mistral/chat_completions/batches.rb +120 -0
  321. data/lib/ruby_llm/providers/mistral/chat_completions.rb +21 -0
  322. data/lib/ruby_llm/providers/mistral/conversations.rb +12 -0
  323. data/lib/ruby_llm/providers/mistral/embeddings.rb +6 -4
  324. data/lib/ruby_llm/providers/mistral/media.rb +6 -18
  325. data/lib/ruby_llm/providers/mistral/models.rb +57 -23
  326. data/lib/ruby_llm/providers/mistral/ocr.rb +47 -0
  327. data/lib/ruby_llm/providers/mistral/speech.rb +51 -0
  328. data/lib/ruby_llm/providers/mistral/transcription.rb +62 -0
  329. data/lib/ruby_llm/providers/mistral.rb +16 -4
  330. data/lib/ruby_llm/providers/ollama/chat.rb +7 -13
  331. data/lib/ruby_llm/providers/ollama/media.rb +6 -15
  332. data/lib/ruby_llm/providers/ollama/models.rb +50 -9
  333. data/lib/ruby_llm/providers/ollama.rb +9 -8
  334. data/lib/ruby_llm/providers/ollama_cloud/models.rb +14 -0
  335. data/lib/ruby_llm/providers/ollama_cloud.rb +40 -0
  336. data/lib/ruby_llm/providers/openai/capabilities.rb +54 -259
  337. data/lib/ruby_llm/providers/openai/models.rb +23 -23
  338. data/lib/ruby_llm/providers/openai/responses.rb +13 -0
  339. data/lib/ruby_llm/providers/openai.rb +92 -11
  340. data/lib/ruby_llm/providers/openrouter/chat.rb +130 -108
  341. data/lib/ruby_llm/providers/openrouter/embeddings.rb +51 -0
  342. data/lib/ruby_llm/providers/openrouter/images.rb +44 -43
  343. data/lib/ruby_llm/providers/openrouter/media.rb +34 -0
  344. data/lib/ruby_llm/providers/openrouter/models.rb +50 -11
  345. data/lib/ruby_llm/providers/openrouter/speech.rb +32 -0
  346. data/lib/ruby_llm/providers/openrouter/streaming.rb +31 -38
  347. data/lib/ruby_llm/providers/openrouter/videos.rb +81 -0
  348. data/lib/ruby_llm/providers/openrouter.rb +78 -20
  349. data/lib/ruby_llm/providers/perplexity/chat.rb +2 -9
  350. data/lib/ruby_llm/providers/perplexity/embeddings.rb +32 -0
  351. data/lib/ruby_llm/providers/perplexity/media.rb +5 -21
  352. data/lib/ruby_llm/providers/perplexity/models.rb +80 -13
  353. data/lib/ruby_llm/providers/perplexity.rb +28 -20
  354. data/lib/ruby_llm/providers/vertexai/anthropic/batches.rb +52 -0
  355. data/lib/ruby_llm/providers/vertexai/anthropic.rb +34 -0
  356. data/lib/ruby_llm/providers/vertexai/capabilities.rb +19 -0
  357. data/lib/ruby_llm/providers/vertexai/chat_completions/batches.rb +54 -0
  358. data/lib/ruby_llm/providers/vertexai/chat_completions.rb +15 -0
  359. data/lib/ruby_llm/providers/vertexai/embed_content.rb +42 -0
  360. data/lib/ruby_llm/providers/vertexai/embeddings.rb +22 -7
  361. data/lib/ruby_llm/providers/vertexai/gemini/batches.rb +42 -0
  362. data/lib/ruby_llm/providers/vertexai/gemini.rb +69 -0
  363. data/lib/ruby_llm/providers/vertexai/live_transcription.rb +24 -0
  364. data/lib/ruby_llm/providers/vertexai/mistral.rb +28 -0
  365. data/lib/ruby_llm/providers/vertexai/models.rb +145 -43
  366. data/lib/ruby_llm/providers/vertexai/transcription.rb +49 -4
  367. data/lib/ruby_llm/providers/vertexai/videos.rb +61 -0
  368. data/lib/ruby_llm/providers/vertexai.rb +160 -17
  369. data/lib/ruby_llm/providers/xai/capabilities.rb +18 -0
  370. data/lib/ruby_llm/providers/xai/chat.rb +3 -2
  371. data/lib/ruby_llm/providers/xai/chat_completions/batches.rb +108 -0
  372. data/lib/ruby_llm/providers/xai/chat_completions.rb +19 -0
  373. data/lib/ruby_llm/providers/xai/images.rb +91 -0
  374. data/lib/ruby_llm/providers/xai/models.rb +32 -36
  375. data/lib/ruby_llm/providers/xai/reported_cost.rb +18 -0
  376. data/lib/ruby_llm/providers/xai/responses.rb +52 -0
  377. data/lib/ruby_llm/providers/xai/speech.rb +45 -0
  378. data/lib/ruby_llm/providers/xai/transcription.rb +48 -0
  379. data/lib/ruby_llm/providers/xai/videos.rb +87 -0
  380. data/lib/ruby_llm/providers/xai.rb +15 -5
  381. data/lib/ruby_llm/railtie.rb +7 -16
  382. data/lib/ruby_llm/rerank.rb +105 -0
  383. data/lib/ruby_llm/research_job.rb +241 -0
  384. data/lib/ruby_llm/search_results.rb +68 -0
  385. data/lib/ruby_llm/server_tool_call.rb +73 -0
  386. data/lib/ruby_llm/speech.rb +159 -0
  387. data/lib/ruby_llm/speech_chunk.rb +33 -0
  388. data/lib/ruby_llm/support/deprecator.rb +22 -0
  389. data/lib/ruby_llm/support/inspectable.rb +49 -0
  390. data/lib/ruby_llm/support/instrumentation.rb +41 -0
  391. data/lib/ruby_llm/support/utils.rb +147 -0
  392. data/lib/ruby_llm/thinking.rb +127 -20
  393. data/lib/ruby_llm/tokenization.rb +59 -0
  394. data/lib/ruby_llm/tokens.rb +103 -33
  395. data/lib/ruby_llm/tool.rb +266 -91
  396. data/lib/ruby_llm/tool_call.rb +36 -3
  397. data/lib/ruby_llm/tools/server_tools.rb +109 -0
  398. data/lib/ruby_llm/transcription/wav_audio.rb +62 -0
  399. data/lib/ruby_llm/transcription.rb +138 -13
  400. data/lib/ruby_llm/transcription_chunk.rb +68 -0
  401. data/lib/ruby_llm/transport/connection.rb +193 -0
  402. data/lib/ruby_llm/transport/error_middleware.rb +131 -0
  403. data/lib/ruby_llm/transport/usage_middleware.rb +28 -0
  404. data/lib/ruby_llm/transport/websocket_connection.rb +220 -0
  405. data/lib/ruby_llm/uploaded_file.rb +144 -0
  406. data/lib/ruby_llm/version.rb +2 -1
  407. data/lib/ruby_llm/video.rb +136 -0
  408. data/lib/ruby_llm/video_job.rb +150 -0
  409. data/lib/ruby_llm/workflow.rb +91 -0
  410. data/lib/ruby_llm.rb +380 -6
  411. data/lib/tasks/ruby_llm.rake +21 -16
  412. data/skills/rubyllm/SKILL.md +81 -0
  413. data/skills/rubyllm/agents/openai.yaml +4 -0
  414. metadata +338 -97
  415. data/lib/generators/ruby_llm/install/templates/add_references_to_chats_tool_calls_and_messages_migration.rb.tt +0 -9
  416. data/lib/generators/ruby_llm/install/templates/create_models_migration.rb.tt +0 -39
  417. data/lib/generators/ruby_llm/install/templates/create_tool_calls_migration.rb.tt +0 -21
  418. data/lib/generators/ruby_llm/install/templates/model_model.rb.tt +0 -3
  419. data/lib/generators/ruby_llm/install/templates/tool_call_model.rb.tt +0 -3
  420. data/lib/generators/ruby_llm/upgrade_to_v1_10/templates/add_v1_10_message_columns.rb.tt +0 -19
  421. data/lib/generators/ruby_llm/upgrade_to_v1_10/upgrade_to_v1_10_generator.rb +0 -50
  422. data/lib/generators/ruby_llm/upgrade_to_v1_14/templates/add_v1_14_tool_call_columns.rb.tt +0 -7
  423. data/lib/generators/ruby_llm/upgrade_to_v1_14/upgrade_to_v1_14_generator.rb +0 -49
  424. data/lib/generators/ruby_llm/upgrade_to_v1_7/templates/migration.rb.tt +0 -145
  425. data/lib/generators/ruby_llm/upgrade_to_v1_7/upgrade_to_v1_7_generator.rb +0 -122
  426. data/lib/generators/ruby_llm/upgrade_to_v1_9/templates/add_v1_9_message_columns.rb.tt +0 -15
  427. data/lib/generators/ruby_llm/upgrade_to_v1_9/upgrade_to_v1_9_generator.rb +0 -49
  428. data/lib/ruby_llm/active_record/acts_as_legacy.rb +0 -597
  429. data/lib/ruby_llm/active_record/model_methods.rb +0 -82
  430. data/lib/ruby_llm/active_record/tool_call_methods.rb +0 -18
  431. data/lib/ruby_llm/aliases.rb +0 -41
  432. data/lib/ruby_llm/connection.rb +0 -159
  433. data/lib/ruby_llm/content.rb +0 -91
  434. data/lib/ruby_llm/deprecator.rb +0 -24
  435. data/lib/ruby_llm/error_middleware.rb +0 -81
  436. data/lib/ruby_llm/instrumentation.rb +0 -36
  437. data/lib/ruby_llm/mime_type.rb +0 -96
  438. data/lib/ruby_llm/model/info.rb +0 -164
  439. data/lib/ruby_llm/model_registry.rb +0 -39
  440. data/lib/ruby_llm/models_schema.json +0 -171
  441. data/lib/ruby_llm/providers/anthropic/chat.rb +0 -291
  442. data/lib/ruby_llm/providers/anthropic/content.rb +0 -44
  443. data/lib/ruby_llm/providers/anthropic/embeddings.rb +0 -20
  444. data/lib/ruby_llm/providers/anthropic/media.rb +0 -92
  445. data/lib/ruby_llm/providers/anthropic/models.rb +0 -59
  446. data/lib/ruby_llm/providers/anthropic/streaming.rb +0 -71
  447. data/lib/ruby_llm/providers/bedrock/chat.rb +0 -405
  448. data/lib/ruby_llm/providers/bedrock/media.rb +0 -108
  449. data/lib/ruby_llm/providers/bedrock/streaming.rb +0 -328
  450. data/lib/ruby_llm/providers/gemini/chat.rb +0 -542
  451. data/lib/ruby_llm/providers/gemini/embeddings.rb +0 -37
  452. data/lib/ruby_llm/providers/gemini/images.rb +0 -47
  453. data/lib/ruby_llm/providers/gemini/models.rb +0 -38
  454. data/lib/ruby_llm/providers/gemini/streaming.rb +0 -98
  455. data/lib/ruby_llm/providers/gemini/tools.rb +0 -234
  456. data/lib/ruby_llm/providers/gpustack/capabilities.rb +0 -20
  457. data/lib/ruby_llm/providers/ollama/capabilities.rb +0 -20
  458. data/lib/ruby_llm/providers/openai/chat.rb +0 -236
  459. data/lib/ruby_llm/providers/openai/embeddings.rb +0 -33
  460. data/lib/ruby_llm/providers/openai/images.rb +0 -90
  461. data/lib/ruby_llm/providers/openai/moderation.rb +0 -34
  462. data/lib/ruby_llm/providers/openai/streaming.rb +0 -55
  463. data/lib/ruby_llm/providers/openai/temperature.rb +0 -28
  464. data/lib/ruby_llm/providers/openai/transcription.rb +0 -71
  465. data/lib/ruby_llm/providers/perplexity/capabilities.rb +0 -72
  466. data/lib/ruby_llm/providers/vertexai/chat.rb +0 -14
  467. data/lib/ruby_llm/providers/vertexai/streaming.rb +0 -14
  468. data/lib/ruby_llm/stream_accumulator.rb +0 -218
  469. data/lib/ruby_llm/streaming.rb +0 -179
  470. data/lib/ruby_llm/tool_concurrency.rb +0 -105
  471. data/lib/ruby_llm/utils.rb +0 -130
  472. data/lib/tasks/models.rake +0 -593
  473. data/lib/tasks/release.rake +0 -94
  474. data/lib/tasks/vcr.rake +0 -124
@@ -0,0 +1,111 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Protocols
5
+ class InvokeModel
6
+ # Titan embedding requests over Bedrock's S3 batch job API.
7
+ module EmbeddingBatches
8
+ include Protocols::Bedrock::Batches
9
+
10
+ private
11
+
12
+ def bedrock_invocation_type
13
+ 'InvokeModel'
14
+ end
15
+
16
+ def validate_bedrock_batch_requests!(requests)
17
+ requests.each do |request|
18
+ values = request.fetch(:text)
19
+ values = [values] unless values.is_a?(Array)
20
+ unless values.any? && values.all? { |value| value.is_a?(String) && !value.empty? }
21
+ raise ArgumentError, 'Bedrock embedding batches require nonempty text strings'
22
+ end
23
+ unless request.fetch(:payload).key?(:inputText)
24
+ raise ArgumentError, 'Bedrock embedding batches require Titan embedding requests'
25
+ end
26
+ end
27
+ end
28
+
29
+ def bedrock_batch_jsonl(requests)
30
+ requests.flat_map do |request|
31
+ array = request.fetch(:text).is_a?(Array)
32
+ texts = array ? request.fetch(:text) : [request.fetch(:text)]
33
+ texts.each_with_index.map do |text, index|
34
+ id = "rllm-#{request.fetch(:custom_id)}-#{index}-#{texts.size}-#{array ? 'a' : 's'}"
35
+ JSON.generate(recordId: id, modelInput: batch_payload(request).merge(inputText: text))
36
+ end
37
+ end.join("\n")
38
+ end
39
+
40
+ def bedrock_job_name(input_uri, requests:)
41
+ "ruby-llm-embed-#{requests.size}-#{Digest::SHA256.hexdigest(input_uri)[0, 16]}"
42
+ end
43
+
44
+ def parse_batch_response(data)
45
+ super.merge(request_count: data['jobName'].to_s[/\Aruby-llm-embed-(\d+)-[0-9a-f]+\z/, 1]&.to_i)
46
+ end
47
+
48
+ def parse_bedrock_outputs(outputs, model:)
49
+ records = outputs.flat_map { |body| body.to_s.each_line.map { |line| JSON.parse(line) } }
50
+ groups = records.group_by { |record| embedding_record_id(record).first }
51
+ groups.filter_map { |index, group| parse_embedding_batch_group(index, group, model:) }
52
+ end
53
+
54
+ def embedding_record_id(record)
55
+ id = record['recordId'] || record['record_id']
56
+ match = /\Arllm-(\d+)-(\d+)-(\d+)-([as])\z/.match(id.to_s)
57
+ raise Error, "Unknown Bedrock embedding record ID: #{id.inspect}" unless match
58
+
59
+ [Integer(match[1]), Integer(match[2]), Integer(match[3]), match[4] == 'a']
60
+ end
61
+
62
+ def parse_embedding_batch_group(index, records, model:)
63
+ metadata = records.map { |record| embedding_record_id(record) }
64
+ count = metadata.first[2]
65
+ array = metadata.first[3]
66
+ error = embedding_group_error(records, metadata)
67
+ return [index, nil, batch_failure(index, error)] if error
68
+ return if records.size < count
69
+
70
+ ordered = records.sort_by { |record| embedding_record_id(record)[1] }
71
+ unless ordered.map { |record| embedding_record_id(record)[1] } == (0...count).to_a
72
+ return [index, nil, batch_failure(index, 'Invalid embedding record positions')]
73
+ end
74
+
75
+ parse_embedding_batch_result(index, ordered, model:, array:)
76
+ end
77
+
78
+ def embedding_group_error(records, metadata)
79
+ return 'Invalid or duplicate embedding record positions' unless consistent_embedding_metadata?(metadata)
80
+
81
+ error = records.find { |record| !record['modelOutput'] }
82
+ return unless error
83
+
84
+ batch_error_value(error['error']) || error['errorMessage'] || 'Bedrock returned no model output'
85
+ end
86
+
87
+ def consistent_embedding_metadata?(metadata)
88
+ metadata.map { |item| item[2..] }.uniq.one? &&
89
+ metadata.map { |item| item[1] }.uniq.size == metadata.size &&
90
+ (metadata.first[3] || metadata.first[2] == 1)
91
+ end
92
+
93
+ def parse_embedding_batch_result(index, records, model:, array:)
94
+ bodies = records.map { |record| record.fetch('modelOutput') }
95
+ vectors = embedding_batch_vectors(bodies)
96
+ return [index, nil, batch_failure(index, 'Bedrock returned no embedding')] unless vectors
97
+
98
+ counts = bodies.map { |body| body['inputTextTokenCount'] }
99
+ tokens = counts.sum if counts.all?
100
+ result = Embedding.new(vectors: array ? vectors : vectors.first, model:, input_tokens: tokens)
101
+ [index, result]
102
+ end
103
+
104
+ def embedding_batch_vectors(bodies)
105
+ vectors = bodies.map { |body| extract_embedding(body) }
106
+ vectors if vectors.all? { |vector| vector.is_a?(Array) && !vector.empty? && vector.all?(Numeric) }
107
+ end
108
+ end
109
+ end
110
+ end
111
+ end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Protocols
5
+ class InvokeModel
6
+ # Amazon Nova multimodal embedding models over Bedrock InvokeModel.
7
+ class NovaEmbeddings < InvokeModel
8
+ DEFAULT_EMBEDDING_PURPOSE = 'GENERIC_INDEX'
9
+
10
+ # rubocop:disable-next Lint/UnusedMethodArgument
11
+ def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
12
+ ensure_no_embedding_media!(with)
13
+ track_usage(:embedding) do
14
+ responses = [text].flatten.map do |value|
15
+ payload = render_embedding_payload(value, dimensions:, task_type:, provider_options:)
16
+ signed_post(embedding_url(model:), payload).tap { |response| record_embedding_attempt(response) }
17
+ end
18
+
19
+ parse_single_embedding_responses(responses, model:, text:)
20
+ end
21
+ end
22
+
23
+ private
24
+
25
+ def render_embedding_payload(text, dimensions:, task_type:, provider_options:)
26
+ params = {
27
+ embeddingPurpose: task_type || DEFAULT_EMBEDDING_PURPOSE,
28
+ text: {
29
+ truncationMode: 'END',
30
+ value: text.to_s
31
+ }
32
+ }
33
+ params[:embeddingDimension] = dimensions if dimensions
34
+
35
+ deep_merge_provider_options(
36
+ {
37
+ taskType: 'SINGLE_EMBEDDING',
38
+ singleEmbeddingParams: params
39
+ },
40
+ provider_options
41
+ )
42
+ end
43
+
44
+ def extract_embedding(body)
45
+ body.dig('embeddings', 0, 'embedding')
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,103 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Protocols
5
+ class InvokeModel
6
+ # Stability image generation and inpainting over Bedrock InvokeModel.
7
+ class StabilityImages < InvokeModel
8
+ GENERATION_MODELS = %w[
9
+ stability.sd3-5-large-v1:0 stability.stable-image-core-v1:1 stability.stable-image-ultra-v1:1
10
+ ].freeze
11
+ INPAINT_MODEL = 'stability.stable-image-inpaint-v1:0'
12
+ MODELS = (GENERATION_MODELS + [INPAINT_MODEL]).freeze
13
+ ASPECT_RATIOS = %w[16:9 1:1 21:9 2:3 3:2 4:5 5:4 9:16 9:21].freeze
14
+ IMAGE_TYPES = %w[image/jpeg image/png image/webp].freeze
15
+
16
+ def paint(prompt, model:, size:, count: nil, with: nil, mask: nil, provider_options: {})
17
+ track_usage(:image) do
18
+ payload = render_image_payload(prompt, model:, size:, count:, with:, mask:, provider_options:)
19
+ response = signed_post("/model/#{model}/invoke", payload)
20
+ images = parse_image_responses(response, model:)
21
+ images.each { |image| image.config = @config }
22
+ images.one? ? images.first : images
23
+ end
24
+ end
25
+
26
+ def render_image_payload(prompt, model:, size:, count: nil, with: nil, mask: nil, provider_options: {})
27
+ RubyLLM.logger.debug { 'Stability image models return one image per request' } if count && count > 1
28
+ attachments = Attachment.wrap(with, config: @config)
29
+ payload = { prompt: }.merge(render_image_inputs(attachments, model:, mask:, size:))
30
+ payload.merge(provider_options.transform_keys(&:to_sym))
31
+ end
32
+
33
+ def parse_image_responses(response, model:)
34
+ images = Array(response.body['images']).compact.reject(&:empty?)
35
+ if images.empty?
36
+ reasons = Array(response.body['finish_reasons']).compact.join(', ')
37
+ message = reasons.empty? ? 'Bedrock returned no images' : "Bedrock image generation failed: #{reasons}"
38
+ raise Error.new(message, response:)
39
+ end
40
+
41
+ images.map do |data|
42
+ mime_type = RubyLLM::Files::MimeType.for(StringIO.new(Base64.decode64(data)))
43
+ raise Error.new('Bedrock returned invalid image data', response:) unless IMAGE_TYPES.include?(mime_type)
44
+
45
+ Image.new(data:, model:, mime_type:)
46
+ end
47
+ end
48
+
49
+ private
50
+
51
+ def render_image_inputs(attachments, model:, mask:, size:)
52
+ return render_inpaint_inputs(attachments, mask:, size:) if model.end_with?(INPAINT_MODEL)
53
+ raise UnsupportedAttachmentError, 'image mask' if mask
54
+ return { aspect_ratio: image_aspect_ratio(size) }.compact if attachments.empty?
55
+ raise UnsupportedAttachmentError, 'image reference' unless model.end_with?('stability.sd3-5-large-v1:0')
56
+
57
+ { image: single_image(attachments, size:), mode: 'image-to-image', strength: 0.5 }
58
+ end
59
+
60
+ def render_inpaint_inputs(attachments, mask:, size:)
61
+ payload = { image: single_image(attachments, size:) }
62
+ payload[:mask] = single_image(Attachment.wrap(mask, config: @config), size: nil) if mask
63
+ payload
64
+ end
65
+
66
+ def single_image(attachments, size:)
67
+ raise ArgumentError, 'with: must contain exactly one image' unless attachments.one?
68
+ raise ArgumentError, 'size: cannot change dimensions when editing a Stability image' if size
69
+
70
+ encoded_image(attachments.first)
71
+ end
72
+
73
+ def encoded_image(attachment)
74
+ raise UnsupportedAttachmentError, attachment.mime_type unless IMAGE_TYPES.include?(attachment.mime_type)
75
+
76
+ attachment.encoded
77
+ end
78
+
79
+ def image_aspect_ratio(size)
80
+ return unless size
81
+
82
+ match = size.to_s.match(/\A(\d+)\s*[x×:]\s*(\d+)\z/i)
83
+ raise ArgumentError, "Invalid image size: #{size.inspect}" unless match
84
+
85
+ width, height = match.captures.map(&:to_i)
86
+ raise ArgumentError, "Invalid image size: #{size.inspect}" unless width.positive? && height.positive?
87
+
88
+ ratio = matching_aspect_ratio(width, height)
89
+ raise ArgumentError, "Unsupported Stability image aspect ratio: #{size}" unless ratio
90
+
91
+ ratio
92
+ end
93
+
94
+ def matching_aspect_ratio(width, height)
95
+ ASPECT_RATIOS.find do |candidate|
96
+ ratio_width, ratio_height = candidate.split(':').map(&:to_i)
97
+ width * ratio_height == height * ratio_width
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
@@ -0,0 +1,33 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Protocols
5
+ class InvokeModel
6
+ # Amazon Titan multimodal embedding models over Bedrock InvokeModel.
7
+ class TitanMultimodalEmbeddings < InvokeModel
8
+ # rubocop:disable-next Lint/UnusedMethodArgument
9
+ def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
10
+ ensure_no_embedding_media!(with)
11
+ track_usage(:embedding) do
12
+ responses = [text].flatten.map do |value|
13
+ payload = render_embedding_payload(value, dimensions:, provider_options:)
14
+ signed_post(embedding_url(model:), payload).tap { |response| record_embedding_attempt(response) }
15
+ end
16
+
17
+ parse_single_embedding_responses(responses, model:, text:)
18
+ end
19
+ end
20
+
21
+ private
22
+
23
+ def render_embedding_payload(text, dimensions:, provider_options:, **)
24
+ payload = {}
25
+ payload[:inputText] = text.to_s unless text.nil? || text.to_s.empty?
26
+ payload[:embeddingConfig] = { outputEmbeddingLength: dimensions } if dimensions
27
+
28
+ deep_merge_provider_options(payload, provider_options)
29
+ end
30
+ end
31
+ end
32
+ end
33
+ end
@@ -0,0 +1,44 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Protocols
5
+ class InvokeModel
6
+ # Amazon Titan text embedding models over Bedrock InvokeModel.
7
+ class TitanTextEmbeddings < InvokeModel
8
+ # rubocop:disable-next Lint/UnusedMethodArgument
9
+ def embed(text, model:, dimensions:, task_type: nil, title: nil, with: nil, provider_options: {})
10
+ ensure_no_embedding_media!(with)
11
+ track_usage(:embedding) do
12
+ responses = [text].flatten.map do |value|
13
+ payload = render_embedding_payload(value, model:, dimensions:, provider_options:)
14
+ signed_post(embedding_url(model:), payload).tap { |response| record_embedding_attempt(response) }
15
+ end
16
+
17
+ parse_single_embedding_responses(responses, model:, text:)
18
+ end
19
+ end
20
+
21
+ private
22
+
23
+ # The G1 and V1 models take inputText alone; Bedrock rejects the V2
24
+ # tuning keys as extraneous.
25
+ def render_embedding_payload(text, model:, dimensions:, provider_options:, **)
26
+ payload = { inputText: text.to_s }
27
+
28
+ if titan_v2?(model)
29
+ payload[:dimensions] = dimensions if dimensions
30
+ payload[:normalize] = true
31
+ elsif dimensions
32
+ raise Error, "#{model} does not support custom dimensions"
33
+ end
34
+
35
+ deep_merge_provider_options(payload, provider_options)
36
+ end
37
+
38
+ def titan_v2?(model)
39
+ model.to_s.include?('titan-embed-text-v2')
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Protocols
5
+ # The AWS InvokeModel API, where the envelope is shared but each model
6
+ # family owns its request and response body. Subclasses under
7
+ # Protocols::InvokeModel implement one family each. Requests are signed
8
+ # by the provider, which knows the credentials and the signing service.
9
+ class InvokeModel < Protocol
10
+ private
11
+
12
+ def signed_post(url, payload, additional_headers = {})
13
+ body = JSON.generate(payload)
14
+
15
+ @connection.post(url, payload, usage: @usage_tracker) do |req|
16
+ req.headers.merge!(@provider.sign_headers('POST', url, body))
17
+ req.headers.merge!(additional_headers) unless additional_headers.empty?
18
+ end
19
+ end
20
+
21
+ def embedding_url(model:)
22
+ "/model/#{model}/invoke"
23
+ end
24
+
25
+ def ensure_no_embedding_media!(with)
26
+ attachments = Attachment.wrap(with)
27
+ raise UnsupportedAttachmentError, attachments.first.mime_type if attachments.any?
28
+ end
29
+
30
+ def parse_single_embedding_responses(responses, model:, text:)
31
+ vectors = responses.map { |response| extract_embedding(response.body) }
32
+ input_tokens = Tokens.aggregate(embedding_attempt_tokens(responses)).input
33
+ vectors = vectors.first unless text.is_a?(Array)
34
+
35
+ Embedding.new(vectors:, model:, input_tokens:)
36
+ end
37
+
38
+ def record_embedding_attempt(response)
39
+ @usage_tracker&.succeed_attempts(tokens: embedding_attempt_tokens([response]))
40
+ end
41
+
42
+ def embedding_attempt_tokens(responses)
43
+ responses.map { |response| Tokens.new(input: response.body['inputTextTokenCount']) }
44
+ end
45
+
46
+ def deep_merge_provider_options(payload, provider_options)
47
+ return payload if provider_options.empty?
48
+
49
+ Support::Utils.deep_merge(payload, provider_options)
50
+ end
51
+
52
+ def extract_embedding(body)
53
+ body['embedding'] || body.dig('embeddingsByType', 'float') || body['embeddingsByType']&.values&.first
54
+ end
55
+ end
56
+ end
57
+ end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Protocols
5
+ module Mistral
6
+ module Content # :nodoc:
7
+ module_function
8
+
9
+ def parse_conversation_content(output)
10
+ result = { text: +'', thinking: +'', attachments: [], citations: [] }
11
+ output.select { |entry| entry['type'] == 'message.output' }.each do |entry|
12
+ parse_conversation_parts(entry['content'], result)
13
+ end
14
+ result[:thinking] = nil if result[:thinking].empty?
15
+ result
16
+ end
17
+
18
+ def parse_conversation_parts(content, result)
19
+ return result[:text] << content if content.is_a?(String)
20
+
21
+ Array(content).compact.each do |part|
22
+ case part['type']
23
+ when 'text' then result[:text] << part['text'].to_s
24
+ when 'thinking' then result[:thinking] << Array(part['thinking']).filter_map { |item| item['text'] }.join
25
+ when 'tool_reference' then result[:citations] << parse_conversation_citation(part, result[:text].length)
26
+ when 'tool_file' then result[:attachments] << parse_conversation_file(part)
27
+ when 'image_url'
28
+ url = part['image_url'].is_a?(Hash) ? part['image_url']['url'] : part['image_url']
29
+ result[:attachments] << Attachment.new(url, config: @config)
30
+ end
31
+ end
32
+ end
33
+
34
+ def parse_conversation_citation(part, offset)
35
+ Citation.new(url: part['url'], title: part['title'], cited_text: part['description'],
36
+ start_index: offset, end_index: offset)
37
+ end
38
+
39
+ def parse_conversation_file(part)
40
+ format = part['file_type'].to_s
41
+ mime_type = format.include?('/') ? format : RubyLLM::Files::MimeType.for(name: "file.#{format}")
42
+ file = UploadedFile.new(id: part.fetch('file_id'), provider: @provider.slug,
43
+ filename: part['file_name'], mime_type: mime_type, downloadable: true, metadata: part)
44
+ Attachment.new(file, config: @config)
45
+ end
46
+ end
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,160 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Protocols
5
+ module Mistral
6
+ class Conversations
7
+ module Chat # :nodoc:
8
+ module_function
9
+
10
+ def completion_url
11
+ 'conversations'
12
+ end
13
+
14
+ def render(...)
15
+ payload = super
16
+ payload[:tools] = Support::Utils.deep_stringify_keys(payload[:tools]).uniq
17
+ if payload[:tools].any? { |tool| Array(tool.dig('tool_configuration', 'requires_confirmation')).any? }
18
+ raise ArgumentError, 'Mistral hosted tool confirmations require provider conversation storage'
19
+ end
20
+
21
+ payload
22
+ end
23
+
24
+ def render_payload(messages, tools:, temperature:, model:, stream: false, max_output_tokens: nil,
25
+ schema: nil, thinking: nil, tool_prefs: nil, **)
26
+ options = super(messages, tools:, temperature:, model:, stream:, max_output_tokens:,
27
+ schema:, thinking:, citations: false, caching: nil, tool_prefs:)
28
+ normalize_conversation_choice(options)
29
+ {
30
+ model: model.id,
31
+ inputs: format_entries(messages),
32
+ instructions: messages.select { |message| message.role == :system }.map(&:content).join("\n\n"),
33
+ completion_args: options.slice(:temperature, :max_tokens, :response_format, :reasoning_effort,
34
+ :tool_choice),
35
+ tools: options.fetch(:tools, []),
36
+ store: false,
37
+ stream: stream
38
+ }
39
+ end
40
+
41
+ def normalize_conversation_choice(options)
42
+ choice = options[:tool_choice]
43
+ if choice.is_a?(Hash)
44
+ raise ArgumentError, 'Mistral Conversations supports :auto, :none, or :required for tool choice'
45
+ end
46
+
47
+ options[:tool_choice] = 'any' if choice == 'required'
48
+ end
49
+
50
+ def format_entries(messages)
51
+ entries = messages.reject { |message| message.role == :system }.flat_map do |message|
52
+ if message.raw_content
53
+ Array(message.raw_content)
54
+ elsif message.tool_result?
55
+ [{ type: 'function.result', tool_call_id: message.tool_call_id, result: message.content.to_s }]
56
+ else
57
+ format_conversation_message(message)
58
+ end
59
+ end
60
+ entries.each_with_index.flat_map { |entry, index| replay_conversation_entry(entry, index) }
61
+ end
62
+
63
+ def replay_conversation_entry(entry, index)
64
+ return replay_conversation_execution(entry, index) if entry['type'] == 'tool.execution'
65
+
66
+ result = entry.except('id', 'object', 'created_at', 'completed_at')
67
+ if entry['type'] == 'message.output' && entry['content'].is_a?(Array)
68
+ result['content'] = entry['content'].map do |part|
69
+ next part unless part['type'] == 'tool_reference'
70
+
71
+ { 'type' => 'text', 'text' => "[#{part['title']}](#{part['url']})" }
72
+ end
73
+ end
74
+ [result]
75
+ end
76
+
77
+ def replay_conversation_execution(entry, index)
78
+ identity = entry['id'] || "#{index}:#{JSON.generate(entry)}"
79
+ id = entry['tool_call_id'] || Digest::SHA256.hexdigest(identity)[0, 9]
80
+ info = entry['info']
81
+ result = info.is_a?(Hash) && info.key?('result') ? info['result'] : info
82
+ [
83
+ { 'type' => 'function.call', 'tool_call_id' => id,
84
+ 'name' => entry['function'] || entry['name'], 'arguments' => entry['arguments'] },
85
+ { 'type' => 'function.result', 'tool_call_id' => id,
86
+ 'result' => result.is_a?(String) ? result : JSON.generate(result) }
87
+ ]
88
+ end
89
+
90
+ def format_conversation_message(message)
91
+ entries = []
92
+ if message.content || message.attachments.any?
93
+ entries << {
94
+ type: 'message.input', role: message.role.to_s,
95
+ content: format_message_content(message)
96
+ }
97
+ end
98
+ message.tool_calls&.each_value do |call|
99
+ entries << { type: 'function.call', tool_call_id: call.id, name: call.name,
100
+ arguments: JSON.generate(call.arguments) }
101
+ end
102
+ entries
103
+ end
104
+
105
+ def parse_completion_body(data, raw:)
106
+ output = data.fetch('outputs')
107
+ content = parse_conversation_content(output)
108
+ calls = parse_conversation_calls(output, raw:)
109
+ response_model = output.filter_map { |entry| entry['model'] }.last || @model&.id
110
+ Message.new(
111
+ role: :assistant, content: content[:text], attachments: content[:attachments],
112
+ citations: content[:citations], thinking: Thinking.build(text: content[:thinking]),
113
+ tool_calls: calls, server_tool_calls: parse_conversation_steps(output),
114
+ raw_content: output, model: response_model,
115
+ finish_reason: calls.empty? ? :stop : :tool_calls, raw: raw,
116
+ **parse_conversation_usage(data['usage'] || {})
117
+ )
118
+ end
119
+
120
+ def parse_conversation_calls(output, raw:)
121
+ if output.any? { |entry| entry['type'] == 'function.call' && entry['confirmation_status'] == 'pending' }
122
+ raise Error.new('Mistral returned a hosted tool confirmation that requires provider conversation storage',
123
+ response: raw)
124
+ end
125
+
126
+ output.select { |entry| pending_conversation_call?(entry) }.to_h do |entry|
127
+ call = parse_tool_calls([
128
+ { 'id' => entry['tool_call_id'], 'type' => 'function',
129
+ 'function' => entry.slice('name', 'arguments') }
130
+ ], response: raw, finish_reason: :tool_calls).values.first
131
+ [call.id, call]
132
+ end
133
+ end
134
+
135
+ def pending_conversation_call?(entry)
136
+ entry['type'] == 'function.call' && entry['confirmation_status'].nil?
137
+ end
138
+
139
+ def parse_conversation_steps(output)
140
+ output.filter_map do |entry|
141
+ next unless entry['type'] == 'tool.execution' ||
142
+ (entry['type'] == 'function.call' && !pending_conversation_call?(entry))
143
+
144
+ ServerToolCall.new(type: entry['type'], name: entry['name'], id: entry['id'],
145
+ input: entry['arguments'], result: entry['info'], raw: entry)
146
+ end
147
+ end
148
+
149
+ def parse_conversation_usage(usage)
150
+ input = usage['prompt_tokens'] && (usage['prompt_tokens'] + usage.fetch('connector_tokens', 0).to_i)
151
+ {
152
+ input_tokens: input,
153
+ output_tokens: usage['completion_tokens'], server_tool_use: usage['connectors']
154
+ }
155
+ end
156
+ end
157
+ end
158
+ end
159
+ end
160
+ end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RubyLLM
4
+ module Protocols
5
+ module Mistral
6
+ class Conversations
7
+ module Images # :nodoc:
8
+ def images_url(**)
9
+ 'conversations'
10
+ end
11
+
12
+ def render_image_payload(prompt, model:, size:, count: nil, with: nil, mask: nil, provider_options: {})
13
+ if size || (count && count != 1)
14
+ raise ArgumentError,
15
+ 'Mistral image generation does not accept size or count options'
16
+ end
17
+ raise UnsupportedAttachmentError, 'image editing' if with || mask
18
+
19
+ payload = { model: model, store: false, inputs: prompt, tools: [{ type: 'image_generation' }] }
20
+ Support::Utils.deep_merge(payload, provider_options)
21
+ end
22
+
23
+ def parse_image_response(response, model:)
24
+ parse_image_responses(response, model:).first
25
+ end
26
+
27
+ def parse_image_responses(response, model:)
28
+ data = response.body
29
+ attachments = parse_conversation_content(data.fetch('outputs'))[:attachments]
30
+ raise Error.new('Mistral returned no generated image', response:) if attachments.empty?
31
+
32
+ usage = parse_conversation_usage(data['usage'] || {}).transform_keys(&:to_s)
33
+ attachments.each_with_index.map do |attachment, index|
34
+ bytes = @provider.download_file(attachment.provider_file_id)
35
+ Image.new(data: Base64.strict_encode64(bytes), mime_type: RubyLLM::Files::MimeType.for(StringIO.new(bytes)),
36
+ model: model, usage: index.zero? ? usage : {})
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end