scout-ai 1.1.8 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f6c4a5eda08ca50c1353e6b36d41bc0b5c8ac1c6a58d40b48f23f005c8566c84
-  data.tar.gz: e61823c2c1344aaab60ed78504db4edbfa3776442436da2e58399619407159f0
+  metadata.gz: a95a3c664d483df9e6c5d968822ac358e74215770e7387186f5de12ac72cedeb
+  data.tar.gz: 55e9c02ef0d7794ba797c4fe9c1fdc023c0b08d32178386d53eabcdcb317a0b9
 SHA512:
-  metadata.gz: 75a8092ba29b8d75a141c0abff7d519e91f140e56b0220ba4a59512fcc9172aa13b9b068face217f418ec2afc3ef4f0377dac4e0d124b6399bc033b9d2a0bf6f
-  data.tar.gz: 1aac91f51c220a78c5f1a4d3aa77a7ccf07d7a8929f20a869ec702e67deff7d43a18d8c19913005b99dace1666a73715ee0fa9cafbfacccfb78f2ed60493482d
+  metadata.gz: 5252115360b664c8353c05f417b62992bc2970fd04cf1c77281c6eb0ebb185d58ce8b4f3909729b00b7d8b205bf09eb2ccf6ee530044393cf5139b0b44e54f16
+  data.tar.gz: 59358958fdae5ed1aac9edcffde28ffc21bd348ffd589d467be59a005b4dd2c2e942981399f412c710eec630e748a9a2438273cb2a569aa09c17388bc2e25927

data/.vimproject

@@ -1,28 +1,6 @@
 scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.txt *.conf" {
- Rakefile
  README.md
- LICENSE.txt
- bin=bin filter="*"{
-  scout-ai
- }
  chats=chats filter="*"{
-  agent_doc
-
-  tool_oeu
-
-  multi_agent.rb
-
-  network
-
-  intro
-
-  dev
-
-  inline_task
-
-  inline_task.rb
-
-  course
   system=system{
    scout-ai
   }
@@ -32,26 +10,6 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
    model
    model.analysis
   }
-  develop=develop{
-   causalLM
-   chat
-   digest
-   finetuning
-   json_format
-   model
-   rf
-   training=training{
-    basics
-    data.tmp
-    intro
-    python
-    data=data{
-     main
-     next_token
-     test.rb
-    }
-   }
-  }
   refactor=refactor{
    chat
   }
@@ -75,12 +33,21 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
    cmd
    web
   }
- }
- doc=doc filter="*"{
-  Agent.md
-  Chat.md
-  LLM.md
-  Model.md
+
+  develop=develop{
+   training=training{
+    data=data{
+    }
+   }
+
+   scout-ai
+   agent
+  }
+
+  log_reponse
+  test_log_resonse
+
+  test_prompt.rb
  }
  lib=lib {
   scout-ai.rb
@@ -92,13 +59,6 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
    }
    llm=llm{
     utils.rb
-    tools.rb
-    tools=tools{
-     mcp.rb
-     workflow.rb
-     knowledge_base.rb
-     call.rb
-    }
     chat.rb
     chat=chat{
      annotation.rb
@@ -109,16 +69,29 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
       files.rb
       clear.rb
       options.rb
+      meta.rb
      }
     }
+    tools.rb
+    tools=tools{
+     mcp.rb
+     workflow.rb
+     knowledge_base.rb
+     call.rb
+    }
 
     backends=backends{
-     openai.rb
+     default.rb
+
      responses.rb
-     anthropic.rb
+     openai.rb
      ollama.rb
-     bedrock.rb
+     anthropic.rb
+     vllm.rb
+
      openwebui.rb
+
+     bedrock.rb
      huggingface.rb
      relay.rb
     }
@@ -183,6 +156,18 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
    }
   }
  }
+ share=share{
+  server=server files="*"{
+   chat.html
+   chat.js
+  }
+ }
+ doc=doc filter="*"{
+  Agent.md
+  Chat.md
+  LLM.md
+  Model.md
+ }
  test=test {
   data=data filter="*"{
    person=person{
@@ -194,7 +179,8 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
   }
   test_helper.rb
  }
- data=data filter="*"{
+ bin=bin filter="*"{
+  scout-ai
  }
  scout_commands=scout_commands filter="*"{
   documenter
@@ -209,10 +195,4 @@ scout-ai=$PWD filter="*.rb *.rake Rakefile *.rdoc *.R *.sh *.js *.haml *.sass *.
    kb
   }
  }
- share=share{
-  server=server files="*"{
-   chat.html
-   chat.js
-  }
- }
 }

data/README.md

@@ -22,8 +22,8 @@ Scout originates from the Rbbt ecosystem (bioinformatics workflows). Numerous en
 
 The sections below summarize the main components (LLM, Chat, Agent, Model), quick starts, and the command‑line interface. For full APIs, see the doc/ directory.
 
-- doc/LLM.md — multi‑backend LLM orchestration, tool calling, embeddings
-- doc/Chat.md — conversation builder/serializer
+- doc/LLM.md — multi‑backend LLM orchestration, tool calling, endpoints, CLI
+- doc/Chat.md — chat files: roles/options, compilation pipeline, persistence
 - doc/Agent.md — stateful agents wired to Workflows and KnowledgeBases
 - doc/Model.md — model wrappers (ScoutModel, Python/Torch/Hugging Face)
 
@@ -37,9 +37,10 @@ Scout is a Ruby framework. Add scout-ai (and the other packages you need) to you
   - Python 3 (installed and visible in PATH)
   - pycall gem (Ruby ↔ Python bridge)
   - Python packages: torch, transformers, numpy, pandas (as needed)
-- For OpenAI or similar backends: set API keys in environment or config (see LLM backend docs)
+- For OpenAI/Anthropic/etc backends: set API keys in environment or config (see `doc/LLM.md`)
 
 Typical Gemfile fragment:
+
 ```ruby
 gem 'scout-essentials', git: 'https://github.com/mikisvaz/scout-essentials'
 gem 'scout-gear',       git: 'https://github.com/mikisvaz/scout-gear'
@@ -47,26 +48,64 @@ gem 'scout-rig',        git: 'https://github.com/mikisvaz/scout-rig'
 gem 'scout-ai',         git: 'https://github.com/mikisvaz/scout-ai'
 ```
 
-Backends and endpoints can be configured under Scout.etc.AI/<endpoint>.yaml (merged into asks), or via environment variables per backend (see doc/LLM.md).
+### Endpoints (recommended)
+
+Backends and endpoints can be configured via:
+
+- per-endpoint YAML files (recommended): `~/.scout/etc/AI/<endpoint>`
+- environment variables per backend (see `doc/LLM.md`)
+
+Most teams create a few named endpoints (e.g. `nano`, `deep`, `ollama`) and then reference them with:
+
+- Ruby: `endpoint: :nano`
+- CLI: `-e nano`
 
 
 ## Quick starts
 
+### Configure an endpoint (once)
+
+Create `~/.scout/etc/AI/nano`:
+
+```yaml
+backend: responses
+model: gpt-5-nano
+```
+
+Or a higher-effort endpoint `~/.scout/etc/AI/deep`:
+
+```yaml
+backend: responses
+model: gpt-5
+reasoning_effort: high
+text_verbosity: high
+```
+
+Keys beyond `backend/url/model` are passed through to the backend.
+
 ### Ask a model
 
+Ruby:
+
 ```ruby
 require 'scout-ai'
-answer = LLM.ask "What is the capital of France?", backend: :openai, model: "gpt-4.1-mini"
+answer = LLM.ask "What is the capital of France?", endpoint: :nano
 puts answer
 ```
 
+CLI:
+
+```bash
+scout-ai llm ask -e nano "What is the capital of France?"
+```
+
 Chat builder:
 
 ```ruby
 chat = Chat.setup []
 chat.system "You are a terse assistant"
 chat.user   "List three colors"
-puts chat.ask
+puts chat.ask(endpoint: :nano)
 ```
 
 ### Tool calling with a Workflow
@@ -83,11 +122,13 @@ m = Module.new do
   input :name, :string
   input :age, :integer
   input :gender, :select, nil, select_options: %w(male female)
-  task :person => :yaml do inputs.to_hash end
+  task :person => :yaml do
+    inputs.to_hash
+  end
 end
 
 puts LLM.workflow_ask(m, "Register Eduard Smith, a 25 yo male, using a tool call",
-                      backend: 'ollama', model: 'llama3')
+                      endpoint: :nano)
 ```
 
 ### Stateful agent with a KnowledgeBase
@@ -102,15 +143,17 @@ TmpFile.with_dir do |dir|
              undirected: true, source: "=>Alias", target: "=>Alias"
   kb.register :parents, datafile_test(:person).parents
 
-  agent = LLM::Agent.new knowledge_base: kb
-  puts agent.ask "Who is Miki's brother in law?"
+  agent = LLM::Agent.new(knowledge_base: kb, endpoint: :nano)
+  agent.start
+  agent.user "Who is Miki's brother in law?"
+  puts agent.chat
 end
 ```
 
 ### Structured iteration
 
 ```ruby
-agent = LLM::Agent.new
+agent = LLM::Agent.new(endpoint: :nano)
 agent.iterate("List three steps to bake bread") { |step| puts "- #{step}" }
 
 agent.iterate_dictionary("Give capital cities for FR, ES, IT") do |country, capital|
@@ -148,33 +191,28 @@ end
 
 A compact, multi‑backend layer to ask LLMs, wire function‑calling tools, parse/print chats, and compute embeddings.
 
-- ask(question, options={}, &block) — normalize a question to messages (LLM.chat), merge endpoint/model/format, run backend, and return assistant output (or messages with return_messages: true)
-- Backends: OpenAI‑style, Responses (multimodal, JSON schema), Ollama, OpenWebUI, AWS Bedrock, and a simple Relay
-- Tools: export Workflow tasks (LLM.workflow_tools) and KnowledgeBase lookups; tool calls are handled via a block
-- Embeddings and a tiny RAG helper
-- Chat/print pipeline: imports, clean, tasks/jobs as function calls, files/directories as tagged content
-- Configuration: endpoint defaults in Scout.etc.AI/endpoint.yaml are merged into options automatically
+- `LLM.ask(question, options={}, &block)` — compile `question` via `LLM.chat`, merge endpoint/model/format options, call backend
+- Backends: Responses, OpenAI, Anthropic, Ollama, vLLM, OpenWebUI, AWS Bedrock, Relay
+- Tools: export Workflow tasks and KnowledgeBase databases as function tools
+- Chat compilation pipeline: imports, clear/skip, tasks/jobs, files/directories
+- Endpoint configuration: `~/.scout/etc/AI/<endpoint>`
 
 ### Chat (doc/Chat.md)
 
-A lightweight builder over an Array of {role:, content:} messages with helpers:
+Chat is both:
 
-- user/system/assistant, file/directory tagging, import/continue
-- tool/workflow task declarations, jobs/inline jobs
-- association declarations (KnowledgeBase)
-- option, endpoint, model, format (including JSON schema requests)
-- ask, chat, json/json_format, print/save/write/write_answer, branch/shed
+- a builder over an Array of `{role:, content:}` messages
+- a stable on-disk “chat file” format used by the CLI
 
-Use Chat to author “chat files” on disk or build conversations programmatically.
+See `doc/Chat.md` for the full list of special roles (options, tools, imports, files, tasks, MCP, KB).
 
 ### Agent (doc/Agent.md)
 
-A thin orchestrator around Chat and LLM that keeps state and injects tools:
+An Agent is a stateful wrapper around Chat and LLM:
 
-- Maintains a live conversation (start_chat, start, current_chat)
-- Auto‑exports Workflow tasks and a KnowledgeBase traversal tool
-- ask/chat/json/iterate helpers; structured iteration over lists/dictionaries
-- load_from_path(dir) — bootstrap from a directory containing workflow.rb, knowledge_base, start_chat
+- maintains a current conversation (`start_chat`, `start`, `current_chat`)
+- auto-exports Workflow tasks and KnowledgeBase databases as tools
+- provides `chat/json/json_format/iterate` helpers
 
 ### Model (doc/Model.md)
 
@@ -188,11 +226,6 @@ A composable framework to wrap models with a consistent API:
   - CausalModel — chat/causal generation (supports apply_chat_template)
   - NextTokenModel — simple next‑token fine‑tuning loop
 
-Pattern:
-- Keep feature extraction separate from evaluation
-- Use eval_list to batch large tables
-- Persist directory state and behavior to reuse
-
 
 ## Example: ExTRI2 workflow (models in practice)
 
@@ -203,47 +236,50 @@ The ExTRI2 Workflow (Rbbt‑Workflows) uses HuggingfaceModel to score TRI senten
 - Adds fields “Valid score” and “Valid” to the TSV
 - Runs a second SequenceClassification model to produce “MoR” and “MoR scores”
 
-See workflows/ExTRI2/workflow.rb in that repository for the full implementation.
+See workflow.rb in that repository for the full implementation. http://github.com/Rbbt-Workflows/ExTRI2
 
 
 ## Command‑Line Interface
 
-The bin/scout dispatcher locates scripts under scout_commands across installed packages and workflows using the Path subsystem. Resolution works by adding terms until a file is found to execute:
+The bin/scout dispatcher locates scripts under scout_commands across installed packages and workflows using the Path subsystem.
+
+You can run it as:
 
-- If the fragment maps to a directory, a listing of available subcommands is shown
-- Scripts can be nested arbitrarily (e.g., agent/kb)
-- Other packages or workflows can define their own scripts under share/scout_commands, and bin/scout will find them
+- `scout ...` (the standard Scout CLI), or
+- `scout-ai ...` (a thin wrapper that loads Scout with `scout-ai` available)
 
 ### scout llm …
 
 Ask an LLM, manage chat files, run a minimal web UI, or process queued requests. Scripts live under scout_commands/llm.
 
 - Ask
-  - scout llm ask [options] [question]
+  - `scout llm ask [options] [question]`
+  - `scout-ai llm ask [options] [question]`
     - -t|--template <file_or_key> — load a prompt template; substitutes “???” or appends
     - -c|--chat <chat_file> — load/extend a conversation (appends the reply)
     - -i|--inline <file> — answer “# ask: …” directives inline in a source file
     - -f|--file <file> — prepend file content or substitute where “...” appears
-    - -m|--model, -e|--endpoint, -b|--backend — select backend/model; merged with Scout.etc.AI
+    - -m|--model, -e|--endpoint, -b|--backend — select backend/model; merged with endpoint configs
     - -d|--dry_run — expand and print the conversation (no ask)
 
 - Relay processor (for the Relay backend)
-  - scout llm process [directory] — watches a queue directory and answers ask JSONs
+  - `scout llm process [directory]` — watches a queue directory and answers ask JSONs
 
 - Web UI server
-  - scout llm server — static chat UI over ./chats with a small JSON API
+  - `scout llm server` — static chat UI over ./chats with a small JSON API
 
 - Templates
-  - scout llm template — list installed prompt templates (Scout.questions)
+  - `scout llm template` — list installed prompt templates (Scout.questions)
 
-Run “scout llm” alone to see available subcommands. If you target a directory (e.g., “scout llm”), a help‑like listing is printed.
+Run `scout llm` alone to see available subcommands.
 
 ### scout agent …
 
 Stateful agents with Workflow and KnowledgeBase tooled up. Scripts live under scout_commands/agent.
 
 - Ask via an Agent
-  - scout agent ask [options] [agent_name] [question]
+  - `scout agent ask [options] [agent_name] [question]`
+  - `scout-ai agent ask [options] [agent_name] [question]`
     - -l|--log <level> — set log severity
     - -t|--template <file_or_key>
     - -c|--chat <chat_file>
@@ -253,31 +289,25 @@ Stateful agents with Workflow and KnowledgeBase tooled up. Scripts live under sc
     - agent_name resolves via Scout.workflows[agent_name] (a workflow) or Scout.chats[agent_name] (an agent directory with workflow.rb/knowledge_base/start_chat)
 
 - KnowledgeBase passthrough
-  - scout agent kb <agent_name> <kb subcommand...>
-    - Loads the agent’s knowledge base and forwards to “scout kb …” (see scout-gear doc/KnowledgeBase.md for kb CLI)
+  - `scout agent kb <agent_name> <kb subcommand...>`
 
-As with other Scout CLIs, if you target a directory of commands (e.g., “scout agent”), bin/scout will show the subcommand listing.
-
-Note: Workflows also have extensive CLI commands (scout workflow …) for job execution, provenance, orchestration, and queue processing. When you integrate models inside tasks, you drive them through the workflow CLI (see scout-gear doc/Workflow.md).
+Note: Workflows also have extensive CLI commands (`scout workflow …`) for job execution, provenance, orchestration, and queue processing.
 
 
 ## Configuration, persistence and reproducibility
 
-- Endpoint presets: place YAML under Scout.etc.AI/<endpoint>.yaml to preconfigure URLs, models, headers, etc.; CLI options and chat inline options override defaults
+- Endpoint presets: place YAML under `~/.scout/etc/AI/<endpoint>` to preconfigure url/model/backend and backend-specific knobs
 - Tool calling: Workflow tasks are exported as JSON schemas per backend; results are serialized back to the model as tool replies
-- Caching: LLM.ask persists responses (by default) using Persist.persist; disable with persist: false
-- Models: pass a directory to persist options/behavior/state (Torch/HF use state files or save_pretrained directories); save/restore to reuse
-- Chats: save printable conversations with Chat#save; reuse with “scout llm ask -c <file>”
-
-For Python models, ensure scout-rig (ScoutPython) is installed and Python packages are present. See doc/Python.md in scout-rig for details.
+- Caching: `LLM.ask` persists responses (by default) using `Persist.persist`; disable with `persist: false`
+- Chats: save printable conversations with Chat#save; reuse with `scout-ai llm ask -c <file>`
 
 
 ## Where to go next
 
 - Explore the API docs shipped in this repository:
-  - doc/LLM.md — orchestration, backends, tools, CLI
-  - doc/Chat.md — conversation DSL and file format
-  - doc/Agent.md — stateful agents, Workflow/KB wiring, iterate helpers
+  - doc/LLM.md — orchestration, endpoints, backends, tools, CLI
+  - doc/Chat.md — chat files: roles/options and compilation behavior
+  - doc/Agent.md — stateful agents, Workflow/KB wiring, delegation, iterate helpers
   - doc/Model.md — model wrappers; ScoutModel, Python/Torch/Hugging Face
 
 - Browse real‑world workflows (including ExTRI2) in Rbbt‑Workflows:
@@ -293,4 +323,4 @@ For Python models, ensure scout-rig (ScoutPython) is installed and Python packag
 
 ## License and contributions
 
-Issues and PRs are welcome across the Scout repositories. Please open tickets in the relevant package (e.g., scout-ai for LLM/Agent/Model topics).
+Issues and PRs are welcome across the Scout repositories. Please open tickets in the relevant package (e.g., scout-ai for LLM/Agent/Model topics).

data/VERSION

@@ -1 +1 @@
-1.1.8
+1.2.0

data/bin/scout-ai

@@ -6,8 +6,8 @@ if _i = ARGV.index("--log")
   require 'scout/log'
   log = ARGV[_i+1]
   Log.severity = log.to_i
-  ARGV.delete "--log"
-  ARGV.delete log
+  ARGV.delete_at _i + 1
+  ARGV.delete_at _i
 end
 
 dev_dir = nil