insika 0.0.1 → 0.2.0

data/bin/insika

@@ -0,0 +1,366 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# `insika` CLI — strict config + doctor --fix. The operator's
+# DX front door to the strict-config discipline:
+#
+#   insika doctor [--fix] [--json]   diagnose the deployment config; --fix applies
+#                                     the safe autofixes; exits non-zero on any error
+#   insika env    [--json]           list the known config keys + current (masked) values
+#   insika version                   print the engine version
+#   insika help                      this text
+#
+# It reads the SAME durable backend the server does (INSIKA_DB -> SQLite, else
+# ephemeral memory), building only the config stores it needs — no full app boot, no
+# DEEPSEEK, no seeding. So it is safe to run against a live deployment's volume.
+
+require "optparse"
+require_relative "../lib/insika"
+
+module Insika
+  module CLI
+    module_function
+
+    def main(argv)
+      command = argv.shift || "help"
+      case command
+      when "doctor"  then doctor(argv)
+      when "env"     then env(argv)
+      when "refine"  then refine(argv)
+      when "evals:import" then evals_import(argv)
+      when "evals:export" then evals_export(argv)
+      when "evals:baseline" then evals_baseline(argv)
+      when "version", "--version", "-v" then puts "insika #{Insika::VERSION}"
+      when "help", "--help", "-h" then puts usage
+      else
+        warn "unknown command: #{command}\n\n#{usage}"
+        exit 2
+      end
+    end
+
+    def usage
+      <<~TXT
+        insika #{Insika::VERSION} — agent engine CLI
+
+        Usage: insika <command> [options]
+
+        Commands:
+          doctor [--fix] [--json]   diagnose the deployment configuration
+          env    [--json]           list known config keys + current (masked) values
+          refine --agent ID [...]   report what broke in an agent's real traffic
+          evals:import [--dir D]    load the golden corpus into the store (editable in Studio)
+          evals:export [--dir D]    write the stored cases back out as YAML
+          evals:baseline <sub>      the accepted state per agent: show | import | export
+          version                   print the engine version
+          help                      show this help
+
+        Environment:
+          INSIKA_DB                 SQLite path (durable). Unset -> ephemeral memory.
+          INSIKA_CONFIG_STRICT      when truthy, the server refuses boot on any finding.
+      TXT
+    end
+
+    # insika doctor [--fix] [--json]
+    def doctor(argv)
+      opts = { fix: false, json: false }
+      OptionParser.new do |o|
+        o.banner = "Usage: insika doctor [--fix] [--json]"
+        o.on("--fix", "apply the safe autofixes, then re-diagnose") { opts[:fix] = true }
+        o.on("--json", "emit the report as JSON") { opts[:json] = true }
+      end.parse!(argv)
+
+      doc = build_doctor
+      report = opts[:fix] ? doc.fix!.last : doc.run
+
+      if opts[:json]
+        require "json"
+        puts JSON.pretty_generate(report.to_h)
+      else
+        puts report.to_s(color: $stdout.tty?)
+      end
+      exit(report.ok? ? 0 : 1)
+    end
+
+    # insika env [--json]
+    def env(argv)
+      json = false
+      OptionParser.new do |o|
+        o.banner = "Usage: insika env [--json]"
+        o.on("--json", "emit as JSON") { json = true }
+      end.parse!(argv)
+
+      rows = Insika::EnvSchema.known_specs.map do |s|
+        raw = ENV[s.name]
+        { "key" => s.name, "type" => s.type.to_s, "secret" => s.secret?,
+          "set" => !raw.nil?, "value" => display_value(s, raw), "description" => s.description }
+      end
+
+      if json
+        require "json"
+        puts JSON.pretty_generate(rows)
+      else
+        rows.each do |r|
+          state = r["set"] ? r["value"] : "(unset)"
+          puts format("%-30s %-8s %s", r["key"], r["type"], state)
+        end
+      end
+    end
+
+    # insika refine --agent ID [--last-sessions N] [--since ISO8601] [--full] [--json]
+    #
+    # reads the agent's own traffic and records a ranked failure
+    # report. Read-only with respect to the agent — it writes only the run record.
+    # There is no scheduler in the engine: this command and the Studio button
+    # are how a run starts, and a cron that wants one calls this.
+    def refine(argv)
+      opts = { json: false, full: false }
+      OptionParser.new do |o|
+        o.banner = "Usage: insika refine --agent ID [options]"
+        o.on("--agent ID", "agent whose traffic to read (required)") { |v| opts[:agent] = v }
+        o.on("--last-sessions N", Integer, "read the N most recent conversations") { |v| opts[:last_sessions] = v }
+        o.on("--since ISO8601", "read only turns from this instant on") { |v| opts[:since] = v }
+        o.on("--full", "ignore the previous run and read the whole window") { opts[:full] = true }
+        o.on("--exclude PREFIXES", "comma-separated session-id prefixes to drop (e.g. loadtest-,debug-)") do |v|
+          opts[:exclude_sessions] = v.split(",").map(&:strip).reject(&:empty?)
+        end
+        o.on("--json", "emit the report as JSON") { opts[:json] = true }
+      end.parse!(argv)
+
+      abort "insika refine: --agent is required" if Insika::Coercion.blank?(opts[:agent])
+
+      # A misspelled agent or a bad window is operator error, not a bug: one line and
+      # exit 2 (like an unknown command), never a Ruby backtrace. Exit 1 stays what it
+      # means everywhere else in this CLI — "it ran and found trouble".
+      run = begin
+        run_refinement(opts)
+      rescue Insika::Error, ArgumentError => e
+        warn "insika refine: #{e.message}"
+        exit 2
+      end
+      if opts[:json]
+        require "json"
+        puts JSON.pretty_generate(run.to_h)
+      else
+        puts refine_report(run)
+      end
+      exit(run.status == :failed ? 1 : 0)
+    end
+
+# insika evals:import [--dir evals/golden] [--keep-existing]
+#
+# The corpus on disk is the SEED and the export format; the store is what a
+# deployment runs and what the Studio edits. Importing is how a
+# fresh deploy gets the cases, and how a broken edit is undone.
+def evals_import(argv)
+  opts = { dir: File.expand_path("../evals/golden", __dir__), overwrite: true }
+  OptionParser.new do |o|
+    o.banner = "Usage: insika evals:import [--dir DIR] [--keep-existing]"
+    o.on("--dir DIR", "corpus directory (default evals/golden)") { |v| opts[:dir] = v }
+    o.on("--keep-existing", "do not overwrite a case already in the store") { opts[:overwrite] = false }
+  end.parse!(argv)
+
+  ids = golden_store.import_dir(opts[:dir], overwrite: opts[:overwrite])
+  puts "imported #{ids.size} case(s) from #{opts[:dir]}"
+  puts ids.sort.map { |id| "  #{id}" }.join("\n") unless ids.empty?
+rescue Insika::Evals::GoldenLoader::InvalidGolden => e
+  warn "insika evals:import: #{e.message}"
+  exit 2
+end
+
+# insika evals:export [--dir DIR] — the store back to YAML, one file per case under
+# <dir>/<agent>/<id>.yml. Same format `evals:import` reads: no converter to keep
+# honest, so a case authored in the Studio can be reviewed in a pull request.
+def evals_export(argv)
+  opts = { dir: File.expand_path("../evals/golden", __dir__), force: false }
+  OptionParser.new do |o|
+    o.banner = "Usage: insika evals:export [--dir DIR] [--force]"
+    o.on("--dir DIR", "destination (default evals/golden)") { |v| opts[:dir] = v }
+    o.on("--force", "rewrite an existing corpus (drops the files' comments)") { opts[:force] = true }
+  end.parse!(argv)
+
+  paths = golden_store.export_dir(opts[:dir], force: opts[:force])
+  puts "exported #{paths.size} case(s) to #{opts[:dir]}"
+rescue Insika::ValidationError => e
+  warn "insika evals:export: #{e.message}"
+  exit 2
+end
+
+# insika evals:baseline show|import|export [--file evals/baseline.json]
+#
+# The accepted state of a golden set used to live ONLY in `evals/baseline.json`,
+# which is fine for the CLI (it runs from a checkout) and useless for the refinement
+# gate, which runs inside a deployment and has to know what "no
+# regression" means without one. So it became a per-agent record.
+#
+# `import` splits the one file into those records, resolving each case's agent from
+# the golden store — which is why the corpus has to be imported first. A case the
+# store does not know is REPORTED, not guessed at: silently dropping it would shrink
+# the accepted state, and a smaller baseline is a weaker gate.
+def evals_baseline(argv)
+  sub = argv.shift || "show"
+  opts = { file: File.expand_path("../evals/baseline.json", __dir__) }
+  OptionParser.new do |o|
+    o.banner = "Usage: insika evals:baseline show|import|export [--file PATH]"
+    o.on("--file PATH", "the baseline JSON (default evals/baseline.json)") { |v| opts[:file] = v }
+  end.parse!(argv)
+
+  case sub
+  when "show"   then baseline_show
+  when "import" then baseline_import(opts[:file])
+  when "export" then baseline_export(opts[:file])
+  else
+    warn "insika evals:baseline: unknown subcommand '#{sub}' (show|import|export)"
+    exit 2
+  end
+rescue Insika::Error => e
+  warn "insika evals:baseline: #{e.message}"
+  exit 2
+end
+
+def baseline_show
+  store = baseline_store
+  agents = store.agents
+  return puts("no baseline recorded — the refinement gate cannot run (insika evals:baseline import)") if agents.empty?
+
+  agents.sort.each do |agent|
+    record = store.get(agent)
+    puts format("%-24s %3d case(s)   recorded %s", agent, (record["cases"] || {}).size, record["at"])
+  end
+end
+
+def baseline_import(path)
+  raise Insika::ValidationError, "no such file: #{path}" unless File.exist?(path)
+
+  blob = JSON.parse(File.read(path))
+  by_agent = Hash.new { |h, k| h[k] = {} }
+  unknown = []
+  (blob["cases"] || {}).each do |case_id, entry|
+    golden = golden_store.find(case_id)
+    next unknown << case_id if golden.nil?
+
+    by_agent[golden.agent][case_id] = entry
+  end
+
+  by_agent.each { |agent, cases| baseline_store.put(agent, { "at" => blob["at"], "cases" => cases }) }
+  puts "imported #{by_agent.values.sum(&:size)} case(s) into #{by_agent.size} agent baseline(s) from #{path}"
+  by_agent.keys.sort.each { |a| puts "  #{a} (#{by_agent[a].size})" }
+  return if unknown.empty?
+
+  warn "  #{unknown.size} case(s) are not in the golden store and were NOT recorded: #{unknown.sort.join(', ')}"
+  warn "  run `insika evals:import` first, then import the baseline again"
+end
+
+def baseline_export(path)
+  store = baseline_store
+  merged = store.agents.each_with_object({}) { |a, acc| acc.merge!(store.get(a)["cases"] || {}) }
+  at = store.agents.filter_map { |a| store.get(a)["at"] }.max
+  File.write(path, JSON.pretty_generate({ "at" => at, "cases" => merged }))
+  puts "exported #{merged.size} case(s) from #{store.agents.size} agent baseline(s) to #{path}"
+end
+
+    # -- helpers -------------------------------------------------------
+
+    def baseline_store
+      backend = Insika::Wiring::Graph.backend_from_env
+      Insika::BaselineStore.new(config_store: Insika::ConfigStore.new(store: backend))
+    end
+
+    def golden_store
+      backend = Insika::Wiring::Graph.backend_from_env
+      Insika::GoldenStore.new(config_store: Insika::ConfigStore.new(store: backend))
+    end
+
+    # Builds the handler over the real durable backend — same "no app boot" rule as
+    # `doctor`: no DEEPSEEK, no seeding, no reactor. Safe against a live volume,
+    # because the only write is the run record.
+    def run_refinement(opts)
+      backend = Insika::Wiring::Graph.backend_from_env
+      config_store = Insika::ConfigStore.new(store: backend)
+      profiles = Insika::StoredProfileSource.new(config_store: config_store)
+      handler = Insika::Commands::RunRefinement.new(
+        profiles: profiles,
+        refinement_store: Insika::RefinementStore.new(store: backend),
+        collector: Insika::Refinement::EvidenceCollector.new(
+          task_store: Insika::TaskStore.new(store: backend),
+          session_store: Insika::SessionStore.new(store: backend),
+          tool_trace_store: Insika::ToolTraceStore.new(store: backend),
+          profiles: profiles,
+          settings_store: Insika::SettingsStore.new(config_store: config_store)
+        ),
+        event_stream: Insika::EventStream.new
+      )
+      handler.call(Insika::Command.build(:run_refinement, {
+                                          agent: opts[:agent], since: opts[:since],
+                                          last_sessions: opts[:last_sessions], full: opts[:full],
+                                          exclude_sessions: opts[:exclude_sessions]
+                                        }.compact, transport: :cli))
+    end
+
+    INDENT = " " * 23 # aligns the provenance line under the finding's title
+
+    def refine_report(run)
+      head = "#{run.agent_id} — #{run.status} (#{refine_window(run.window)})"
+      return "#{head}\n  #{run.error}" if run.status == :failed
+      return "#{head}\n  nothing found in this window" if run.findings.empty?
+
+      rows = run.findings.map do |f|
+        line = format("  %-14s ×%-4d %s", f["kind"], f["count"], f["title"])
+        line += "\n#{INDENT}#{f['detail']}" if f["detail"]
+        line += "\n#{INDENT}sessions: #{Array(f['sessions']).join(', ')}" if Array(f["sessions"]).any?
+        line
+      end
+      ([head] + rows).join("\n")
+    end
+
+    # An empty window in the record means "the collector's own default" — resolve it
+    # for display instead of printing the word "default".
+    def refine_window(window)
+      return "since #{window['since']}" if window["since"]
+
+      n = window["last_sessions"] || Insika::Refinement::EvidenceCollector::DEFAULT_WINDOW
+      "last #{n} session(s)"
+    end
+
+    # Config stores over the real durable backend — no app boot / no DEEPSEEK.
+    def build_doctor
+      backend = Insika::Wiring::Graph.backend_from_env
+      config_store = Insika::ConfigStore.new(store: backend)
+      Insika::Doctor.new(
+        env: ENV,
+        settings_store: Insika::SettingsStore.new(config_store: config_store),
+        llm_provider_store: Insika::LLMProviderStore.new(config_store: config_store),
+        tool_store: Insika::ToolStore.new(config_store: config_store),
+        # Prompt-file corruption is invisible from the outside (the file is there, the
+        # agent answers) — the sweep is the only thing that names it.
+        agent_file_store: Insika::AgentFileStore.new(config_store: config_store),
+        # Skills + profiles: the eagerness/drift checks are about the RELATION between
+        # a skill and the agents holding it, so neither half alone is enough. The
+        # catalog brings the DISK seeds into the sweep (deploy/skills in a deployment,
+        # skills/ in the minimal wiring — whichever exists here); without it a stale
+        # `eager:` or a Natura-in-shared-body sitting in a seed pack is invisible.
+        skill_store: Insika::SkillStore.new(config_store: config_store),
+        skill_catalog: Insika::SkillCatalog.new(skill_roots),
+        profile_source: Insika::StoredProfileSource.new(config_store: config_store),
+        backend: backend
+      )
+    end
+
+    # The conventional seed roots, relative to where the operator runs the CLI —
+    # same "no app boot" rule as the rest of `doctor`, so the composition root that
+    # actually declares them is never loaded.
+    def skill_roots
+      %w[deploy/skills skills].map { |p| File.join(Dir.pwd, p) }.select { |p| Dir.exist?(p) }
+    end
+
+    def display_value(spec, raw)
+      return "(unset)" if raw.nil?
+
+      spec.secret? ? "•••• (set, #{raw.to_s.length} chars)" : raw
+    end
+  end
+end
+
+# No `if $PROGRAM_NAME == __FILE__` guard: the RubyGems wrapper `load`s this file
+# (Gem.activate_bin_path), so under `gem install` the guard is false and the CLI
+# would silently do nothing. This file is only ever an entrypoint.
+Insika::CLI.main(ARGV)