@remnic/cli 1.0.3 → 1.0.5

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Joshua Warren
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -36,10 +36,54 @@ remnic query "hello" --explain  # Test query with tier breakdown
 | `remnic dedup` | Find and merge duplicate memories |
 | `remnic sync` | Diff-aware sync with external sources |
 | `remnic spaces` | Manage memory namespaces |
-| `remnic bench` | Run memory retrieval benchmarks |
+| `remnic bench list` | List published benchmark packs |
+| `remnic bench datasets status/download` | Check or download local benchmark datasets |
+| `remnic bench runs list/show/delete` | Manage stored benchmark result files |
+| `remnic bench run` | Run one or more published benchmark packs |
+| `remnic bench compare` | Compare two stored benchmark results |
+| `remnic bench baseline` | Save or list named benchmark baselines |
+| `remnic bench export` | Export a stored benchmark result as JSON, CSV, or HTML |
+| `remnic bench providers discover` | Auto-detect local provider backends |
+| `remnic bench publish --target remnic-ai` | Build the Remnic.ai benchmark feed from stored results |
 
 Run `remnic --help` for the full command list.
 
+## Benchmarks
+
+The phase-1 benchmark surface is exposed through `remnic bench`, with `remnic benchmark`
+kept as a compatibility alias.
+
+```bash
+remnic bench list
+remnic bench datasets status
+remnic bench datasets download longmemeval
+remnic bench datasets download --all
+remnic bench runs list
+remnic bench runs show candidate-run --detail
+remnic bench runs delete candidate-run
+remnic bench run --quick longmemeval
+remnic bench run longmemeval --dataset-dir ~/datasets/longmemeval
+remnic bench compare base-run candidate-run
+remnic bench baseline save main candidate-run
+remnic bench baseline list
+remnic bench export candidate-run --format csv --output ./candidate.csv
+remnic bench export candidate-run --format html --output ./report.html
+remnic bench providers discover
+remnic bench publish --target remnic-ai
+remnic benchmark run --quick longmemeval
+```
+
+`--quick` uses the lightweight benchmark path with a single-item limit so you can
+smoke-test the harness without running a full benchmark pass. When a benchmark
+ships a bundled smoke fixture, `--quick` uses that tracked fixture by default;
+full runs need a real benchmark dataset. In a repo checkout the CLI will use
+`evals/datasets/<benchmark>` automatically; in packaged installs pass
+`--dataset-dir <path>` explicitly.
+
+`remnic bench datasets download` currently manages the script-backed published
+datasets for `ama-bench`, `memory-arena`, `amemgym`, `longmemeval`, and `locomo`.
+Other benchmark fixtures remain repo-managed or need manual dataset wiring.
+
 ## Connecting agents
 
 Once the daemon is running, connect any supported agent:

package/dist/assets/download-datasets.sh

@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+# Honor an explicit DATASETS_DIR from the environment so packaged CLI
+# installs can route downloads to a user-writable location (e.g.
+# ~/.remnic/bench/datasets) instead of a sibling of the script dir.
+DATASETS_DIR="${DATASETS_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)/datasets}"
+
+usage() {
+  echo "Usage: $0 [--benchmark <name>]"
+  echo ""
+  echo "Downloads benchmark datasets for the Engram eval suite."
+  echo ""
+  echo "Benchmarks: ama-bench, longmemeval, amemgym, locomo, memory-arena, all"
+  echo ""
+  echo "Options:"
+  echo "  --benchmark <name>   Download only the specified benchmark (default: all)"
+  echo "  --help               Show this help"
+  exit 0
+}
+
+BENCHMARK="all"
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --benchmark) BENCHMARK="$2"; shift 2 ;;
+    --help) usage ;;
+    *) echo "Unknown option: $1"; usage ;;
+  esac
+done
+
+check_deps() {
+  for cmd in git curl; do
+    if ! command -v "$cmd" &>/dev/null; then
+      echo "ERROR: $cmd is required but not found"
+      exit 1
+    fi
+  done
+}
+
+download_ama_bench() {
+  local dir="$DATASETS_DIR/ama-bench"
+  if [[ -f "$dir/open_end_qa_set.jsonl" ]]; then
+    echo "[ama-bench] Already downloaded at $dir"
+    return
+  fi
+  echo "[ama-bench] Downloading from HuggingFace (AMA-bench/AMA-bench)..."
+  mkdir -p "$dir"
+  local tmpdir
+  tmpdir=$(mktemp -d)
+  git clone --depth 1 https://huggingface.co/datasets/AMA-bench/AMA-bench "$tmpdir/repo" 2>/dev/null || {
+    echo "[ama-bench] ERROR: Could not clone. Try manually:"
+    echo "  git clone --depth 1 https://huggingface.co/datasets/AMA-bench/AMA-bench /tmp/amabench"
+    echo "  cp /tmp/amabench/test/open_end_qa_set.jsonl $dir/"
+    rm -rf "$tmpdir"
+    return 1
+  }
+  cp "$tmpdir/repo/test/open_end_qa_set.jsonl" "$dir/" 2>/dev/null || true
+  rm -rf "$tmpdir"
+  echo "[ama-bench] Downloaded to $dir ($(wc -l < "$dir/open_end_qa_set.jsonl") episodes)"
+}
+
+download_longmemeval() {
+  local dir="$DATASETS_DIR/longmemeval"
+  if [[ -f "$dir/longmemeval_oracle.json" ]]; then
+    echo "[longmemeval] Already downloaded at $dir"
+    return
+  fi
+  echo "[longmemeval] Downloading from HuggingFace (xiaowu0162/longmemeval-cleaned)..."
+  mkdir -p "$dir"
+  curl -sL "https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_oracle.json" \
+    -o "$dir/longmemeval_oracle.json"
+  if [[ ! -s "$dir/longmemeval_oracle.json" ]]; then
+    echo "[longmemeval] ERROR: Download failed. Try manually:"
+    echo "  curl -sL https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main/longmemeval_oracle.json -o $dir/longmemeval_oracle.json"
+    rm -f "$dir/longmemeval_oracle.json"
+    return 1
+  fi
+  echo "[longmemeval] Downloaded to $dir ($(du -h "$dir/longmemeval_oracle.json" | cut -f1))"
+}
+
+download_amemgym() {
+  local dir="$DATASETS_DIR/amemgym"
+  if [[ -f "$dir/amemgym-v1-base.json" ]]; then
+    echo "[amemgym] Already downloaded at $dir"
+    return
+  fi
+  echo "[amemgym] Downloading from HuggingFace (AGI-Eval/AMemGym)..."
+  mkdir -p "$dir"
+  local tmpdir
+  tmpdir=$(mktemp -d)
+  git clone --depth 1 https://huggingface.co/datasets/AGI-Eval/AMemGym "$tmpdir/repo" 2>/dev/null || {
+    echo "[amemgym] ERROR: Could not clone. Try manually:"
+    echo "  git clone --depth 1 https://huggingface.co/datasets/AGI-Eval/AMemGym /tmp/amemgym"
+    echo "  cp /tmp/amemgym/v1.base/data.json $dir/amemgym-v1-base.json"
+    rm -rf "$tmpdir"
+    return 1
+  }
+  cp "$tmpdir/repo/v1.base/data.json" "$dir/amemgym-v1-base.json" 2>/dev/null || true
+  rm -rf "$tmpdir"
+  echo "[amemgym] Downloaded to $dir"
+}
+
+download_locomo() {
+  local dir="$DATASETS_DIR/locomo"
+  if [[ -f "$dir/locomo10.json" ]]; then
+    echo "[locomo] Already downloaded at $dir"
+    return
+  fi
+  echo "[locomo] Downloading from GitHub (snap-research/locomo)..."
+  mkdir -p "$dir"
+  local tmpdir
+  tmpdir=$(mktemp -d)
+  git clone --depth 1 https://github.com/snap-research/locomo.git "$tmpdir/repo" 2>/dev/null || {
+    echo "[locomo] ERROR: Could not clone. Try manually:"
+    echo "  git clone --depth 1 https://github.com/snap-research/locomo.git /tmp/locomo"
+    echo "  cp /tmp/locomo/data/locomo10.json $dir/"
+    rm -rf "$tmpdir"
+    return 1
+  }
+  cp "$tmpdir/repo/data/locomo10.json" "$dir/" 2>/dev/null || true
+  rm -rf "$tmpdir"
+  echo "[locomo] Downloaded to $dir ($(du -h "$dir/locomo10.json" | cut -f1))"
+}
+
+download_memory_arena() {
+  local dir="$DATASETS_DIR/memory-arena"
+  if [[ -d "$dir" ]] && ls "$dir"/*.jsonl &>/dev/null; then
+    echo "[memory-arena] Already downloaded at $dir"
+    return
+  fi
+  echo "[memory-arena] Downloading from HuggingFace (ZexueHe/memoryarena)..."
+  mkdir -p "$dir"
+  local tmpdir
+  tmpdir=$(mktemp -d)
+  git clone --depth 1 https://huggingface.co/datasets/ZexueHe/memoryarena "$tmpdir/repo" 2>/dev/null || {
+    echo "[memory-arena] ERROR: Could not clone. Try manually:"
+    echo "  git clone --depth 1 https://huggingface.co/datasets/ZexueHe/memoryarena /tmp/memoryarena"
+    echo "  for d in /tmp/memoryarena/*/; do cp \"\$d/data.jsonl\" \"$dir/\$(basename \$d).jsonl\"; done"
+    rm -rf "$tmpdir"
+    return 1
+  }
+  for d in "$tmpdir/repo"/*/; do
+    local name
+    name=$(basename "$d")
+    if [[ -f "$d/data.jsonl" ]]; then
+      cp "$d/data.jsonl" "$dir/${name}.jsonl"
+    fi
+  done
+  rm -rf "$tmpdir"
+  local count
+  count=$(ls "$dir"/*.jsonl 2>/dev/null | wc -l | tr -d ' ')
+  echo "[memory-arena] Downloaded to $dir ($count domains)"
+}
+
+# ── Main ──
+
+check_deps
+mkdir -p "$DATASETS_DIR"
+
+case "$BENCHMARK" in
+  ama-bench)      download_ama_bench ;;
+  longmemeval)    download_longmemeval ;;
+  amemgym)        download_amemgym ;;
+  locomo)         download_locomo ;;
+  memory-arena)   download_memory_arena ;;
+  all)
+    download_ama_bench
+    download_longmemeval
+    download_amemgym
+    download_locomo
+    download_memory_arena
+    ;;
+  *)
+    echo "Unknown benchmark: $BENCHMARK"
+    echo "Available: ama-bench, longmemeval, amemgym, locomo, memory-arena, all"
+    exit 1
+    ;;
+esac
+
+echo ""
+echo "Done. Datasets at: $DATASETS_DIR"