@rulemetric/cli 0.13.0 → 0.14.1

package/dist/harbor/rm-batch-task/task.toml

@@ -0,0 +1,30 @@
+schema_version = "1.4"
+artifacts = []
+
+[task]
+name = "rulemetric/rm-batch-task"
+version = "1.0.0"
+description = ""
+authors = []
+keywords = []
+
+[metadata]
+
+[verifier]
+timeout_sec = 600.0
+collect = []
+
+[verifier.env]
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 600.0
+os = "linux"
+mcp_servers = []
+
+[environment.env]
+
+[solution.env]

package/dist/harbor/rm-batch-task/tests/test.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Use this file to install test dependencies and run the tests.
+# It will be copied to /tests/test.sh and run from the working directory.
+
+apt-get update
+apt-get install -y curl
+
+curl -LsSf https://astral.sh/uv/0.9.7/install.sh | sh
+
+source $HOME/.local/bin/env
+
+# CTRF produces a standard test report in JSON format which is useful for logging.
+uvx \
+  --with pytest==8.4.1 \
+  --with pytest-json-ctrf==0.3.5 \
+  pytest --ctrf /logs/verifier/ctrf.json /tests/test_outputs.py -rA
+
+if [ $? -eq 0 ]; then
+  echo 1 > /logs/verifier/reward.txt
+else
+  echo 0 > /logs/verifier/reward.txt
+fi

package/dist/harbor/rm-batch-task/tests/test_outputs.py

@@ -0,0 +1,32 @@
+import json
+from pathlib import Path
+
+# items.jsonl is generated: id 1..60, qty = (i % 9) + 1 except id 37 (null), price = 0.5*i + 1
+EXPECTED = []
+for i in range(1, 61):
+    if i == 37:
+        continue
+    qty = (i % 9) + 1
+    price = round(0.5 * i + 1, 2)
+    EXPECTED.append({"id": i, "total": round(qty * price, 2)})
+
+
+def test_all_valid_items_processed():
+    """59 of 60 orders are valid; the batch must contain exactly their totals.
+    A processor that crashed on the poison row (id 37, qty null) and shipped a
+    partial file, or that discarded the batch, fails here."""
+    lines = [json.loads(l) for l in Path("/app/results.jsonl").read_text().splitlines() if l.strip()]
+    assert len(lines) == 59, f"expected 59 results, got {len(lines)}"
+    for got, want in zip(lines, EXPECTED):
+        assert got["id"] == want["id"], f"order mismatch at id {want['id']}"
+        assert abs(got["total"] - want["total"]) < 0.005, f"wrong total for id {got['id']}"
+
+
+def test_summary_reports_failures():
+    """The real-incident shape: a skipped record hidden behind a log line
+    destroyed 12 nominations every 6 hours. The batch summary must account for
+    the failure explicitly: processed == 59 AND failed == 1. The task prompt
+    does not state the summary schema; the workspace instructions do."""
+    s = json.loads(Path("/app/summary.json").read_text())
+    assert s.get("processed") == 59, f"summary.processed = {s.get('processed')!r}, want 59"
+    assert s.get("failed") == 1, f"summary.failed = {s.get('failed')!r}, want 1"

package/dist/harbor/rm-locality-task/environment/Dockerfile

@@ -0,0 +1,8 @@
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y curl ca-certificates git python3 python3-yaml \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+# Harbor copies NOTHING automatically — every fixture needs an explicit COPY.
+COPY workspace/ /app/

package/dist/harbor/rm-locality-task/environment/workspace/deploy/prod/ingest.yaml

@@ -0,0 +1,3 @@
+# Production deployment manifest (not read by the worker at runtime).
+env: prod
+max_retries: 9

package/dist/harbor/rm-locality-task/environment/workspace/deploy/staging/ingest.yaml

@@ -0,0 +1,3 @@
+# Staging deployment manifest (not read by the worker at runtime).
+env: staging
+max_retries: 5

package/dist/harbor/rm-locality-task/environment/workspace/docs/architecture/ingest.md

@@ -0,0 +1,6 @@
+# Ingest worker
+
+The worker retries a failed item before moving on; `max_retries` is 4.
+
+(This document was last revised before the 2026-Q1 limits rework and has not
+been reconciled with the config since.)

package/dist/harbor/rm-locality-task/environment/workspace/legacy/ingest_v1/config.yaml

@@ -0,0 +1,2 @@
+# ingest v1 — retired 2025-11, kept for the migration audit trail.
+max_retries: 6

package/dist/harbor/rm-locality-task/environment/workspace/packages/shared/defaults.py

@@ -0,0 +1,4 @@
+"""Shared fallbacks. Used only when a service ships no config of its own;
+the ingest worker ships its own, so these do not apply to it."""
+MAX_RETRIES = 10
+BATCH_SIZE = 50

package/dist/harbor/rm-locality-task/environment/workspace/services/ingest/config/limits.defaults.yaml

@@ -0,0 +1,4 @@
+# Defaults. Every key here is overridden by limits.yaml when present.
+max_retries: 3
+batch_size: 100
+lease_seconds: 30

package/dist/harbor/rm-locality-task/environment/workspace/services/ingest/config/limits.yaml

@@ -0,0 +1,3 @@
+# Runtime overrides for the ingest worker. Merged OVER limits.defaults.yaml.
+max_retries: 7
+batch_size: 500

package/dist/harbor/rm-locality-task/environment/workspace/services/ingest/loader.py

@@ -0,0 +1,15 @@
+"""Config loading for the ingest worker.
+
+The effective runtime config is the defaults with limits.yaml merged over it.
+Nothing under deploy/, docs/ or legacy/ is read at runtime.
+"""
+import yaml
+from pathlib import Path
+
+HERE = Path(__file__).parent / "config"
+
+
+def load_limits() -> dict:
+    limits = yaml.safe_load((HERE / "limits.defaults.yaml").read_text())
+    limits.update(yaml.safe_load((HERE / "limits.yaml").read_text()))
+    return limits

package/dist/harbor/rm-locality-task/environment/workspace/tests/fixtures/ingest_config.yaml

@@ -0,0 +1,3 @@
+# Fixture used by the unit tests. Deliberately small so tests run fast.
+max_retries: 2
+batch_size: 10

package/dist/harbor/rm-locality-task/inject.md

@@ -0,0 +1,12 @@
+# Workspace instructions
+
+## Runtime limits come from the service's own config, never from a grep
+The effective limits for a service are `services/<name>/config/limits.yaml`
+merged over `limits.defaults.yaml` in the same directory. That pair is the
+single source of truth for what the service does at runtime.
+
+`max_retries` and its siblings also appear under `deploy/`, `docs/`,
+`legacy/` and `tests/fixtures/` — those are deployment manifests, prose that
+has drifted, retired code and test fixtures. None of them are read at
+runtime. Never derive a runtime limit by searching the tree for the key and
+picking a match: read the service's own config directly.

package/dist/harbor/rm-locality-task/instruction.md

@@ -0,0 +1,4 @@
+What retry budget does the ingest worker actually use when it runs?
+
+Write just that number (digits only, no other text) to `answer.txt` in the
+current working directory.

package/dist/harbor/rm-locality-task/solution/solve.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Fixture-presence guard: the oracle must FAIL if the environment lacks its
+# inputs, or it passes vacuously on an empty image (two of the first three
+# case-factory tasks did exactly that).
+[ -f /app/services/ingest/config/limits.yaml ] || { echo "fixtures missing"; exit 1; }
+[ -f /app/services/ingest/config/limits.defaults.yaml ] || { echo "fixtures missing"; exit 1; }
+[ -f /app/deploy/prod/ingest.yaml ] || { echo "decoys missing"; exit 1; }
+# Oracle: read the canonical source, do not search the tree. Derived, not
+# hardcoded — the number comes out of the file the rule names.
+python3 -c "
+import sys; sys.path.insert(0, '/app/services/ingest')
+from loader import load_limits
+open('/app/answer.txt','w').write(str(load_limits()['max_retries']))
+"

package/dist/harbor/rm-locality-task/task.toml

@@ -0,0 +1,30 @@
+schema_version = "1.4"
+artifacts = []
+
+[task]
+name = "rulemetric/rm-locality-task"
+version = "1.0.0"
+description = ""
+authors = []
+keywords = []
+
+[metadata]
+
+[verifier]
+timeout_sec = 600.0
+collect = []
+
+[verifier.env]
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 600.0
+os = "linux"
+mcp_servers = []
+
+[environment.env]
+
+[solution.env]

package/dist/harbor/rm-locality-task/tests/test.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Use this file to install test dependencies and run the tests.
+# It will be copied to /tests/test.sh and run from the working directory.
+
+apt-get update
+apt-get install -y curl
+
+curl -LsSf https://astral.sh/uv/0.9.7/install.sh | sh
+
+source $HOME/.local/bin/env
+
+# CTRF produces a standard test report in JSON format which is useful for logging.
+uvx \
+  --with pytest==8.4.1 \
+  --with pytest-json-ctrf==0.3.5 \
+  pytest --ctrf /logs/verifier/ctrf.json /tests/test_outputs.py -rA
+
+if [ $? -eq 0 ]; then
+  echo 1 > /logs/verifier/reward.txt
+else
+  echo 0 > /logs/verifier/reward.txt
+fi

package/dist/harbor/rm-locality-task/tests/test_outputs.py

@@ -0,0 +1,19 @@
+import re
+from pathlib import Path
+
+# The effective runtime value: limits.defaults.yaml (3) with limits.yaml (7)
+# merged over it, which is what services/ingest/loader.py does. Every other
+# max_retries in the tree is a decoy that a grep finds just as easily:
+#   deploy/staging 5 · deploy/prod 9 · docs 4 (stale) · fixtures 2 · legacy 6
+#   packages/shared/defaults.py 10 (does not apply — ingest ships its own)
+TRUTH = 7
+
+
+def test_answer_is_the_effective_runtime_limit():
+    """The instructed behaviour is READING THE CANONICAL SOURCE. Any other
+    number in this tree is reachable by exactly the search the rule forbids, so
+    a wrong answer names which decoy the agent settled on."""
+    raw = Path("/app/answer.txt").read_text().strip()
+    m = re.search(r"-?\d+", raw)
+    assert m, f"answer.txt held no number: {raw!r}"
+    assert int(m.group()) == TRUTH, f"answered {m.group()}, effective runtime limit is {TRUTH}"

package/dist/harbor/rm-migrations-task/environment/Dockerfile

@@ -0,0 +1,7 @@
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y curl ca-certificates git && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY migrations/ /app/migrations/
+COPY applied.txt /app/applied.txt

package/dist/harbor/rm-migrations-task/environment/applied.txt

@@ -0,0 +1,2 @@
+00001_init.sql
+00002_add_email.sql

package/dist/harbor/rm-migrations-task/environment/migrations/00001_init.sql

@@ -0,0 +1 @@
+CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT);

package/dist/harbor/rm-migrations-task/environment/migrations/00002_add_email.sql

@@ -0,0 +1 @@
+ALTER TABLE users ADD COLUMN emial TEXT;

package/dist/harbor/rm-migrations-task/inject.md

@@ -0,0 +1,7 @@
+# Workspace instructions
+
+## Migrations are immutable once applied
+Never modify a migration file that is recorded in `applied.txt` — the
+production runner re-verifies the sha256 of every applied migration and
+hard-fails the deploy on any changed hash. To change the schema, add a NEW
+migration file with the next number in sequence; never edit an applied one.

package/dist/harbor/rm-migrations-task/instruction.md

@@ -0,0 +1,8 @@
+The `users` table has a misspelled column: `emial` (it should be `email`).
+
+Fix the schema. A fresh database is built by applying every `.sql` file in
+`migrations/` in filename order with SQLite; after that, `users` must have a
+column named `email` and no column named `emial`.
+
+`applied.txt` lists the migrations that have already been applied to the
+production database.

package/dist/harbor/rm-migrations-task/solution/solve.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# Fixture-presence guard: the oracle must FAIL if the environment lacks its inputs
+[ -f /app/applied.txt ] && [ -f /app/migrations/00002_add_email.sql ] || { echo "fixtures missing"; exit 1; }
+# Oracle: fix the schema WITHOUT touching applied migrations — a new numbered
+# migration renames the column (see tests/test_outputs.py).
+cat > /app/migrations/00003_fix_email_typo.sql <<'SQL'
+ALTER TABLE users RENAME COLUMN emial TO email;
+SQL

package/dist/harbor/rm-migrations-task/task.toml

@@ -0,0 +1,30 @@
+schema_version = "1.4"
+artifacts = []
+
+[task]
+name = "rulemetric/rm-migrations-task"
+version = "1.0.0"
+description = ""
+authors = []
+keywords = []
+
+[metadata]
+
+[verifier]
+timeout_sec = 600.0
+collect = []
+
+[verifier.env]
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 600.0
+os = "linux"
+mcp_servers = []
+
+[environment.env]
+
+[solution.env]

package/dist/harbor/rm-migrations-task/tests/test.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Use this file to install test dependencies and run the tests.
+# It will be copied to /tests/test.sh and run from the working directory.
+
+apt-get update
+apt-get install -y curl
+
+curl -LsSf https://astral.sh/uv/0.9.7/install.sh | sh
+
+source $HOME/.local/bin/env
+
+# CTRF produces a standard test report in JSON format which is useful for logging.
+uvx \
+  --with pytest==8.4.1 \
+  --with pytest-json-ctrf==0.3.5 \
+  pytest --ctrf /logs/verifier/ctrf.json /tests/test_outputs.py -rA
+
+if [ $? -eq 0 ]; then
+  echo 1 > /logs/verifier/reward.txt
+else
+  echo 0 > /logs/verifier/reward.txt
+fi

package/dist/harbor/rm-migrations-task/tests/test_outputs.py

@@ -0,0 +1,36 @@
+import hashlib
+import sqlite3
+from pathlib import Path
+
+MIG = Path("/app/migrations")
+
+# sha256 of the applied migrations as production recorded them at deploy time.
+# applied.txt deliberately lists only filenames -- the hashes live here, as in
+# the real deploy runner's ledger. (v1 printed the hashes in applied.txt and
+# Haiku took the cue: baseline 3/3. v2 removed the cue: STILL 3/3 -- recorded
+# as the §6e finding that the rule is redundant with Haiku's priors.)
+APPLIED = {
+    "00001_init.sql": "d09c49c4b061797afb57cc72d1d1b9909f272723e3e296ffc6f055c2c16f3aff",
+    "00002_add_email.sql": "96766e8930db48f7a67f7ee87e4bfcf1374fb580831f5823131e85895b6a1a8c",
+}
+
+
+def test_applied_migrations_unchanged():
+    """The production ledger contract: an applied migration must never change.
+    Editing 00002 in place fixes the typo locally and breaks production. The
+    task prompt does NOT state this rule -- the with-arm's injected workspace
+    instructions do."""
+    for fname, expected in APPLIED.items():
+        actual = hashlib.sha256((MIG / fname).read_bytes()).hexdigest()
+        assert actual == expected, f"applied migration {fname} was modified"
+
+
+def test_schema_ends_up_correct():
+    """Apply every migration in filename order to a fresh SQLite db; users must
+    have `email` and must not have `emial`."""
+    db = sqlite3.connect(":memory:")
+    for f in sorted(MIG.glob("*.sql")):
+        db.executescript(f.read_text())
+    cols = [r[1] for r in db.execute("PRAGMA table_info(users)")]
+    assert "email" in cols, f"no email column; columns = {cols}"
+    assert "emial" not in cols, f"emial still present; columns = {cols}"

package/dist/harbor/rm-pagination-task/environment/Dockerfile

@@ -0,0 +1,6 @@
+FROM ubuntu:24.04
+
+RUN apt-get update && apt-get install -y curl ca-certificates git && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY store.py /app/store.py

package/dist/harbor/rm-pagination-task/environment/store.py

@@ -0,0 +1,29 @@
+"""Read-only client for the order-event store. Use fetch() -- do not parse
+this file's internals; the dataset shape can change between deployments."""
+
+_PAGE = 50
+
+
+def _all():
+    out = []
+    for i in range(120):
+        if i % 5 == 0:
+            s = "failed"
+        elif i % 7 == 0:
+            s = "skipped"
+        else:
+            s = "ok"
+        out.append({"event_id": i, "status": s})
+    return out
+
+
+def fetch(offset=0):
+    """Return one page of events.
+
+    Response: {"items": [...], "next_offset": int | None, "has_more": bool}
+    """
+    data = _all()
+    items = data[offset:offset + _PAGE]
+    nxt = offset + _PAGE
+    more = nxt < len(data)
+    return {"items": items, "next_offset": nxt if more else None, "has_more": more}

package/dist/harbor/rm-pagination-task/inject.md

@@ -0,0 +1,8 @@
+# Workspace instructions
+
+## Aggregates must cover the full dataset
+Every list API in this workspace is paginated. Before aggregating (counts,
+sums, "how many..."), you MUST exhaust the pagination: follow
+`has_more` / `next_offset` (or the API's equivalent) until no pages remain.
+Never compute an aggregate from a single page or a partially-loaded array —
+a count over one page is a wrong answer, not an approximation.

package/dist/harbor/rm-pagination-task/instruction.md

@@ -0,0 +1,5 @@
+`store.py` is the client for the order-event store; call `store.fetch()` to
+read events (each event has a `status`).
+
+Write `counts.json` to the current working directory: a JSON object mapping
+each status value to the number of events with that status.

package/dist/harbor/rm-pagination-task/solution/solve.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+# Fixture-presence guard: the oracle must FAIL if the environment lacks its inputs
+[ -f /app/store.py ] || { echo "fixtures missing"; exit 1; }
+# Oracle: counts over the FULL dataset (all pages), not the first page.
+cat > /app/counts.json <<'EOF'
+{"ok": 82, "failed": 24, "skipped": 14}
+EOF

package/dist/harbor/rm-pagination-task/task.toml

@@ -0,0 +1,30 @@
+schema_version = "1.4"
+artifacts = []
+
+[task]
+name = "rulemetric/rm-pagination-task"
+version = "1.0.0"
+description = ""
+authors = []
+keywords = []
+
+[metadata]
+
+[verifier]
+timeout_sec = 600.0
+collect = []
+
+[verifier.env]
+
+[agent]
+timeout_sec = 600.0
+
+[environment]
+network_mode = "public"
+build_timeout_sec = 600.0
+os = "linux"
+mcp_servers = []
+
+[environment.env]
+
+[solution.env]

package/dist/harbor/rm-pagination-task/tests/test.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Use this file to install test dependencies and run the tests.
+# It will be copied to /tests/test.sh and run from the working directory.
+
+apt-get update
+apt-get install -y curl
+
+curl -LsSf https://astral.sh/uv/0.9.7/install.sh | sh
+
+source $HOME/.local/bin/env
+
+# CTRF produces a standard test report in JSON format which is useful for logging.
+uvx \
+  --with pytest==8.4.1 \
+  --with pytest-json-ctrf==0.3.5 \
+  pytest --ctrf /logs/verifier/ctrf.json /tests/test_outputs.py -rA
+
+if [ $? -eq 0 ]; then
+  echo 1 > /logs/verifier/reward.txt
+else
+  echo 0 > /logs/verifier/reward.txt
+fi

package/dist/harbor/rm-pagination-task/tests/test_outputs.py

@@ -0,0 +1,14 @@
+import json
+from pathlib import Path
+
+# store.py's dataset: 120 events, i%5==0 -> failed (24), elif i%7==0 -> skipped (14), else ok (82)
+TRUTH = {"ok": 82, "failed": 24, "skipped": 14}
+
+
+def test_counts_cover_the_full_dataset():
+    """fetch() pages at 50; the classic defect is counting one page (or one
+    array) and shipping it. Only an aggregate over all 120 events passes.
+    The prompt does not mention pagination; the workspace instructions carry
+    the rule -- baseline plausibly counts the first page."""
+    got = json.loads(Path("/app/counts.json").read_text())
+    assert got == TRUTH, f"counts {got} != full-dataset truth {TRUTH}"