claude-smart 0.2.44 → 0.2.46

package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/state.py

@@ -0,0 +1,323 @@
+"""Per-session JSONL buffer for interactions awaiting publish to reflexio.
+
+Each openClaw session gets one file at
+``~/.openclaw-smart/sessions/{session_id}.jsonl``. Lines are one of:
+
+- ``{"role": "User", ...}`` — a user turn (see InteractionData fields)
+- ``{"role": "Assistant", ...}`` — a finalized assistant turn
+- ``{"role": "Assistant_tool", ...}`` — a single tool invocation, attached
+  to the next assistant turn at ``agent_end`` time
+- ``{"published_up_to": N}`` — high-water mark so ``agent_end`` /
+  ``session_end`` don't re-publish rows already sent to reflexio
+
+The buffer exists for offline resilience: when reflexio is unreachable,
+``agent_end`` appends without publishing and the next successful hook drains.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import re
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+try:
+    import fcntl  # POSIX only — Windows hooks fall back to append-without-lock.
+except ImportError:  # pragma: no cover — non-POSIX platforms
+    fcntl = None  # type: ignore[assignment]
+
+_LOGGER = logging.getLogger(__name__)
+
+_ENV_STATE_DIR = "OPENCLAW_SMART_STATE_DIR"
+_DEFAULT_STATE_DIR = Path.home() / ".openclaw-smart" / "sessions"
+
+_TOOL_DATA_FIELD_MAX_LEN = 256
+
+_VALID_CITATION_KINDS = frozenset({"playbook", "profile"})
+
+
+def _truncate_tool_data_field(value: Any) -> Any:
+    """Truncate a single tool_data field value to ``_TOOL_DATA_FIELD_MAX_LEN``.
+
+    Only *top-level string* values are shortened. Nested containers
+    (dicts, lists) and non-string scalars pass through unchanged, even if
+    the container holds overlong strings — extractor prompts built from
+    this payload are bounded upstream by reflexio, and truncating a mid-
+    structure string risks producing invalid JSON when the caller later
+    serializes. The cap keeps long fields (``Edit.old_string`` /
+    ``new_string`` diffs, multi-line ``Bash`` scripts) from inflating the
+    extractor's input; short fields like file paths, URLs, and typical
+    commands stay intact.
+    """
+    if isinstance(value, str) and len(value) > _TOOL_DATA_FIELD_MAX_LEN:
+        return value[:_TOOL_DATA_FIELD_MAX_LEN]
+    return value
+
+
+def state_dir() -> Path:
+    """Root directory for session JSONL files. Honours ``OPENCLAW_SMART_STATE_DIR``."""
+    override = os.environ.get(_ENV_STATE_DIR)
+    return Path(override) if override else _DEFAULT_STATE_DIR
+
+
+# openClaw session ids are opaque tokens. The real shape we see in
+# production is ``agent:<id>:<run>`` (e.g. ``agent:main:main``) — colons
+# are part of the protocol, not a path separator on POSIX, so they're
+# safe in filesystem paths. Restrict to a conservative
+# alphanumeric + dot/underscore/hyphen/colon set so a crafted value
+# with `/` or `\` path separators or shell metacharacters can't write or
+# read outside ``state_dir()``. Any session id outside this charset is
+# treated as malformed and the resulting filesystem op becomes a no-op
+# upstream (see ``append`` / ``read_all``).
+#
+# We additionally reject ids that are pure-dot tokens (``.``, ``..``) or
+# contain a ``..`` substring even though the underlying regex would
+# accept them — those are not valid filenames and signal an escape
+# attempt.
+_SESSION_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
+
+
+def _safe_session_id(session_id: str) -> str | None:
+    """Return ``session_id`` iff it matches the safe-id charset, else ``None``.
+
+    Args:
+        session_id (str): Caller-supplied session key.
+
+    Returns:
+        str | None: The session id unchanged when safe, ``None`` otherwise.
+    """
+    if not isinstance(session_id, str):
+        return None
+    if not _SESSION_ID_RE.fullmatch(session_id):
+        return None
+    if ".." in session_id:
+        return None
+    if set(session_id) == {"."}:
+        return None
+    return session_id
+
+
+def session_path(session_id: str) -> Path | None:
+    """Return the JSONL path for a given session id, or ``None`` if unsafe."""
+    sid = _safe_session_id(session_id)
+    if sid is None:
+        return None
+    return state_dir() / f"{sid}.jsonl"
+
+
+def injected_path(session_id: str) -> Path | None:
+    """Return the per-session oc-cite registry path, or ``None`` if unsafe."""
+    sid = _safe_session_id(session_id)
+    if sid is None:
+        return None
+    return state_dir() / f"{sid}.injected.jsonl"
+
+
+def publish_lock_path(session_id: str) -> Path | None:
+    """Return the per-session publish lock path, or ``None`` if unsafe."""
+    sid = _safe_session_id(session_id)
+    if sid is None:
+        return None
+    return state_dir() / f"{sid}.publish.lock"
+
+
+def append_injected(session_id: str, entries: Iterable[dict[str, Any]]) -> None:
+    """Append citation-registry entries to the per-session injected-items file.
+
+    Each entry maps a short ``id`` (4-hex-char) back to the skill or
+    preference it came from so the ``agent_end`` hook can resolve ids cited
+    by openClaw via ``oc-cite`` into human-readable titles for the
+    dashboard. Silently no-ops when ``entries`` is empty.
+    """
+    records = list(entries)
+    if not records:
+        return
+    path = injected_path(session_id)
+    if path is None:
+        _LOGGER.warning(
+            "rejected unsafe session_id for append_injected: %r", session_id
+        )
+        return
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as fh:
+        if fcntl is not None:
+            try:
+                fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
+            except OSError as exc:
+                _LOGGER.debug("flock failed on %s: %s", path, exc)
+        for rec in records:
+            fh.write(json.dumps(rec, ensure_ascii=False) + "\n")
+
+
+def read_injected(session_id: str) -> dict[str, dict[str, Any]]:
+    """Return the per-session citation registry keyed by id.
+
+    Later entries win when the same id was injected multiple times
+    (identical content produces the same hash-derived id, so the extra
+    record only refreshes metadata).
+    """
+    path = injected_path(session_id)
+    if path is None or not path.exists():
+        return {}
+    registry: dict[str, dict[str, Any]] = {}
+    with path.open("r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError as exc:
+                _LOGGER.warning("Skipping malformed injected line in %s: %s", path, exc)
+                continue
+            item_id = entry.get("id")
+            if isinstance(item_id, str) and item_id:
+                registry[item_id] = entry
+    return registry
+
+
+def append(session_id: str, record: dict[str, Any]) -> None:
+    """Append one JSON record to the session buffer. Creates the dir if needed.
+
+    Holds an exclusive ``flock`` on the buffer file across the write so
+    concurrent hooks (e.g. parallel ``after_tool_call`` fires) cannot
+    interleave JSON lines when a payload exceeds the buffered-writer's
+    flush size.
+    """
+    path = session_path(session_id)
+    if path is None:
+        _LOGGER.warning("rejected unsafe session_id for append: %r", session_id)
+        return
+    path.parent.mkdir(parents=True, exist_ok=True)
+    line = json.dumps(record, ensure_ascii=False) + "\n"
+    with path.open("a", encoding="utf-8") as fh:
+        if fcntl is not None:
+            try:
+                fcntl.flock(fh.fileno(), fcntl.LOCK_EX)
+            except OSError as exc:
+                _LOGGER.debug("flock failed on %s: %s", path, exc)
+        fh.write(line)
+
+
+def read_all(session_id: str) -> list[dict[str, Any]]:
+    """Return every record in the buffer as a list of dicts. Missing file → []."""
+    path = session_path(session_id)
+    if path is None or not path.exists():
+        return []
+    records: list[dict[str, Any]] = []
+    with path.open("r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                records.append(json.loads(line))
+            except json.JSONDecodeError as exc:
+                _LOGGER.warning("Skipping malformed buffer line in %s: %s", path, exc)
+    return records
+
+
+def _to_wire_citations(cited_items: Any) -> list[dict[str, str]]:
+    """Map local ``cited_items`` to the wire ``Citation`` shape.
+
+    Local entries (from ``events.agent_end._resolve_cited_items``) carry
+    ``{id, kind, title, real_id}``; reflexio's ``InteractionData.citations``
+    wants ``{kind, real_id, tag, title}`` where ``tag`` is the rank id
+    (``s1-301``-style) we already keep under ``id``. Entries without a
+    ``real_id`` (unresolved injections) are dropped — the server can't
+    join them back to a stored row.
+    """
+    if not isinstance(cited_items, list):
+        return []
+    out: list[dict[str, str]] = []
+    for item in cited_items:
+        if not isinstance(item, dict):
+            continue
+        real_id = item.get("real_id")
+        kind = item.get("kind")
+        if not isinstance(real_id, str) or not real_id:
+            continue
+        if kind not in _VALID_CITATION_KINDS:
+            continue
+        tag = item.get("id")
+        title = item.get("title")
+        out.append(
+            {
+                "kind": kind,
+                "real_id": real_id,
+                "tag": tag if isinstance(tag, str) else "",
+                "title": title if isinstance(title, str) else "",
+            }
+        )
+    return out
+
+
+def unpublished_slice(
+    records: Iterable[dict[str, Any]],
+) -> tuple[int, list[dict[str, Any]]]:
+    """Split records into (last-published index, unpublished turn records).
+
+    Walks the records in order, tracking the most recent ``published_up_to``
+    marker and collecting turn records (anything with a ``role``) that come
+    after it. Tool records are folded into the closest following Assistant
+    turn's ``tools_used``.
+
+    Returns:
+        tuple[int, list[dict]]: ``(published_up_to, interactions)``. The
+            integer is the watermark after which all turns are unpublished;
+            the list is formatted for ``InteractionData`` construction.
+    """
+    published = 0
+    pending_tools: list[dict[str, Any]] = []
+    turns: list[dict[str, Any]] = []
+    for idx, rec in enumerate(records):
+        if "published_up_to" in rec:
+            marker = rec.get("published_up_to")
+            # Clamp to the number of records seen before this marker (``idx``).
+            # An over-range marker (e.g. from a tampered/corrupt buffer) would
+            # otherwise skip every subsequent turn via the ``idx < published``
+            # gate below, silently dropping valid unpublished records.
+            if isinstance(marker, int) and 0 <= marker <= idx:
+                published = marker
+            pending_tools = []
+            turns = []
+            continue
+        if idx < published:
+            continue
+        role = rec.get("role")
+        if role == "Assistant_tool":
+            raw_tool_input = rec.get("tool_input")
+            tool_input = raw_tool_input if isinstance(raw_tool_input, dict) else {}
+            tool_output = rec.get("tool_output") or ""
+            tool_entry: dict[str, Any] = {
+                "tool_name": rec.get("tool_name", ""),
+                "status": rec.get("status", "success"),
+            }
+            tool_data: dict[str, Any] = {}
+            if tool_input:
+                tool_data["input"] = {
+                    k: _truncate_tool_data_field(v) for k, v in tool_input.items()
+                }
+            if tool_output:
+                tool_data["output"] = _truncate_tool_data_field(tool_output)
+            if tool_data:
+                tool_entry["tool_data"] = tool_data
+            pending_tools.append(tool_entry)
+            continue
+        if role in {"User", "Assistant"}:
+            turn = {
+                k: v for k, v in rec.items() if k not in {"role", "ts", "cited_items"}
+            }
+            turn["role"] = role
+            if role == "Assistant":
+                citations = _to_wire_citations(rec.get("cited_items"))
+                if citations:
+                    turn["citations"] = citations
+                if pending_tools:
+                    turn["tools_used"] = pending_tools
+                    pending_tools = []
+            turns.append(turn)
+    return published, turns

package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/integration/test_e2e_session_loop.py

@@ -0,0 +1,190 @@
+"""End-to-end: drive all 6 hooks with synthetic payloads, assert buffer state.
+
+Exercises the hook dispatcher → event-handler → state buffer pipeline in
+order, the same sequence openClaw would trigger during a real session. The
+reflexio adapter is patched to a no-op so the test never depends on a live
+backend; failures inside ``publish`` would otherwise be tolerated silently
+(per ``reflexio_adapter.Adapter.publish``) but explicit patching keeps the
+test deterministic across CI runs.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+pytestmark = pytest.mark.e2e
+
+
+@pytest.fixture
+def isolated_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
+    """Force state.py to use a tmp directory so the test never touches ~/.openclaw-smart/."""
+    state_dir = tmp_path / "sessions"
+    monkeypatch.setenv("OPENCLAW_SMART_STATE_DIR", str(state_dir))
+    return state_dir
+
+
+@pytest.fixture
+def stub_adapter(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
+    """Replace every imported ``Adapter`` symbol with a MagicMock that returns success.
+
+    Each consumer module does ``from openclaw_smart.reflexio_adapter import Adapter``
+    at import time, so patching the source module alone leaves stale bindings in
+    consumers. We patch the consumer-side aliases as well so the test is
+    isolated regardless of import order.
+    """
+    from openclaw_smart import (  # noqa: F401 — force-import the consumers
+        context_inject,
+        publish,
+        reflexio_adapter,
+    )
+    from openclaw_smart.events import session_start
+
+    fake = MagicMock()
+    fake.publish.return_value = True
+    fake.apply_extraction_defaults.return_value = True
+    fake.apply_optimizer_defaults.return_value = True
+    fake.fetch_stall_state.return_value = None
+    fake.search_all.return_value = ([], [], [])
+
+    factory = lambda *a, **kw: fake  # noqa: E731 — single-line lambda is clearer here
+    monkeypatch.setattr(reflexio_adapter, "Adapter", factory)
+    monkeypatch.setattr(publish, "Adapter", factory)
+    monkeypatch.setattr(context_inject, "Adapter", factory)
+    monkeypatch.setattr(session_start, "Adapter", factory)
+    return fake
+
+
+def _drive(monkeypatch: pytest.MonkeyPatch, event: str, payload: dict) -> int:
+    from openclaw_smart import hook
+
+    monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(payload)))
+    return hook.main(["openclaw", event])
+
+
+def test_full_session_loop_records_all_roles(
+    monkeypatch: pytest.MonkeyPatch,
+    isolated_state: Path,
+    stub_adapter: MagicMock,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    """Walk the 6 hooks; assert User, Assistant_tool, Assistant records land."""
+    from openclaw_smart import state
+
+    session = "e2e-1"
+    project = "test-project"
+
+    # session-start: pushes defaults + emits a stall banner (or empty stdout).
+    assert (
+        _drive(
+            monkeypatch, "session-start", {"sessionKey": session, "agentId": project}
+        )
+        == 0
+    )
+    capsys.readouterr()  # drain
+
+    # before-prompt-build: appends a "User" record.
+    assert (
+        _drive(
+            monkeypatch,
+            "before-prompt-build",
+            {
+                "sessionKey": session,
+                "prompt": "implement OAuth flow",
+                "agentId": project,
+            },
+        )
+        == 0
+    )
+    capsys.readouterr()
+
+    # before-tool-call: observe-only stub; no record, no stdout.
+    assert (
+        _drive(
+            monkeypatch,
+            "before-tool-call",
+            {
+                "sessionKey": session,
+                "toolName": "Edit",
+                "params": {"file_path": "x.py"},
+                "agentId": project,
+            },
+        )
+        == 0
+    )
+    assert capsys.readouterr().out.strip() == ""
+
+    # after-tool-call: appends an "Assistant_tool" record (camelCase translated).
+    assert (
+        _drive(
+            monkeypatch,
+            "after-tool-call",
+            {
+                "sessionKey": session,
+                "toolName": "Edit",
+                "params": {"file_path": "x.py", "new_string": "..."},
+                "result": "wrote 10 lines",
+                "agentId": project,
+            },
+        )
+        == 0
+    )
+    capsys.readouterr()
+
+    # agent-end: extracts the assistant text and appends an "Assistant" record.
+    assert (
+        _drive(
+            monkeypatch,
+            "agent-end",
+            {
+                "sessionKey": session,
+                "agentId": project,
+                "messages": [
+                    {"role": "user", "content": "implement OAuth flow"},
+                    {"role": "assistant", "content": "Done."},
+                ],
+            },
+        )
+        == 0
+    )
+    capsys.readouterr()
+
+    # session-end: force-publish; should not crash.
+    assert (
+        _drive(monkeypatch, "session-end", {"sessionKey": session, "agentId": project})
+        == 0
+    )
+    capsys.readouterr()
+
+    records = state.read_all(session)
+    roles = [r.get("role") for r in records if "role" in r]
+    assert "User" in roles
+    assert "Assistant_tool" in roles
+    assert "Assistant" in roles
+
+    # session_end publishes; we asserted publish was called via the stub.
+    assert stub_adapter.publish.called
+
+    # The agent_end / session_end path writes a published_up_to watermark
+    # once a publish succeeds. The stub returns True, so we should see it.
+    assert any("published_up_to" in r for r in records), (
+        "expected published_up_to watermark after session_end"
+    )
+
+
+def test_unknown_event_is_silent(
+    monkeypatch: pytest.MonkeyPatch,
+    isolated_state: Path,
+    capsys: pytest.CaptureFixture[str],
+) -> None:
+    """An unrecognized hook name short-circuits with empty stdout, never crashes."""
+    monkeypatch.setattr("sys.stdin", io.StringIO("{}"))
+    from openclaw_smart import hook
+
+    rc = hook.main(["openclaw", "this-does-not-exist"])
+    assert rc == 0
+    assert capsys.readouterr().out.strip() == ""

package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/integration/test_publish_to_local_reflexio_integration.py

@@ -0,0 +1,112 @@
+"""Integration: publish buffered turns to a local SQLite-backed reflexio.
+
+Requires a reflexio backend reachable at ``REFLEXIO_URL`` (default
+``http://localhost:8061/``). Skipped automatically when the backend is
+unreachable so the suite is portable across machines.
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.integration
+
+
+def _reflexio_url() -> str:
+    return os.environ.get("REFLEXIO_URL", "http://localhost:8061/")
+
+
+def _backend_alive(url: str) -> bool:
+    """True iff ``GET <url>health`` returns 200 within a short window."""
+    import urllib.error
+    import urllib.request
+
+    health = url.rstrip("/") + "/health"
+    try:
+        with urllib.request.urlopen(health, timeout=1.5) as resp:  # nosec — local only
+            return resp.status == 200
+    except (urllib.error.URLError, TimeoutError, OSError):
+        return False
+
+
+@pytest.fixture
+def require_backend() -> None:
+    if not _backend_alive(_reflexio_url()):
+        pytest.skip(f"reflexio backend not reachable at {_reflexio_url()}")
+
+
+@pytest.fixture
+def isolated_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
+    """Force state.py to write under tmp_path so the test doesn't leak fixtures."""
+    state_dir = tmp_path / "sessions"
+    monkeypatch.setenv("OPENCLAW_SMART_STATE_DIR", str(state_dir))
+    return state_dir
+
+
+def test_publish_lands_in_reflexio_storage(
+    require_backend: None,
+    monkeypatch: pytest.MonkeyPatch,
+    isolated_state: Path,
+    worker_id: str = "test",
+) -> None:
+    """Append two turns to the JSONL buffer → publish → assert success status + count."""
+    from openclaw_smart import state
+    from openclaw_smart.publish import publish_unpublished
+
+    session_id = f"oc-int-{worker_id}"
+    project_id = f"oc-test-proj-{worker_id}"
+
+    state.append(session_id, {"role": "User", "content": "implement auth"})
+    state.append(
+        session_id,
+        {"role": "Assistant", "content": "Wrote login/logout endpoints."},
+    )
+
+    status, count = publish_unpublished(
+        session_id=session_id,
+        project_id=project_id,
+        force_extraction=False,
+        skip_aggregation=False,
+    )
+
+    assert status == "ok", f"publish returned {status!r} — check backend logs"
+    assert count == 2
+
+    # The watermark is now stamped — a second publish with no new turns is a no-op.
+    status2, count2 = publish_unpublished(
+        session_id=session_id,
+        project_id=project_id,
+        force_extraction=False,
+        skip_aggregation=False,
+    )
+    assert (status2, count2) == ("nothing", 0)
+
+
+def test_publish_failed_when_backend_url_invalid(
+    monkeypatch: pytest.MonkeyPatch, isolated_state: Path
+) -> None:
+    """Pointing the adapter at a dead port surfaces 'failed' without stamping."""
+    from openclaw_smart import state
+    from openclaw_smart.publish import publish_unpublished
+    from openclaw_smart.reflexio_adapter import Adapter
+
+    # Pick a port that's almost certainly closed.
+    bogus = Adapter(url="http://127.0.0.1:9/")
+    session_id = "oc-int-bogus"
+    state.append(session_id, {"role": "User", "content": "hi"})
+
+    status, count = publish_unpublished(
+        session_id=session_id,
+        project_id="proj-bogus",
+        force_extraction=False,
+        skip_aggregation=False,
+        adapter=bogus,
+    )
+    assert status == "failed"
+    assert count == 1
+    # No watermark stamped — a retry would re-publish the same record.
+    records = state.read_all(session_id)
+    assert not any("published_up_to" in r for r in records)