@salesforce/afv-skills 1.66.0 → 1.68.0

package/skills/agentforce-offline-experimentation-run/scripts/btlib/sfap.py

@@ -0,0 +1,238 @@
+"""SFAP /chat/generations client and the production request shape.
+
+The token is read from its 0600 file on every call: it never enters argv, the environment
+of child processes, or any artifact, and a token refreshed in place is picked up without a
+restart. On a 401, on_unauthorized(token_used) may refresh it (tokens.TokenRefresher); a True
+return sends the request once more. Failures come back as status 'infra_error' after
+retries, never as an answer.
+
+A timeout is retried once only: each one costs the whole timeout, and a model that runs away on a
+request runs away again. The provider has its own limit (EinsteinBYOM: ~180 s, then a 502 whose
+body says "Timeout"), recorded as provider_timeout; the default --timeout is above it, so that the
+provider, not the client, cuts a slow answer off.
+
+Production request fields (from captured gateway payloads) reproduced here:
+  tools                 [{"function": {name, description, parameters}}]  (no "type")
+  tool_config           {"mode": "auto", "allowed_tools": [all tools], "parallel_calls": true}
+  generation_settings   {"num_generations": 1, penalties 0, "stop_sequences": [],
+                         "parameters": {"seed": n}}  — no max_tokens, no temperature
+  system_prompt_strategy "use_model_parameter";  enable_pii_masking false
+Not reproduced: safety/bias scoring flags, trust settings, localization, turn ids, tags.
+"""
+
+import copy
+import hashlib
+import json
+import socket
+import time
+import urllib.error
+import urllib.request
+
+from .redact import redact, register_secret
+from .units import parse_arguments
+
+RETRY_STATUS = {408, 425, 429, 500, 502, 503, 504}
+# No answer in time: the client's timeout, or the provider's (a 502/504 whose body says so).
+TIMEOUT_KINDS = ("timeout", "provider_timeout")
+DEFAULT_TIMEOUT = 200
+
+
+def sanitize_tools_for_sfap(tools):
+    """Drop additionalProperties and empty `required` (SFAP has rejected both with a 400)."""
+    sanitized = []
+    for tool in tools:
+        tool = copy.deepcopy(tool)
+        params = (tool.get("function") or {}).get("parameters") or {}
+        params.pop("additionalProperties", None)
+        if "required" in params and not params["required"]:
+            del params["required"]
+        sanitized.append(tool)
+    return sanitized
+
+
+def to_sfap_tools(tools):
+    """Step tool definitions (gen_ai.request.tools) in the production request shape."""
+    return [{"function": t.get("function") or {}} for t in sanitize_tools_for_sfap(tools)]
+
+
+def build_payload(model, messages, tools=None, seed=None, parameters=None):
+    payload = {"model": model, "messages": messages}
+    if tools:
+        payload["tools"] = to_sfap_tools(tools)
+        payload["tool_config"] = {
+            "mode": "auto",
+            "allowed_tools": [{"name": (t.get("function") or {}).get("name"), "type": "function"}
+                              for t in tools],
+            "parallel_calls": True,
+        }
+    gen_params = {}
+    if seed is not None:
+        gen_params["seed"] = seed
+    gen_params.update(parameters or {})
+    payload["generation_settings"] = {
+        "num_generations": 1, "frequency_penalty": 0.0, "presence_penalty": 0.0,
+        "stop_sequences": [], "parameters": gen_params,
+    }
+    payload["system_prompt_strategy"] = "use_model_parameter"
+    payload["enable_pii_masking"] = False
+    return payload
+
+
+def request_sha(messages, tools):
+    """Identity of a request's content (messages + tools), independent of model and seed."""
+    blob = json.dumps({"messages": messages, "tools": tools}, sort_keys=True,
+                      separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(blob.encode("utf-8")).hexdigest()
+
+
+def _usage(body):
+    gd = body.get("generation_details") or {}
+    for holder in (gd.get("parameters"), body.get("parameters"), body):
+        if isinstance(holder, dict) and isinstance(holder.get("usage"), dict):
+            return holder["usage"]
+    return None
+
+
+def parse_generation(body):
+    """{text, tool_calls, finish_reason, usage} from a 200 body, or None if it has no generation."""
+    gens = ((body or {}).get("generation_details") or {}).get("generations") or []
+    if not gens:
+        return None
+    g = gens[0]
+    calls = []
+    for inv in g.get("tool_invocations") or []:
+        fn = inv.get("function") or {}
+        args, raw = parse_arguments(fn.get("arguments"))
+        call = {"name": fn.get("name"), "args": args}
+        if raw is not None:
+            call["args_raw"] = raw
+        calls.append(call)
+    params = g.get("parameters") if isinstance(g.get("parameters"), dict) else {}
+    return {"text": g.get("content") or "", "tool_calls": calls,
+            "finish_reason": params.get("finish_reason") or g.get("finish_reason"),
+            "usage": _usage(body)}
+
+
+class SfapClient:
+    def __init__(self, base_url, token_file, core_tenant_id, timeout=DEFAULT_TIMEOUT, max_retries=3,
+                 backoff=2.0, rate_limit_backoff=10.0, timeout_retries=1, feature_id="AgentBacktesting",
+                 app_context="EinsteinGPT", opener=None, sleep=time.sleep, on_unauthorized=None):
+        missing = [n for n, v in (("base_url", base_url), ("token_file", token_file),
+                                  ("core_tenant_id", core_tenant_id)) if not v]
+        if missing:
+            raise ValueError("SFAP config missing: " + ", ".join(missing)
+                             + " (run scripts/resolve-token.sh and pass --token-info)")
+        self.url = base_url.rstrip("/") + "/chat/generations"
+        self.token_file = token_file
+        self.core_tenant_id = core_tenant_id
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self.backoff = backoff
+        self.rate_limit_backoff = rate_limit_backoff
+        self.timeout_retries = timeout_retries
+        self.feature_id = feature_id
+        self.app_context = app_context
+        self._open = (opener.open if opener else urllib.request.urlopen)
+        self._sleep = sleep
+        self._on_unauthorized = on_unauthorized
+
+    def _token(self):
+        with open(self.token_file) as f:
+            token = f.read().strip()
+        if not token:
+            raise RuntimeError("SFAP token file is empty")
+        register_secret(token)
+        return token
+
+    def _delay(self, attempt, retry_after=None, http_status=None):
+        if retry_after:
+            try:
+                return min(60.0, float(retry_after))
+            except ValueError:
+                pass
+        if http_status == 429:  # a quota, usually per minute: a few seconds' wait meets it again
+            return min(60.0, self.rate_limit_backoff * (2 ** (attempt - 1)))
+        return min(30.0, self.backoff * (2 ** (attempt - 1)))
+
+    def post(self, payload):
+        """POST once per attempt. Returns {status, http_status, body, error, error_kind, attempts, latency_ms}."""
+        data = json.dumps(payload).encode("utf-8")
+        attempts, timeouts, refreshed, t0 = 0, 0, False, time.monotonic()
+        while True:
+            attempts += 1
+            err_kind, err, http_status, retry_after = None, None, None, None
+            try:
+                token = self._token()
+            except (OSError, RuntimeError) as e:
+                return self._fail("token", f"cannot read the SFAP token file: {type(e).__name__}",
+                                  None, attempts, t0)
+            try:
+                req = urllib.request.Request(self.url, data=data, method="POST", headers={
+                    "Authorization": f"Bearer {token}",
+                    "Content-Type": "application/json",
+                    "x-sfdc-app-context": self.app_context,
+                    "x-client-feature-id": self.feature_id,
+                    "x-sfdc-core-tenant-id": self.core_tenant_id,
+                })
+                with self._open(req, timeout=self.timeout) as resp:
+                    raw = resp.read().decode("utf-8", "replace")
+                    http_status = resp.status
+                try:
+                    body = json.loads(raw)
+                except json.JSONDecodeError:
+                    return self._fail("bad_json", f"HTTP {http_status}: response is not JSON: {redact(raw)[:200]}",
+                                      http_status, attempts, t0)
+                return {"status": "ok", "http_status": http_status, "body": body, "error": None,
+                        "error_kind": None, "attempts": attempts,
+                        "latency_ms": int((time.monotonic() - t0) * 1000)}
+            except urllib.error.HTTPError as e:
+                http_status = e.code
+                retry_after = e.headers.get("Retry-After") if e.headers else None
+                try:
+                    body = e.read().decode("utf-8", "replace")
+                except Exception:
+                    body = ""
+                finally:
+                    e.close()
+                err_kind, err = f"http_{e.code}", f"HTTP {e.code}: {redact(body)[:300]}"  # redact before cutting
+                if e.code in (502, 504) and "timeout" in body.lower():
+                    err_kind = "provider_timeout"
+                if e.code == 401 and self._on_unauthorized and not refreshed:
+                    refreshed = True
+                    if self._on_unauthorized(token):
+                        continue
+                if e.code not in RETRY_STATUS:
+                    return self._fail(err_kind, err, http_status, attempts, t0)
+            except (socket.timeout, TimeoutError):
+                err_kind, err = "timeout", f"timeout after {self.timeout}s"
+            except urllib.error.URLError as e:
+                if isinstance(e.reason, (socket.timeout, TimeoutError)):
+                    err_kind, err = "timeout", f"timeout after {self.timeout}s"
+                else:
+                    err_kind, err = "network", f"network error: {e.reason}"
+            except (ConnectionError, OSError) as e:
+                err_kind, err = "network", f"network error: {e}"
+            if err_kind in TIMEOUT_KINDS:
+                timeouts += 1
+            if attempts > self.max_retries or timeouts > self.timeout_retries:
+                return self._fail(err_kind, err, http_status, attempts, t0)
+            self._sleep(self._delay(attempts, retry_after, http_status))
+
+    @staticmethod
+    def _fail(kind, err, http_status, attempts, t0):
+        return {"status": "infra_error", "http_status": http_status, "body": None,
+                "error": redact(err), "error_kind": kind, "attempts": attempts,
+                "latency_ms": int((time.monotonic() - t0) * 1000)}
+
+    def generate(self, payload):
+        """post() plus parsing; a 200 without a generation is an infra error too."""
+        res = self.post(payload)
+        if res["status"] != "ok":
+            res["response"] = None
+            return res
+        parsed = parse_generation(res["body"])
+        if parsed is None:
+            res.update(status="infra_error", error_kind="empty_response",
+                       error="HTTP 200 without generation_details.generations")
+        res["response"] = parsed
+        return res

package/skills/agentforce-offline-experimentation-run/scripts/btlib/splits.py

@@ -0,0 +1,136 @@
+"""Session splits (--val-fraction, --split) and excluded sessions (--exclude-sessions).
+
+A unit's split depends on its session alone: sha256 of the salt and the session ID, read as a
+number in [0, 1), gives "val" below the val fraction and "test" otherwise. So every step of a
+session lands in one split, a session keeps its split when other sessions are added or
+removed, and another salt gives an independent split.
+"""
+
+import csv
+import hashlib
+import io
+import json
+import re
+from collections import Counter
+
+from . import stdm
+
+DEFAULT_SALT = "bt-split/1"
+SPLITS = ("test", "val")
+
+
+class SplitError(ValueError):
+    pass
+
+
+def session_split(session_id, val_fraction, salt=DEFAULT_SALT):
+    h = hashlib.sha256(f"{salt}\0{session_id}".encode("utf-8")).hexdigest()
+    return "val" if int(h[:13], 16) / 16 ** 13 < val_fraction else "test"
+
+
+def label(units, val_fraction, salt=DEFAULT_SALT):
+    """Set each unit's split; returns {split: [sessions, units]}. Units that already carry
+    another split (an earlier run's labels) are refused: relabelling would mix the sets."""
+    new = [session_split(u["session_id"], val_fraction, salt) for u in units]
+    clash = sum(1 for u, s in zip(units, new) if u.get("split") not in (None, s))
+    if clash:
+        raise SplitError(f"{clash} units already carry another split (labelled with another salt or "
+                         "fraction); use those, or split anew from --details")
+    for u, s in zip(units, new):
+        u["split"] = s
+    sessions = {u["session_id"]: u["split"] for u in units}
+    counts = Counter(sessions.values())
+    return {s: [counts[s], new.count(s)] for s in SPLITS}
+
+
+def read_session_ids(path):
+    """Session IDs from a file that names sessions:
+    - one ID per line (blank lines and '#' comments skipped);
+    - JSONL with a session_id on each line: units.jsonl, results.jsonl;
+    - JSON: a list of objects with a session_id (bt_fetch's sessions.json or details.json, the
+      Apex class's output, also as an `sf apex run --json` file), or the old replay's
+      replay_results.json. Not a list of strings: step and generation IDs look like session IDs;
+    - CSV with a session_id column: replay_results_sfap.csv, the old replay's too.
+    Any other file is refused, so that it never passes as a list of IDs that excludes nothing. So is
+    an ID that is not letters, digits, '_', '.' and '-': matched exactly, " s1" would exclude nothing.
+    """
+    with open(path, encoding="utf-8-sig") as f:  # -sig: a CSV saved by Excel starts with a BOM
+        text = f.read()
+    lines = [(n, s.strip()) for n, s in enumerate(text.split("\n"), 1)]
+    lines = [(n, s) for n, s in lines if s and not s.startswith("#")]
+    if not lines:
+        return set()
+    first = lines[0][1]
+    if first[0] in "[{":
+        try:
+            doc = json.loads(text)
+        except ValueError as e:
+            if first[0] == "[":
+                raise SplitError(f"not valid JSON ({e})") from None
+            return _jsonl_ids(lines)
+        return _document_ids(doc)
+    if "," in first or first == "session_id":
+        return _csv_ids(text)
+    for n, s in lines:
+        if not _SESSION_ID.match(s):
+            raise SplitError(f"line {n} is not a session ID, and the file is not JSON, JSONL or CSV")
+    return {s for _, s in lines}
+
+
+_SESSION_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*\Z")
+
+
+def _jsonl_ids(lines):
+    ids = set()
+    for n, s in lines:
+        try:
+            row = json.loads(s)
+        except ValueError:
+            raise SplitError(f"line {n}: not JSON") from None
+        ids.add(_session_id(row.get("session_id") if isinstance(row, dict) else None, f"line {n}"))
+    return ids
+
+
+def _document_ids(doc):
+    if isinstance(doc, list):
+        items = doc
+    elif not isinstance(doc, dict):
+        raise SplitError("JSON that is neither a list nor an object")
+    else:
+        try:
+            found = stdm.parse_sf_output(doc, "sf apex run output")
+        except stdm.StdmError as e:
+            raise SplitError(str(e)) from None
+        if found is not None:  # an `sf apex run --json` file
+            if not found:
+                raise SplitError("an sf apex run output without an STDM_RESULT line")
+            items = [x for payload in found.values() for x in (payload if isinstance(payload, list) else [payload])]
+        elif isinstance(doc.get("rows"), list):  # the old replay's replay_results.json
+            items = [{"session_id": r.get("AiAgentSessionId")} if isinstance(r, dict) else r for r in doc["rows"]]
+        else:
+            items = [doc]
+    ids = set()
+    for i, item in enumerate(items, 1):
+        if isinstance(item, str):
+            raise SplitError(f"item {i} is a string, not an object with a session_id (step IDs look the "
+                             "same); write a list of session IDs one per line")
+        ids.add(_session_id(item.get("session_id") if isinstance(item, dict) else None, f"item {i}"))
+    return ids
+
+
+def _csv_ids(text):
+    rows = csv.DictReader(io.StringIO(text))
+    if "session_id" not in (rows.fieldnames or ()):
+        raise SplitError("a CSV without a session_id column")
+    ids = set()
+    for i, row in enumerate(rows, 1):
+        ids.add(_session_id((row.get("session_id") or "").strip(), f"CSV row {i}"))
+    return ids
+
+
+def _session_id(value, where):
+    if not isinstance(value, str) or not value:
+        raise SplitError(f"{where}: no session_id")
+    if not _SESSION_ID.match(value):
+        raise SplitError(f"{where}: its session_id is not a session ID")
+    return value

package/skills/agentforce-offline-experimentation-run/scripts/btlib/stdm.py

@@ -0,0 +1,98 @@
+"""Read STDM query results.
+
+The skill's fetch phases run Apex with `sf apex run --json` and print each result as one
+debug line, `...|DEBUG|STDM_RESULT:<json>` (or `STDM_RESULT_<suffix>:` when one script
+prints several). The source echo of the script also contains `STDM_RESULT:`, which is why
+only the `|DEBUG|` form is matched. Inside a debug message the log writes '|' as '&#124;';
+that is undone before parsing (the old skill parser kept the entity, e.g. in prompts).
+bt_fetch.py's scripts print several pages per query, `STDM_RESULT_<query>_<page>:`.
+"""
+
+import json
+import re
+
+from .redact import redact
+
+_MARKER = re.compile(r"\|DEBUG\|(STDM_RESULT(?:_[A-Za-z0-9_]+)?):")
+
+
+class StdmError(Exception):
+    pass
+
+
+def parse_apex_logs(logs):
+    """{marker: parsed JSON} for every STDM_RESULT debug line in an Apex log."""
+    found = {}
+    for m in _MARKER.finditer(logs):
+        line_end = logs.find("\n", m.end())
+        raw = logs[m.end():] if line_end < 0 else logs[m.end():line_end]
+        # The debug log escapes its own field separator inside messages: '|' -> '&#124;'.
+        raw = raw.strip().replace("&#124;", "|")
+        try:
+            found[m.group(1)] = json.loads(raw)
+        except json.JSONDecodeError as e:
+            raise StdmError(
+                f"{m.group(1)}: debug line is not valid JSON ({e.msg} at char {e.pos} of {len(raw)}). "
+                "The debug log was probably truncated (Apex caps a log at 20 MB); "
+                "fetch fewer sessions or steps per batch.") from None
+    return found
+
+
+def parse_sf_output(data, where):
+    """{marker: payload} from the parsed output of `sf apex run --json`: {} when its log has no
+    STDM_RESULT line, None when data is not that output. A run that failed to compile or threw,
+    and an error of the sf CLI itself (no org, expired session), raise StdmError."""
+    if not isinstance(data, dict):
+        return None
+    res = data.get("result")
+    if isinstance(res, dict) and "logs" in res:
+        if res.get("success") is False or res.get("compiled") is False:
+            msg = str(res.get("compileProblem") or res.get("exceptionMessage") or "unknown error")
+            if res.get("compiled") is False and isinstance(res.get("line"), int) and res["line"] > 0:
+                msg += f" (line {res['line']}, column {res.get('column')})"
+            raise StdmError(f"{where}: Apex run failed: {redact(msg)[:300]}")
+        return parse_apex_logs(res.get("logs") or "")
+    if data.get("status") not in (None, 0) and isinstance(data.get("message"), str):
+        name = data.get("name")
+        raise StdmError(f"{where}: sf failed" + (f" ({name})" if isinstance(name, str) else "")
+                        + f": {redact(data['message'])[:300]}")
+    return None
+
+
+def load_result_file(path):
+    """{marker: payload} from an `sf apex run --json` output file.
+
+    A file holding plain JSON (a list or an object without the sf CLI envelope) is
+    returned as {None: payload}, so already-extracted files work too.
+    """
+    with open(path, encoding="utf-8") as f:
+        data = json.load(f)
+    found = parse_sf_output(data, path)
+    if found is None:
+        return {None: data}
+    if not found:
+        raise StdmError(f"{path}: no |DEBUG|STDM_RESULT line in the Apex log")
+    return found
+
+
+def load_conversations(paths):
+    """Merge ConversationData lists from one or more details files.
+
+    Accepts getMultipleConversationDetails output (a list), getConversationDetails output
+    (one object), or several markers per file. The first copy of a session wins.
+    Returns (conversations, duplicate_count).
+    """
+    convos, seen, dupes = [], set(), 0
+    for path in paths:
+        for payload in load_result_file(path).values():
+            items = payload if isinstance(payload, list) else [payload]
+            for c in items:
+                if not isinstance(c, dict) or "turns" not in c or not isinstance(c.get("session_id"), str) \
+                        or not c["session_id"].strip():
+                    raise StdmError(f"{path}: expected ConversationData objects (a session_id, turns)")
+                if c["session_id"] in seen:
+                    dupes += 1
+                    continue
+                seen.add(c["session_id"])
+                convos.append(c)
+    return convos, dupes

package/skills/agentforce-offline-experimentation-run/scripts/btlib/tokens.py

@@ -0,0 +1,173 @@
+"""SFAP token refresh (--token-org): mint a new token when the run's token expires.
+
+An SFAP token lasts about two hours, so a long replay outlives it. On an HTTP 401 the client
+calls the refresher with the token that was refused. The refresher runs
+
+    bash scripts/resolve-token.sh --org <alias>
+
+which writes a new token to a new 0600 temp file and prints that file's path. The new token
+replaces the old one in the run's token file (atomically, mode 0600), the script's file is
+deleted, and the client sends the request again, once.
+
+- One mint per expiry: requests refused with the same token wait for the first refresh, then
+  retry with its token. A lock file next to the token file (<token file>.lock, empty) extends
+  this to other processes reading the same file.
+- No loops: a token minted here and refused within FRESH_SECONDS is not an expired token (wrong
+  org, missing scope), and a mint that failed is not repeated for the same token. Those requests
+  stay http_401 infra errors.
+- The new token must be for the run's SFAP endpoint and core tenant: another org's is not used.
+- The token never enters argv or output: the script gets the org alias only, and its stdout (a
+  file path and IDs) is parsed, never printed.
+"""
+
+import contextlib
+import fcntl
+import hashlib
+import json
+import os
+import signal
+import subprocess
+import tempfile
+import threading
+import time
+
+from .redact import redact, register_secret
+
+RESOLVE_TOKEN = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "resolve-token.sh")
+FRESH_SECONDS = 300
+SCRIPT_FILE_PREFIX = "offexp-jwt."  # resolve-token.sh's temp files: the only files deleted here
+
+
+def read_token(path):
+    try:
+        with open(path) as f:
+            return f.read().strip()
+    except OSError:
+        return ""
+
+
+def _digest(token):
+    return hashlib.sha256((token or "").encode("utf-8")).hexdigest()
+
+
+def _last_json(out):
+    """The script's JSON line, {} if there is none."""
+    for line in reversed((out or "").strip().splitlines()):
+        try:
+            obj = json.loads(line)
+        except ValueError:
+            continue
+        return obj if isinstance(obj, dict) else {}
+    return {}
+
+
+def _print(msg):
+    print(msg, flush=True)
+
+
+class TokenRefresher:
+    """SfapClient's on_unauthorized(token_used): True when the token file now holds a token worth a retry."""
+
+    def __init__(self, token_file, org, base_url, core_tenant_id, script=None, timeout=180, log=_print,
+                 clock=time.monotonic):
+        self.token_file = os.path.abspath(token_file)
+        self.org = org
+        self.base_url = (base_url or "").rstrip("/")
+        self.core_tenant_id = core_tenant_id
+        self.script = script or RESOLVE_TOKEN
+        self.timeout = timeout
+        self.log = log
+        self._clock = clock
+        self.refreshes = self.failures = 0
+        self._lock = threading.Lock()
+        self._minted = {}  # digest of a token minted here -> when
+        self._given_up = set()  # digests of tokens not to refresh again
+
+    def __call__(self, token_used):
+        used = _digest(token_used)
+        with self._lock:
+            try:
+                with self._file_lock():
+                    if read_token(self.token_file) not in ("", token_used):
+                        return True  # refreshed meanwhile, by another thread or process
+                    if used in self._given_up:
+                        return False
+                    minted = self._minted.get(used)
+                    if minted is not None and self._clock() - minted < FRESH_SECONDS:
+                        why = "a token minted moments ago was refused too, so expiry is not the problem"
+                    else:
+                        why = self._mint()
+            except OSError as e:
+                why = f"cannot update the token file ({type(e).__name__})"
+            if why is None:
+                self.refreshes += 1
+                self.log(f"SFAP token refreshed (resolve-token.sh --org {self.org})")
+                return True
+            self._given_up.add(used)
+            self.failures += 1
+            self.log(f"WARNING: SFAP token not refreshed: {redact(why)}. The calls refused with HTTP 401 are "
+                     "infra errors; with --cache-dir a rerun sends only those")
+            return False
+
+    def _mint(self):
+        """Run the script and install its token: None on success, else why not."""
+        try:
+            proc = subprocess.Popen(["bash", self.script, "--org", self.org], stdin=subprocess.DEVNULL,
+                                    stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
+                                    universal_newlines=True, start_new_session=True)
+        except OSError as e:
+            return f"cannot run resolve-token.sh ({type(e).__name__})"
+        try:
+            out, _ = proc.communicate(timeout=self.timeout)
+        except subprocess.TimeoutExpired:
+            with contextlib.suppress(OSError):
+                os.killpg(proc.pid, signal.SIGKILL)  # the script's sf and curl children too
+            proc.communicate()
+            return f"resolve-token.sh --org {self.org} took more than {self.timeout:g}s"
+        info = _last_json(out)
+        new_file = info.get("tokenFile")
+        try:
+            if info.get("error"):
+                return f"resolve-token.sh --org {self.org}: {info['error']} ({info.get('hint') or 'no hint'})"
+            if proc.returncode or not new_file:
+                return f"resolve-token.sh --org {self.org} exited {proc.returncode} without a token file"
+            tenant = info.get("coreTenantId")
+            if (info.get("baseUrl") or "").rstrip("/") != self.base_url or (tenant and tenant != self.core_tenant_id):
+                return (f"resolve-token.sh --org {self.org} gives a token for another SFAP endpoint or core tenant "
+                        "than the run's (is --token-org the org --token-info was made for?)")
+            token = read_token(new_file)
+            if not token:
+                return "resolve-token.sh wrote an empty token file"
+            register_secret(token)
+            self._install(token)
+            self._minted[_digest(token)] = self._clock()
+            return None
+        finally:
+            if (new_file and os.path.abspath(new_file) != self.token_file
+                    and os.path.basename(new_file).startswith(SCRIPT_FILE_PREFIX) and os.path.isfile(new_file)):
+                os.unlink(new_file)
+
+    def _install(self, token):
+        """Replace the token file atomically: a reader sees the old token or the new one, never a partial file."""
+        fd, tmp = tempfile.mkstemp(dir=os.path.dirname(self.token_file), prefix=".bt-token-")  # mode 0600
+        try:
+            with os.fdopen(fd, "w") as f:
+                f.write(token)
+            os.replace(tmp, self.token_file)
+        except BaseException:
+            with contextlib.suppress(OSError):
+                os.unlink(tmp)
+            raise
+
+    @contextlib.contextmanager
+    def _file_lock(self):
+        fd = os.open(self.token_file + ".lock", os.O_RDWR | os.O_CREAT, 0o600)
+        try:
+            fcntl.flock(fd, fcntl.LOCK_EX)
+            yield
+        finally:
+            os.close(fd)  # releases the lock
+
+    def stats(self):
+        with self._lock:
+            return {"refreshes": self.refreshes, "failures": self.failures}