failproofai 1.0.1-beta.1 → 1.0.2-beta.0

package/.next/standalone/sdk/python/failproofai_sdk/integrations/__init__.py

@@ -0,0 +1,274 @@
+"""Framework adapters: the registry behind `failproofai_sdk.instrument()`.
+
+    import failproofai_sdk
+    failproofai_sdk.instrument()            # every framework already imported
+    failproofai_sdk.instrument("crewai")    # exactly one
+    failproofai_sdk.uninstrument()          # put everything back
+
+**This module is stdlib only, and imports no adapter until asked.** The registry
+maps a name to a dotted module path *as a string*; `importlib.import_module`
+runs on demand. That is what keeps `import failproofai_sdk` free of LangChain.
+
+Auto-detection reads `sys.modules`, deliberately **not**
+`importlib.util.find_spec`. `find_spec` would report "installed", and acting on
+that means importing a framework the user is not using — 200ms and a pile of
+transitive imports charged to a library that was supposed to be invisible. If
+the framework is not imported, there is nothing to instrument.
+
+Writing an adapter
+------------------
+A module registered here must expose a module-level object named `adapter`
+implementing `failproofai_sdk.integrations._core.Adapter`:
+
+    name: str          # the registry name
+    module: str        # the framework module auto-detect looks for in sys.modules
+    def install(**options) -> None
+    def uninstall() -> None
+
+`install()` must save the **original attribute object** it replaces — use
+`_core.Patcher`, which also does the "somebody patched on top of us" check — and
+`uninstall()` must restore that saved object. Never re-import to restore: that
+hands back whatever the current value happens to be, which is how two
+instrumentation libraries silently un-patch each other.
+
+Every callback the adapter hands to the framework goes through `_core.safe`, and
+every event goes through a `_core.RunTracker`. Adapters own the translation
+table and nothing else.
+
+The four names are registered here **now**, before their modules exist, so that
+adding an adapter is one new file rather than an edit to this one.
+"""
+
+import importlib
+import logging
+import sys
+import threading
+from typing import Any
+
+from failproofai_sdk.integrations import _compat, _core
+from failproofai_sdk.integrations._core import Adapter
+
+logger = logging.getLogger("failproofai_sdk.integrations")
+
+__all__ = ["instrument", "uninstrument", "active", "available"]
+
+# name -> dotted module path. A string, imported on demand.
+_REGISTRY: dict[str, str] = {
+    "langchain": "failproofai_sdk.integrations.langchain",
+    "crewai": "failproofai_sdk.integrations.crewai",
+    "llama_index": "failproofai_sdk.integrations.llama_index",
+    "pydantic_ai": "failproofai_sdk.integrations.pydantic_ai",
+}
+
+# Spellings people actually type. LangGraph is served by the LangChain adapter
+# because LangGraph runs on langchain-core's callback manager.
+_ALIASES: dict[str, str] = {
+    "langgraph": "langchain",
+    "langchain_core": "langchain",
+    "llamaindex": "llama_index",
+    "llama-index": "llama_index",
+    "pydantic-ai": "pydantic_ai",
+    "pydanticai": "pydantic_ai",
+}
+
+# name -> the framework modules whose presence in sys.modules means "this
+# framework is in use". Kept here rather than read off `adapter.module` so that
+# detection imports nothing at all, not even our own adapter module.
+_DETECT: dict[str, tuple[str, ...]] = {
+    "langchain": ("langchain_core", "langchain", "langgraph"),
+    "crewai": ("crewai",),
+    "llama_index": ("llama_index", "llama_index.core"),
+    "pydantic_ai": ("pydantic_ai",),
+}
+
+# Guards _ACTIVE and every install/uninstall. `instrument()` is called from
+# application startup, which in a web server can be several threads at once.
+_LOCK = threading.Lock()
+_ACTIVE: dict[str, Adapter] = {}
+
+
+def available() -> tuple[str, ...]:
+    """Every registry name that can be instrumented, aliases excluded."""
+    return tuple(sorted(_REGISTRY))
+
+
+def active() -> tuple[str, ...]:
+    """Names currently instrumented."""
+    with _LOCK:
+        return tuple(sorted(_ACTIVE))
+
+
+def _canonical(name: str) -> str:
+    key = str(name).strip().lower().replace(" ", "")
+    key = _ALIASES.get(key, key)
+    if key not in _REGISTRY:
+        valid = ", ".join(sorted(set(_REGISTRY) | set(_ALIASES)))
+        raise ValueError(
+            f"failproofai_sdk: unknown framework {name!r}. Valid names are: {valid}. "
+            f"(Call failproofai_sdk.instrument() with no argument to auto-detect.)"
+        )
+    return key
+
+
+def _detected() -> list[str]:
+    """Registry names whose framework is already imported in this process."""
+    found = []
+    for name in _REGISTRY:
+        modules = _DETECT.get(name, (name,))
+        if any(module in sys.modules for module in modules):
+            found.append(name)
+    return found
+
+
+def _load(name: str) -> Adapter:
+    module = importlib.import_module(_REGISTRY[name])
+    adapter = getattr(module, "adapter", None)
+    if adapter is None:
+        # A module that is itself the adapter is fine too; the attribute is the
+        # convention, not the contract.
+        adapter = module
+    for attribute in ("install", "uninstall"):
+        if not callable(getattr(adapter, attribute, None)):
+            raise TypeError(
+                f"failproofai_sdk: adapter {_REGISTRY[name]!r} does not implement "
+                f"{attribute}() — see failproofai_sdk.integrations._core.Adapter."
+            )
+    return adapter  # type: ignore[return-value]
+
+
+def instrument(framework: str | None = None, **options: Any) -> tuple[str, ...]:
+    """Install the adapters. Returns the names newly instrumented.
+
+    With no argument, instruments every framework already imported in this
+    process. Instrumenting something already active is a no-op that returns
+    `()`, so calling this from two code paths (or from a reloading dev server)
+    cannot double-record.
+
+    An unknown name raises `ValueError` listing the valid ones — a typo that
+    silently records nothing is the worst outcome available. An adapter whose
+    `install()` raises is logged and skipped; the others still install, because
+    a broken LlamaIndex should not cost you LangGraph. `FAILPROOFAI_SDK_STRICT=1` turns
+    that skip back into a raise.
+    """
+    if framework is None:
+        names = _detected()
+        if not names:
+            # WARNING, not debug. This fires only when somebody explicitly asked
+            # for instrumentation and got none — the import-order mistake of
+            # calling instrument() above the `import langchain` line — and the
+            # result is a process that records nothing at all, with the adapter
+            # installed and the docs followed. At debug level the message that
+            # names the exact fix was invisible under every default logging
+            # config, which made the one mistake that costs you all your
+            # telemetry the one mistake we said nothing about.
+            logger.warning(
+                "failproofai_sdk: instrument() found no supported framework in sys.modules, "
+                "so NOTHING was instrumented. Import your framework before calling "
+                "instrument(), or name one explicitly: %s.",
+                # The names this call would have accepted, rather than one
+                # hardcoded example. A reader who is not using CrewAI has to
+                # work out for themselves whether the message is a suggestion
+                # or a diagnosis, which is a poor use of the one line they get.
+                ", ".join(f"instrument({name!r})" for name in sorted(_REGISTRY)),
+            )
+    else:
+        names = [_canonical(framework)]
+
+    installed: list[str] = []
+    with _LOCK:
+        for name in names:
+            if name in _ACTIVE:
+                continue
+            adapter = None
+            try:
+                adapter = _load(name)
+                adapter.install(**options)
+            except Exception as exc:
+                # An install is NOT atomic, so a failure part-way through leaves
+                # global state behind. `langchain.install()` sets `_STATE.enabled`,
+                # registers an append-only configure hook whose own uninstall
+                # comment says "There is no deregister API", builds the tracer and
+                # exports an env var — and only THEN probes a capability that can
+                # raise (a `_compat.probe` under FAILPROOFAI_SDK_STRICT_INTEGRATIONS=1
+                # always does). Catching here left the adapter fully patched and
+                # never recorded in `_ACTIVE`, so `active()` denied it existed and
+                # `uninstrument()` — which iterates `_ACTIVE` — could never undo it.
+                # It recorded for the life of the process and could not be removed.
+                if adapter is not None:
+                    try:
+                        adapter.uninstall()
+                    except Exception:
+                        logger.debug(
+                            "failproofai_sdk: rollback of a failed %r install did not "
+                            "complete cleanly",
+                            name,
+                            exc_info=True,
+                        )
+                if _core.strict():
+                    raise
+                # `FAILPROOFAI_SDK_STRICT_INTEGRATIONS=1` is documented as the
+                # supported way to make a compat problem loud — `_compat.py`'s
+                # own docstring calls warn-by-default "only defensible because
+                # there is a supported way to make it fail loudly". It was not
+                # one: the flag makes `_compat.warn` RAISE, this handler caught
+                # it, and the operator got neither behaviour — not the raise the
+                # flag promises, and not the best-effort instrumentation the
+                # warning text promises ("Instrumenting anyway"). An adapter
+                # silently recorded nothing while the log said it was fine.
+                if isinstance(exc, _compat.FailproofAICompatWarning) and (
+                    _compat.strict_integrations()
+                ):
+                    raise
+                logger.warning(
+                    "failproofai_sdk: could not instrument %r; the rest of your process is "
+                    "unaffected and other adapters still installed. Set "
+                    "FAILPROOFAI_SDK_STRICT=1 to raise instead.",
+                    name,
+                    exc_info=True,
+                )
+                continue
+            _ACTIVE[name] = adapter
+            installed.append(name)
+    return tuple(installed)
+
+
+def uninstrument(framework: str | None = None) -> tuple[str, ...]:
+    """Reverse `instrument()`. Returns the names removed. Never raises.
+
+    With no argument, removes everything. An unknown name, or a name that was
+    never instrumented, is a no-op — teardown that can fail is teardown people
+    stop calling.
+    """
+    if framework is None:
+        with _LOCK:
+            names = list(_ACTIVE)
+    else:
+        try:
+            names = [_canonical(framework)]
+        except ValueError as exc:
+            logger.warning("%s", exc)
+            return ()
+
+    removed: list[str] = []
+    with _LOCK:
+        for name in names:
+            adapter = _ACTIVE.pop(name, None)
+            if adapter is None:
+                continue
+            try:
+                adapter.uninstall()
+            except Exception:
+                logger.warning(
+                    "failproofai_sdk: %r did not uninstall cleanly; it is no longer "
+                    "registered, but some patches may remain.",
+                    name,
+                    exc_info=True,
+                )
+            removed.append(name)
+        if not _ACTIVE:
+            # Nothing is instrumented any more, so a later instrument() starts
+            # from a clean slate rather than inheriting a degraded call site or
+            # a warning that has "already been shown".
+            _core.reset_failures()
+            _compat.reset_warnings()
+    return tuple(removed)

package/.next/standalone/sdk/python/failproofai_sdk/integrations/_compat.py

@@ -0,0 +1,247 @@
+"""Version and capability probes for the framework adapters.
+
+Extras (`failproofai_sdk[langchain]`) express a *floor*, not enforcement: most users
+already have the framework and will never install an extra. So the real check
+happens here, at `instrument()` time, in three tiers:
+
+1. **framework not importable** -> a hard `ImportError` whose message contains
+   the literal install command. Instrumenting is an explicit user action, so
+   silently doing nothing is never the right answer.
+2. **importable but outside the declared range** -> `FailproofAICompatWarning`,
+   once, then best-effort. A ceiling exists because without one a clean build a
+   year from now pulls the next major, the callback API shifts, and the adapter
+   stops receiving events *while raising nothing*.
+3. **a capability probe fails** -> warn and no-op **that hook only**, never the
+   whole adapter.
+
+`FAILPROOFAI_SDK_STRICT_INTEGRATIONS=1` promotes every warning here to an exception.
+Warn-by-default is only defensible because there is a supported way to make it
+fail loudly.
+
+Why the version comparison is naive
+-----------------------------------
+`import failproofai_sdk` is contractually zero-dependency, so this cannot use
+`packaging`. `parse_version` therefore reads the **leading numeric components
+only** and stops at the first component that is not purely numeric:
+
+    "1.5.2"      -> (1, 5, 2)
+    "2.0.0b1"    -> (2, 0, 0)      # pre-release suffix ignored
+    "0.14.23.post1" -> (0, 14, 23) # local/post segments ignored
+    "1.2.dev0"   -> (1, 2)
+
+That means `2.0.0b1` compares **equal** to `2.0.0`, so a pre-release of a major
+we have declared a ceiling against will not be flagged. That is deliberate:
+the alternative is shipping a PEP 440 parser, and being wrong about a release
+candidate is much cheaper than a runtime dependency.
+
+Versions are always read with `importlib.metadata.version(dist)` and **never**
+`module.__version__` — that attribute is not guaranteed to exist and several of
+the frameworks we target do not define it.
+"""
+
+import importlib
+import importlib.metadata
+import logging
+import os
+import threading
+import warnings
+from types import ModuleType
+from typing import Any, Callable
+
+logger = logging.getLogger("failproofai_sdk.integrations")
+
+__all__ = [
+    "FailproofAICompatWarning",
+    "parse_version",
+    "version_string",
+    "version_tuple",
+    "require_module",
+    "check_version",
+    "probe",
+    "warn",
+    "strict_integrations",
+    "set_strict_integrations",
+    "reset_warnings",
+]
+
+
+class FailproofAICompatWarning(UserWarning):
+    """A framework is outside the range this adapter was written against."""
+
+
+_TRUTHY = frozenset({"1", "true", "yes", "on"})
+
+
+def _env_flag(name: str) -> bool:
+    """Read a boolean env var. Shared with `_core` so both flags parse alike."""
+    return os.environ.get(name, "").strip().lower() in _TRUTHY
+
+
+# Cached, because it is read on every warning and every `safe()` failure, and
+# resettable, because a test that cannot flip the switch cannot test the policy.
+# Same shape as `failproofai_sdk._environment`: a module global that overrides the env
+# var, with None meaning "not decided yet, go look".
+_strict_integrations: bool | None = None
+
+
+def strict_integrations() -> bool:
+    global _strict_integrations
+    if _strict_integrations is None:
+        _strict_integrations = _env_flag("FAILPROOFAI_SDK_STRICT_INTEGRATIONS")
+    return _strict_integrations
+
+
+def set_strict_integrations(value: bool | None) -> None:
+    """Override the flag. `None` re-reads `FAILPROOFAI_SDK_STRICT_INTEGRATIONS`."""
+    global _strict_integrations
+    _strict_integrations = value
+
+
+_warned: set[str] = set()
+_warn_lock = threading.Lock()
+
+
+def reset_warnings() -> None:
+    """Forget which warnings have already fired (tests; also `uninstrument()`)."""
+    with _warn_lock:
+        _warned.clear()
+
+
+def warn(message: str, *, key: str | None = None) -> None:
+    """Warn once per `key`, or raise if strict.
+
+    Deduplicated because these fire from `install()` *and* from hot callbacks:
+    a per-call warning on a chatty framework is its own outage.
+    """
+    if strict_integrations():
+        raise FailproofAICompatWarning(message)
+    dedup = key or message
+    with _warn_lock:
+        if dedup in _warned:
+            return
+        _warned.add(dedup)
+    warnings.warn(message, FailproofAICompatWarning, stacklevel=3)
+    logger.warning("%s", message)
+
+
+def parse_version(text: str) -> tuple[int, ...]:
+    """Leading numeric components of a version string. See the module docstring."""
+    parts: list[int] = []
+    for chunk in str(text).split("."):
+        digits = ""
+        for ch in chunk:
+            if not ch.isdigit():
+                break
+            digits += ch
+        if not digits:
+            break
+        parts.append(int(digits))
+        if len(digits) != len(chunk):
+            # A partially numeric component ("0b1", "dev0", "post1") ends the
+            # numeric prefix — everything after it is a pre/post/local segment.
+            break
+    return tuple(parts)
+
+
+def version_string(dist: str) -> str | None:
+    """The installed version of a distribution, or None if it is not installed.
+
+    `dist` is the *distribution* name (`langchain-core`), which is frequently
+    not the module name (`langchain_core`).
+    """
+    try:
+        return importlib.metadata.version(dist)
+    except importlib.metadata.PackageNotFoundError:
+        return None
+    except Exception:  # pragma: no cover - a broken METADATA must not break us
+        logger.debug("failproofai_sdk: could not read version of %r", dist, exc_info=True)
+        return None
+
+
+def version_tuple(dist: str) -> tuple[int, ...] | None:
+    text = version_string(dist)
+    return parse_version(text) if text else None
+
+
+def require_module(module: str, *, dist: str, extra: str) -> ModuleType:
+    """Import a framework module or raise with the literal install command.
+
+    Tier 1. `instrument("langchain")` on a machine without LangChain is a
+    mistake the user can fix in one command, so we hand them the command.
+    """
+    try:
+        return importlib.import_module(module)
+    except ImportError as exc:
+        raise ImportError(
+            f"failproofai_sdk: cannot instrument {extra!r} because {module!r} is not importable. "
+            f"Install it with:  pip install 'failproofai_sdk[{extra}]'  "
+            f"(or install {dist} directly)."
+        ) from exc
+
+
+def check_version(
+    framework: str,
+    dist: str,
+    *,
+    minimum: str | None = None,
+    below: str | None = None,
+    reason: str | None = None,
+) -> bool:
+    """Tier 2. True when `dist` is inside [minimum, below); warns once if not.
+
+    Returns True (best effort) for an unknown version too — a framework
+    installed from a git checkout has no usable metadata, and refusing to
+    instrument it would be a worse answer than trying.
+    """
+    found = version_string(dist)
+    if found is None:
+        return True
+    got = parse_version(found)
+    if not got:
+        return True
+
+    if minimum is not None and got < parse_version(minimum):
+        warn(
+            f"failproofai_sdk: {dist} {found} is older than the {minimum} this {framework} "
+            f"adapter was written against"
+            + (f" ({reason})" if reason else "")
+            + ". Instrumenting anyway; some events may be missing.",
+            key=f"{framework}:{dist}:min",
+        )
+        return False
+    if below is not None and got >= parse_version(below):
+        warn(
+            f"failproofai_sdk: {dist} {found} is newer than the <{below} this {framework} "
+            f"adapter was written against. Instrumenting anyway, but a callback API "
+            f"change would make it stop recording silently — please report this.",
+            key=f"{framework}:{dist}:max",
+        )
+        return False
+    return True
+
+
+def probe(framework: str, hook: str, check: Callable[[], Any]) -> bool:
+    """Tier 3. Run a capability probe; on failure warn and disable ONE hook.
+
+        if probe("langchain", "on_interrupt", lambda: langgraph.callbacks.GraphCallbackHandler):
+            ...wire it up...
+
+    A missing capability is never a reason to abandon the whole adapter: the
+    other 90% of the events are still correct and still worth having.
+    """
+    try:
+        ok = bool(check())
+    except Exception as exc:
+        warn(
+            f"failproofai_sdk: {framework} capability probe for {hook!r} failed ({exc!r}); "
+            f"that hook is disabled, the rest of the adapter is unaffected.",
+            key=f"{framework}:{hook}",
+        )
+        return False
+    if not ok:
+        warn(
+            f"failproofai_sdk: {framework} does not provide {hook!r} in this version; "
+            f"that hook is disabled, the rest of the adapter is unaffected.",
+            key=f"{framework}:{hook}",
+        )
+    return ok