bookmark-research 0.6.0 → 0.7.0

package/src/readiness.py

@@ -64,8 +64,10 @@ class Readiness:
     def _signature(self, row, salt):
         # Only a salted signature is persisted; no keys, headers, or signatures
         # are included in CLI/MCP results. Rotation invalidates cached checks.
+        cred_env = row.get("credential_env")
+        cred_val = self.credentials.get(cred_env) if cred_env and cred_env in self.credentials.NAMES else None
         content = {"row": row, "settings": self.settings.load(),
-                   "credential": self.credentials.get(row["credential_env"]),
+                   "credential": cred_val,
                    "registry": self.web.registry, "version": self.VERSION}
         return hmac.new(salt.encode(), json.dumps(content, sort_keys=True).encode(), hashlib.sha256).hexdigest()
 
@@ -84,12 +86,19 @@ class Readiness:
             preferences["search"]["providers"] + preferences["search"]["fallback_providers"] +
             Settings.fetch_providers(preferences)))
         described = self.web.describe(names)["providers"]
+
+        def get_cred_env(p):
+            key = p.upper() + "_API_KEY"
+            return key if key in self.credentials.NAMES else None
+
         return [{"id": "retrieval:" + row["provider"], "kind": "retrieval", "provider": row["provider"],
-                 "credential_env": row["provider"].upper() + "_API_KEY",
+                 "credential_env": get_cred_env(row["provider"]),
                  "credential_configured": row["authentication_state"]["credential_configured"],
                  "authentication_mode": row["authentication_state"]["mode"],
                  "authentication_verified": False, "execution_verified": False,
-                 "status": "unchecked", "docs": row["docs"], "operations": row["operations"],
+                 "status": ("disabled" if row["provider"] == "browser" and not preferences.get("browser", {}).get("enabled")
+                            else "unchecked"),
+                 "docs": row["docs"], "operations": row["operations"],
                  "installation": "included_adapter", "network_checked": False,
                  "next_step": "Run readiness --refresh for connection checks; --test-retrieval also uses retrieval quota."}
                 for row in described]
@@ -101,12 +110,25 @@ class Readiness:
             key = ResearchHttp.PROVIDERS[name]["key_env"]
             enabled = config["enabled"] and (config["provider"] in (None, name) or service == name)
             configured = bool(self.credentials.get(key))
+            if not enabled and not configured:
+                diagnosis = "disabled_and_unconfigured"
+                next_step = "Run setup to enable professional_research and configure " + key + "."
+            elif not enabled and configured:
+                diagnosis = "disabled_with_credential_present"
+                next_step = "Credential " + key + " is present. Enable via settings.professional_research.enabled=true."
+            elif enabled and not configured:
+                diagnosis = "enabled_missing_credential"
+                next_step = "Service is enabled in settings, but missing API key. Configure " + key + " in environment or private credential file."
+            else:
+                diagnosis = "ready_for_check"
+                next_step = "Ready for deep research. Run readiness --refresh to verify connectivity."
             rows.append({"id": "service:" + name, "kind": "professional_research", "provider": name,
                          "credential_env": key, "credential_configured": configured,
                          "authentication_verified": False, "execution_verified": False,
                          "network_checked": False, "enabled": enabled,
                          "status": "unchecked" if configured and enabled else "disabled" if not enabled else "missing_credentials",
-                         "next_step": "Run setup to enable this optional service and configure " + key + ".",
+                         "diagnosis": diagnosis,
+                         "next_step": next_step,
                          "docs": "https://developers.openai.com/api/docs/guides/deep-research" if name == "openai"
                          else "https://docs.parallel.ai/task-api/guides/execute-task-run"})
         return rows
@@ -116,6 +138,9 @@ class Readiness:
         if test_retrieval:
             operations = copy.deepcopy(row["operations"])
             for purpose in ("search", "fetch"):
+                if not self.web.registry["providers"][name].get(purpose + "_tools"):
+                    operations[purpose]["status"] = "unsupported"
+                    continue
                 if operations[purpose]["missing"]:
                     operations[purpose]["status"] = "missing_credentials"
                     continue
@@ -140,8 +165,18 @@ class Readiness:
                     "capabilities": probe["capabilities"],
                     "next_step": "Search/fetch can be attempted. Catalog discovery does not verify retrieval authorization or quota."}
         if probe["status"] == "not_probed":
+            missing = sorted({key for operation in row.get("operations", {}).values()
+                              for key in operation.get("missing", [])})
+            supported = [purpose for purpose in ("search", "fetch")
+                         if self.web.registry["providers"][name].get(purpose + "_tools")]
+            guidance = []
+            if missing:
+                guidance.append("Configure " + ", ".join(missing) + ".")
+            if supported:
+                guidance.append("Supported operations: " + ", ".join(supported) + ".")
+            guidance.append("Use --test-retrieval for an actual check; this may use provider quota.")
             return {"status": "http_unchecked", "network_checked": False,
-                    "next_step": "Reader can be attempted anonymously; Search needs JINA_API_KEY. Use --test-retrieval for an actual check."}
+                    "next_step": " ".join(guidance)}
         return {"status": "failed", "network_checked": True,
                 "error_kind": probe.get("error_kind", "check_failed"),
                 "next_step": "Check credentials and connectivity, then refresh; another configured provider may remain available."}

package/src/ref_codec.py

@@ -0,0 +1,70 @@
+"""Process-local short reference registry.
+
+Short refs reduce repeated addressing cost while the registry keeps the
+authoritative source/version/object identity available to read and walk.
+References are opaque handles; callers must resolve them through this module.
+"""
+
+import hashlib
+import threading
+from collections import OrderedDict
+
+
+_MAX_REFS = 4096
+_LOCK = threading.RLock()
+_BY_REF = OrderedDict()
+_BY_IDENTITY = {}
+
+
+def _scope(kind, identity):
+    digest = hashlib.sha256((kind + "\0" + "\0".join(str(v) for v in identity)).encode("utf-8")).hexdigest()
+    return digest[:4]
+
+
+def register(kind, identity):
+    """Register an identity and return a collision-checked short ref."""
+    if not isinstance(kind, str) or len(kind) != 1 or kind not in ("b", "w", "a"):
+        raise ValueError("ref kind must be b, w or a")
+    identity = tuple(str(value) for value in identity)
+    key = (kind, identity)
+    with _LOCK:
+        existing = _BY_IDENTITY.get(key)
+        if existing is not None:
+            _BY_REF.move_to_end(existing)
+            return existing
+        scope = _scope(kind, identity)
+        sequence = 1
+        while True:
+            ref = f"{kind}-{scope}-{sequence}"
+            previous = _BY_REF.get(ref)
+            if previous is None or previous == key:
+                break
+            sequence += 1
+        _BY_REF[ref] = key
+        _BY_IDENTITY[key] = ref
+        _BY_REF.move_to_end(ref)
+        while len(_BY_REF) > _MAX_REFS:
+            old_ref, old_key = _BY_REF.popitem(last=False)
+            if _BY_IDENTITY.get(old_key) == old_ref:
+                del _BY_IDENTITY[old_key]
+        return ref
+
+
+def resolve(ref):
+    """Resolve a short ref to its kind and bound identity, or return None."""
+    if not isinstance(ref, str):
+        return None
+    with _LOCK:
+        key = _BY_REF.get(ref)
+        if key is None:
+            return None
+        _BY_REF.move_to_end(ref)
+        kind, identity = key
+        return {"kind": kind, "identity": list(identity)}
+
+
+def clear():
+    """Clear the process-local registry; intended for isolated tests."""
+    with _LOCK:
+        _BY_REF.clear()
+        _BY_IDENTITY.clear()

package/src/research.py

@@ -89,6 +89,18 @@ class ResearchSessions:
             raise ValueError("Invalid research_id")
         return research_id
 
+    @staticmethod
+    def _slug(value, fallback=None):
+        raw = (value or "").strip().lower()
+        slug = re.sub(r"[^a-z0-9]+", "-", raw).strip("-")
+        slug = re.sub(r"-+", "-", slug)[:80]
+        if slug and re.fullmatch(r"[a-z0-9][a-z0-9-]{0,79}", slug):
+            return slug
+        if fallback is not None:
+            return ResearchSessions._slug(fallback)
+        from wiki import WikiStore
+        return WikiStore._slug(value)
+
     @staticmethod
     def _registry():
         """Return the location registry; an unreadable registry means legacy lookup only."""
@@ -526,7 +538,8 @@ class ResearchSessions:
             self._integer(limit, key, 0, self.LIMITS[key])
         preferences = self.settings.load()
         names = self._providers(providers if providers is not None else preferences["search"]["providers"])
-        fetch_names = names[:] if providers is not None else Settings.fetch_providers(preferences)
+        fetch_names = ([name for name in names if name in Settings.FETCH_PROVIDERS]
+                       or Settings.fetch_providers(preferences)) if providers is not None else Settings.fetch_providers(preferences)
         search_fallbacks = [] if providers is not None else [name for name in preferences["search"]["fallback_providers"]
                                                           if name not in names]
         if not isinstance(scope, str) or len(scope) > 12000 or "\x00" in scope:
@@ -703,11 +716,22 @@ class ResearchSessions:
                 with BookmarkIndex(self.db_path) as index:
                     manager = SourceManager(index)
                     with index.read_snapshot():
+                        resolved_ids = {sid: index.resolve_source_id(sid) for sid in ids}
+                        resolved_refs = ([{**ref, "source_id": index.resolve_source_id(ref["source_id"])}
+                                          for ref in bookmark_refs] if bookmark_refs else None)
                         snapshot = {"version": index.input_version(ids),
-                                    "states": {sid: manager.source_status(sid)["state"] for sid in ids}}
+                                    "states": {sid: manager.source_status(resolved_ids[sid])["state"] for sid in ids},
+                                    "resolved_ids": resolved_ids}
                         if bookmark_refs:
                             snapshot["inventory"] = index.inventory(ids) if snapshot["version"] != frozen else None
                 cache[key] = snapshot
+            else:
+                with BookmarkIndex(self.db_path) as index:
+                    resolved_refs = ([{**ref, "source_id": index.resolve_source_id(ref["source_id"])}
+                                      for ref in bookmark_refs] if bookmark_refs else None)
+            redirected = {old: new for old, new in snapshot.get("resolved_ids", {}).items() if old != new}
+            if redirected:
+                result["resolved_via_redirect"] = redirected
             current, states = snapshot["version"], snapshot["states"]
             changed = current != frozen
             result.update(current_input_version=current, state="changed" if changed else "unchanged",
@@ -722,8 +746,18 @@ class ResearchSessions:
                     if hashlib.sha256(raw).hexdigest() != state.get("inventory_manifest_sha256"):
                         raise ValueError("Frozen input inventory hash changed")
                     cache[frozen_key] = json.loads(raw)
-                relevant = scoped_input(cache[frozen_key], bookmark_refs) != scoped_input(
-                    snapshot["inventory"], bookmark_refs)
+                frozen_inventory = cache[frozen_key]
+                if redirected:
+                    def remap(value):
+                        if isinstance(value, dict):
+                            return {key: (redirected.get(child, child) if key == "source_id" else remap(child))
+                                    for key, child in value.items()}
+                        if isinstance(value, list):
+                            return [remap(child) for child in value]
+                        return value
+                    frozen_inventory = remap(frozen_inventory)
+                relevant = scoped_input(frozen_inventory, resolved_refs) != scoped_input(
+                    snapshot["inventory"], resolved_refs)
                 result.update(state="changed" if relevant else "unchanged", requires_review=relevant,
                               package_changed=True, comparison_scope="cited_bookmarks_and_context")
         except (OSError, ValueError, KeyError, TypeError, sqlite3.Error) as exc:
@@ -1154,7 +1188,7 @@ class ResearchSessions:
                                {"fetch_calls": Settings.fetch_call_count(names[0], fresh) if fresh else 0}, execute)
 
     def import_evidence(self, research_id, operation_id, question_id, url, text, provenance,
-                        inventory_ids=None, title=""):
+                        inventory_ids=None, title="", bookmark_refs=None):
         """Save host-read page text or a secondary report without fetching it.
 
         A report can cite original input IDs, but those references never count
@@ -1188,14 +1222,16 @@ class ResearchSessions:
         if "run_id" in provenance:
             self._text(provenance["run_id"], "External run id", 2000)
         supplied_ids = None
+        is_internal_wiki = state.get("topic", "").startswith("Wiki: ") or state.get("brief", "").startswith("Wiki: ")
         if inventory_ids is not None:
             if not isinstance(inventory_ids, list):
                 raise ValueError("inventory_ids must be a list")
             supplied_ids = list(dict.fromkeys(self._identifier(value, "Inventory id") for value in inventory_ids))
-            for identifier in supplied_ids:
-                self._find(state.get("inventory", []), identifier, "inventory item")
+            if state.get("inventory") or not is_internal_wiki:
+                for identifier in supplied_ids:
+                    self._find(state.get("inventory", []), identifier, "inventory item")
         linked_ids, _ = self._source_links(state, url)
-        if kind != "external_report" and supplied_ids is not None and set(supplied_ids) - set(linked_ids):
+        if kind != "external_report" and supplied_ids is not None and (state.get("inventory") or not is_internal_wiki) and set(supplied_ids) - set(linked_ids):
             raise ValueError("Imported page URL does not match the selected original inventory URL")
         digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
         if "text_sha256" in provenance and provenance["text_sha256"] != digest:
@@ -1228,7 +1264,7 @@ class ResearchSessions:
                       "provenance": provenance, "review": {"verdict": "unreviewed", "text": None},
                       "inventory_ids": [] if kind == "external_report" else identifiers,
                       "referenced_inventory_ids": (supplied_ids or []) if kind == "external_report" else [],
-                      "bookmark_refs": [] if kind == "external_report" else refs}
+                      "bookmark_refs": bookmark_refs if bookmark_refs is not None else ([] if kind == "external_report" else refs)}
             # The commit assigns the source id and writes its manifest together.
             # The request snapshot cannot predict ids allocated by other writers.
             return {"status": "ok", "sources": [source], "next_action":
@@ -1730,6 +1766,8 @@ class ResearchSessions:
             policy = "suggest"
         if policy == "off":
             return {"policy": "off"}
+        if state.get("topic", "").startswith("Wiki: ") or state.get("brief", "").startswith("Wiki: "):
+            return {"policy": policy, "skip_reason": "internal_wiki_session", "candidates": [], "draft": None}
         eligible = [claim["id"] for claim in state["claims"] if claim.get("status", "active") == "active"
                     and all(self._find(state["sources"], citation["source_id"], "source").get("review", {}).get("verdict")
                             == "accepted" for citation in claim["citations"])]
@@ -1746,7 +1784,75 @@ class ResearchSessions:
             ranked = sorted(candidates.values(), key=lambda row: (-len(row["matched_terms"]), row["page_id"]))[:5]
         except Exception:  # A Wiki problem must never undo a saved report.
             ranked = []
+        draft_page = self._wiki_page(drafts, state)
+        written_page = None
+        skip_reason = None
         if policy == "auto":
+            if not drafts:
+                skip_reason = "insufficient_evidence" if not state.get("claims") else "no_accepted_claims"
+            elif state.get("status") == "active":
+                skip_reason = "session_active"
+            elif state.get("dry_run"):
+                skip_reason = "dry_run"
+            else:
+                try:
+                    from wiki import WikiStore
+                    store = WikiStore(settings=self.settings, research_sessions=self)
+                    auto_page = copy.deepcopy(draft_page["page"])
+                    auto_page["review"] = {
+                        "method": "model",
+                        "reviewer": "auto_research",
+                        "note": "Auto-synthesized reviewed claims from research session %s" % state["research_id"]
+                    }
+                    if ranked:
+                        candidate_page_id = ranked[0]["page_id"]
+                    else:
+                        title = state.get("title") or state.get("topic") or state.get("brief") or state["research_id"]
+                        candidate_page_id = self._slug(title, fallback=state["research_id"])
+                    expected_rev = 0
+                    existing = None
+                    try:
+                        existing = store.get(candidate_page_id)
+                    except Exception:
+                        existing = None
+                    if existing and existing.get("page"):
+                        expected_rev = existing.get("revision", 0)
+                        merged_sections = copy.deepcopy(existing["page"].get("sections", []))
+                        existing_headings = {s.get("heading") for s in merged_sections}
+                        for s in auto_page.get("sections", []):
+                            if s.get("heading") not in existing_headings and len(merged_sections) < WikiStore.MAX_SECTIONS:
+                                merged_sections.append(s)
+                                existing_headings.add(s.get("heading"))
+                        auto_page["sections"] = merged_sections
+                        existing_links = existing["page"].get("links", [])
+                        merged_links = copy.deepcopy(existing_links)
+                        existing_link_targets = {l.get("page_id") for l in merged_links}
+                        for l in auto_page.get("links", []):
+                            if l.get("page_id") not in existing_link_targets and len(merged_links) < 32:
+                                merged_links.append(l)
+                                existing_link_targets.add(l.get("page_id"))
+                        auto_page["links"] = merged_links
+                        if existing["page"].get("aliases"):
+                            auto_page["aliases"] = existing["page"]["aliases"]
+                    write_result = store.write(
+                        candidate_page_id,
+                        auto_page,
+                        change_note="Auto-saved research %s" % state["research_id"],
+                        expected_revision=expected_rev
+                    )
+                    written_page = {
+                        "status": "written",
+                        "page_id": write_result["page_id"],
+                        "revision": write_result["revision"],
+                        "markdown_file": write_result.get("markdown_file")
+                    }
+                except Exception as exc:
+                    conflict = "revision changed" in str(exc).lower()
+                    written_page = {
+                        "status": "conflict_aborted" if conflict else "failed",
+                        "error": str(exc),
+                        "skip_reason": "revision_conflict" if conflict else "write_failed"
+                    }
             instruction = ("Without asking the user, write or update Wiki pages with the reviewed claims: update a matching "
                            "candidate page (read it with wiki_get and pass its expected_revision) or create a new page from "
                            "eligible_claims, citing only those claims and marking inference statements as inference. "
@@ -1755,9 +1861,16 @@ class ResearchSessions:
             instruction = ("Present the candidate Wiki pages to update, or a proposed new page built from eligible_claims "
                            "(group them by question_id into draft sections; each claim carries exact first quotes and an "
                            "inference mark), to the user. Honor existing authorization; otherwise obtain confirmation before writing.")
-        return {"policy": policy, "candidates": ranked, "eligible_claim_ids": eligible, "eligible_claims": drafts,
-                "draft": self._wiki_page(drafts, state), "instruction": instruction,
-                "work": {"tool": "wiki_work", "arguments": {"research_id": state["research_id"]}}}
+        follow_up = {"policy": policy, "candidates": ranked, "eligible_claim_ids": eligible, "eligible_claims": drafts,
+                     "draft": draft_page, "instruction": instruction,
+                     "work": {"tool": "wiki_work", "arguments": {"research_id": state["research_id"]}}}
+        if written_page:
+            follow_up["written_page"] = written_page
+            follow_up.update(written_page)
+        elif skip_reason:
+            follow_up["status"] = "skipped"
+            follow_up["skip_reason"] = skip_reason
+        return follow_up
 
     @staticmethod
     def _question_gap(question, state):

package/src/research_plan.py

@@ -87,7 +87,17 @@ class ResearchPlan:
         if not terms:
             return [], None
         try:
-            return self._wiki().search_terms(terms, limit=limit), None
+            results = self._wiki().search_terms(terms, limit=limit)
+            if hasattr(self._wiki(), "_log_metric"):
+                pids = [row.get("page_id") for row in results if row.get("page_id")]
+                self._wiki()._log_metric(
+                    "plan_research",
+                    page_id=pids[0] if len(pids) == 1 else None,
+                    page_ids=pids if len(pids) > 1 else None,
+                    hit_count=len(results),
+                    reused=bool(results)
+                )
+            return results, None
         except Exception as exc:  # local optional check; never hide the bookmark scope
             return [], "Wiki preflight unavailable: " + str(exc)
 

package/src/routing.py

@@ -201,10 +201,12 @@ class ResearchRouting:
                                        if host_search else "always"),
                      "deadline_seconds": settings["search"]["deadline_seconds"],
                      "first_usable_result_wins": True,
+                     "backflow_tool": "merge_search_results",
                      "note": ("Host-native search is the default discovery channel. Use search_web when it is "
                               "unavailable, returns nothing or insufficient results, or the user asks for provider "
                               "diversity. For latency-sensitive work, issue both together and take the first "
-                              "usable result; do not wait for a slow channel."
+                              "usable result; do not wait for a slow channel. Host search results can be merged and "
+                              "archived via merge_search_results."
                               if host_search else
                               "The host has no observed native search; search_web runs its configured providers "
                               "in parallel with fallbacks for unresolved queries.")}
@@ -213,7 +215,8 @@ class ResearchRouting:
         return {"depth": selected_depth, "host": host, "task_shape": task_shape, "selected": selected,
                 "candidates": candidates, "missing_or_unconfirmed": missing,
                 "host_search": {"available": host_search is not None, "tool": host_search,
-                                "role": "discovery_only; fetch_web remains the evidence read"},
+                                "role": "discovery_only; fetch_web remains the evidence read",
+                                "backflow_tool": "merge_search_results"},
                 "discovery": discovery,
                 "next_action": next_action, "workflow": workflow, "research_process": process,
                 "fallback": {"execution_owner": "host", "failed_routes": failed,

package/src/search_results.py

@@ -7,6 +7,7 @@ Only snippets are retained; full source text belongs in a later fetch step.
 
 from collections import OrderedDict
 from urllib.parse import urlsplit, urlunsplit
+import hashlib
 import ipaddress
 import re
 
@@ -71,13 +72,27 @@ def _mode(providers, query_count):
     return "single_provider_multi_query" if query_count > 1 else "single_provider_single_query"
 
 
-def fuse_results(payload):
+def fuse_results(payload, archive=False, archive_dir=None, settings=None):
     """Return independently ranked targets, provenance, failures and coverage.
 
     Malformed top-level configuration raises ValueError. Invalid batches/results
     are reported and do not discard valid siblings. Provider counts include
     failed providers; successful-provider coverage is reported separately.
+    Deduplicates URLs and content SHA-256 hashes, with optional archive persistence.
     """
+    if isinstance(payload, dict) and "batches" not in payload and isinstance(payload.get("streams"), list):
+        default_target = payload.get("target") or payload.get("query") or "default"
+        payload = dict(payload)
+        payload["batches"] = []
+        for s in payload["streams"]:
+            if isinstance(s, dict):
+                payload["batches"].append({
+                    "target": s.get("target", default_target),
+                    "provider": s.get("provider", "host"),
+                    "query": s.get("query", payload.get("query", default_target)),
+                    "status": s.get("status", "ok"),
+                    "results": s.get("results") if "results" in s else s.get("items", [])
+                })
     if not isinstance(payload, dict) or not isinstance(payload.get("batches"), list):
         raise ValueError("payload must be an object with a batches array")
     limit = payload.get("limit_per_target", 10)
@@ -88,6 +103,7 @@ def fuse_results(payload):
         raise ValueError("targets must be an array of nonempty strings")
     buckets = OrderedDict((target.strip(), _bucket()) for target in requested)
     errors, invalid_results = [], []
+    total_submitted_results = 0
 
     def error(batch_index, message, target=None, provider=None, query=None, kind="invalid_batch"):
         errors.append({"batch_index": batch_index, "target": target,
@@ -130,6 +146,7 @@ def fuse_results(payload):
             continue
         stream["ok"] = True
         for position, item in enumerate(results, 1):
+            total_submitted_results += 1
             try:
                 if not isinstance(item, dict):
                     raise ValueError("result must be an object")
@@ -146,6 +163,9 @@ def fuse_results(payload):
                     snippet = item.get("text", "")
                 if not isinstance(snippet, str):
                     raise ValueError("text or snippet must be a string")
+                content = item.get("content")
+                if content is not None and not isinstance(content, str):
+                    raise ValueError("content must be a string")
             except ValueError as exc:
                 invalid_results.append({"batch_index": batch_index, "result_position": position,
                                         "target": target, "provider": provider, "query": query,
@@ -153,25 +173,36 @@ def fuse_results(payload):
                                         "reason": str(exc)})
                 bucket["invalid_count"] += 1
                 continue
+
+            text_body = content if content is not None else snippet
+            digest = hashlib.sha256(text_body.encode("utf-8")).hexdigest() if text_body else None
+
             snippet, truncated = snippet[:MAX_SNIPPET_CHARS], len(snippet) > MAX_SNIPPET_CHARS
             existing = stream["urls"].get(canonical)
             if existing is None:
                 stream["urls"][canonical] = {
                     "rank": rank, "raw_urls": {raw_url}, "title": title,
                     "snippet": snippet, "snippet_truncated": truncated,
+                    "content": content, "sha256": digest,
                 }
             else:
                 existing["rank"] = min(existing["rank"], rank)
                 existing["raw_urls"].add(raw_url)
                 if title:
                     existing["title"] = min(t for t in (existing["title"], title) if t)
+                if content and (not existing.get("content") or len(content) > len(existing["content"])):
+                    existing["content"] = content
+                    existing["sha256"] = digest
                 old = (len(existing["snippet"]), existing["snippet"], existing["snippet_truncated"])
                 new = (len(snippet), snippet, truncated)
                 if new > old:
                     existing["snippet"], existing["snippet_truncated"] = snippet, truncated
+                    if not existing.get("content"):
+                        existing["sha256"] = digest
 
     targets, all_providers, all_successful_providers = [], set(), set()
     query_count = 0
+    all_ranked_by_target = {}
     for target, bucket in buckets.items():
         streams = bucket["streams"]
         by_url, by_provider = {}, {}
@@ -188,17 +219,21 @@ def fuse_results(payload):
                     "provider": provider, "query": query, "rank": hit["rank"],
                     "url": raw_urls[0], "raw_urls": raw_urls, "title": hit["title"],
                     "snippet": hit["snippet"], "snippet_truncated": hit["snippet_truncated"],
+                    "content": hit.get("content"), "sha256": hit.get("sha256"),
                 })
         ranked = []
         for canonical, sources in by_url.items():
             sources.sort(key=lambda source: (source["rank"], source["provider"], source["query"]))
             provider_count = len({source["provider"] for source in sources})
+            preferred_sha = next((s["sha256"] for s in sources if s.get("sha256")), None)
             ranked.append({
                 "url": sources[0]["url"], "canonical_url": canonical,
                 "raw_urls": sorted({url for source in sources for url in source["raw_urls"]}),
                 "title": next((source["title"] for source in sources if source["title"]), ""),
+                "sha256": preferred_sha,
                 "rrf_score": sum(1.0 / (RRF_K + source["rank"]) for source in sources),
                 "provider_count": provider_count,
+                "providers": sorted({source["provider"] for source in sources}),
                 "corroborated": provider_count > 1,
                 "evidence_count": len(sources), "sources": sources,
             })
@@ -217,6 +252,7 @@ def fuse_results(payload):
         successful_providers = sorted(p for p, coverage in by_provider.items() if coverage["successful_queries"])
         for coverage in by_provider.values():
             coverage["unique_urls"] = len(coverage["unique_urls"])
+        all_ranked_by_target[target] = ranked
         targets.append({
             "target": target, "status": status, "providers": sorted(by_provider),
             "provider_count": len(by_provider), "query_count": len(streams),
@@ -232,6 +268,44 @@ def fuse_results(payload):
         all_providers.update(by_provider)
         all_successful_providers.update(successful_providers)
         query_count += len(streams)
+
+    all_canonical_urls = set()
+    sha_to_urls = {}
+    for target_items in all_ranked_by_target.values():
+        for item in target_items:
+            all_canonical_urls.add(item["canonical_url"])
+            h = item.get("sha256")
+            if h:
+                sha_to_urls.setdefault(h, set()).add(item["canonical_url"])
+
+    content_duplicates = [
+        {"sha256": h, "canonical_urls": sorted(urls)}
+        for h, urls in sha_to_urls.items() if len(urls) > 1
+    ]
+    total_ranked = sum(len(items) for items in all_ranked_by_target.values())
+    deduplication = {
+        "submitted_results": total_submitted_results,
+        "unique_canonical_urls": len(all_canonical_urls),
+        "unique_content_hashes": len(sha_to_urls),
+        "duplicate_urls_merged": max(0, total_submitted_results - total_ranked),
+        "cross_target_shared_urls": max(0, total_ranked - len(all_canonical_urls)),
+        "content_duplicates": content_duplicates,
+    }
+
+    archive_report = None
+    if archive or payload.get("archive") is True:
+        from archive import SourceArchive
+        from settings import Settings
+        target_dir = archive_dir
+        if not target_dir:
+            if settings:
+                target_dir = settings.load()["archive"]["directory"]
+            else:
+                target_dir = str(Settings.defaults()["archive"]["directory"])
+        store = SourceArchive(target_dir)
+        all_ranked_hits = [hit for target_items in all_ranked_by_target.values() for hit in target_items]
+        archive_report = store.save_search_batch(payload, all_ranked_hits)
+
     return {
         "provider_count": len(all_providers), "providers": sorted(all_providers),
         "successful_provider_count": len(all_successful_providers),
@@ -243,5 +317,9 @@ def fuse_results(payload):
                     "independent_fact_confirmation": False},
         "missing_targets": [item["target"] for item in targets if item["status"] == "missing"],
         "uncovered_targets": [item["target"] for item in targets if not item["results"]],
-        "targets": targets, "errors": errors, "invalid_results": invalid_results,
+        "targets": targets,
+        "errors": errors, "invalid_results": invalid_results,
+        "deduplication": deduplication,
+        "archive": archive_report if archive_report is not None else {"status": "disabled"},
+        "backflow_contract": "normalized_host_search_v1",
     }