bookmark-research 0.6.0 → 0.7.0

package/src/bookmark_index.py

@@ -14,6 +14,7 @@ from datetime import datetime, timezone
 from pathlib import Path, PurePosixPath
 from urllib.parse import urlsplit
 
+import storage
 from package_paths import (
     PROTOCOL_DIRS,
     dir_matches_section_type,
@@ -65,6 +66,7 @@ def url_kind(url):
 # into bigrams because a phrase the user typed rarely appears verbatim.
 PLAN_RUN = re.compile(r"[0-9A-Za-z][0-9A-Za-z+._-]*|[㐀-䶿一-鿿]+")
 QUESTION_URL = re.compile(r"https?://[^\s<>()\"']+")
+PLAN_CJK_SPLITTERS = re.compile(r"(?:有哪些|有什么|是什么|怎么|如何|为什么|怎样|[有的和在与或等及从到把被让我你他它这那])")
 PLAN_MAX_TERMS = 12
 
 
@@ -192,6 +194,10 @@ CREATE TABLE IF NOT EXISTS sources (
  source_id TEXT PRIMARY KEY, package_path TEXT NOT NULL,
  created_at TEXT NOT NULL, last_sync TEXT NOT NULL
 );
+CREATE TABLE IF NOT EXISTS source_redirects (
+ old_source_id TEXT PRIMARY KEY, new_source_id TEXT NOT NULL,
+ created_at TEXT NOT NULL, reason TEXT NOT NULL
+);
 CREATE TABLE IF NOT EXISTS files (
  source_id TEXT NOT NULL, file_path TEXT NOT NULL, kind TEXT NOT NULL,
  sha256 TEXT NOT NULL, raw_json TEXT NOT NULL, revision INTEGER NOT NULL DEFAULT 1,
@@ -222,6 +228,7 @@ CREATE TABLE IF NOT EXISTS items (
 );
 CREATE INDEX IF NOT EXISTS items_parent ON items(source_id,section_id,parent_id);
 CREATE INDEX IF NOT EXISTS items_url ON items(source_id,url);
+CREATE INDEX IF NOT EXISTS items_url_lookup ON items(url);
 CREATE TABLE IF NOT EXISTS item_tags (
  item_pk INTEGER NOT NULL REFERENCES items(pk) ON DELETE CASCADE,
  position INTEGER NOT NULL, color TEXT NOT NULL, text TEXT NOT NULL,
@@ -268,38 +275,68 @@ PRAGMA user_version=2;
 
 
 class BookmarkIndex:
-    def __init__(self, db_path):
+    def __init__(self, db_path, readonly=False):
         self.db_path = None if str(db_path) == ":memory:" else Path(db_path).expanduser().resolve()
+        self.connection = None
+        self.readonly = False
         if self.db_path is not None:
             # Refuse before connecting, so even an accidentally chosen package
             # path does not acquire an empty database, journal or WAL file.
             for parent in self.db_path.parents:
                 if is_package_dir(parent):
                     raise ValueError("Database must be outside imported package directories")
+        modes = [bool(readonly), not bool(readonly)] if self.db_path is not None else [False]
+        first_error = None
+        for mode in modes:
+            try:
+                self._initialize(mode)
+                break
+            except (OSError, sqlite3.DatabaseError) as error:
+                self.close()
+                if first_error is None:
+                    first_error = error
+                if not mode and not self._read_only_open_failure(error):
+                    raise
+            except Exception:
+                self.close()
+                raise
+        else:
+            raise first_error
+
+    @staticmethod
+    def _read_only_open_failure(error):
+        return storage.is_readonly_failure(error)
+
+    def _initialize(self, readonly):
+        if self.db_path is None:
+            connection = sqlite3.connect(":memory:")
+        elif readonly:
+            connection = storage.connect(self.db_path, readonly=True)
+        else:
             self.db_path.parent.mkdir(parents=True, exist_ok=True)
-        self.connection = sqlite3.connect(str(self.db_path) if self.db_path else ":memory:")
+            connection = sqlite3.connect(str(self.db_path))
+        self.connection = connection
+        self.readonly = readonly
         self.connection.row_factory = sqlite3.Row
         self.connection.execute("PRAGMA foreign_keys=ON")
         self.connection.execute("PRAGMA busy_timeout=5000")
         version = self.connection.execute("PRAGMA user_version").fetchone()[0]
         if version not in (0, 1, 2):
-            self.close()
             raise ValueError("Unsupported bookmark index schema version: %s" % version)
-        if version == 1:
-            # Add the added-at columns and fill them from stored raw_json before
-            # the schema script runs: CREATE TABLE IF NOT EXISTS cannot add a
-            # column, and the script would advance the version past this repair.
-            try:
+        if readonly:
+            if version != 2:
+                raise sqlite3.OperationalError("Read-only bookmark index requires schema version 2")
+            row = self.connection.execute(
+                "SELECT sql FROM sqlite_master WHERE name='items_fts'").fetchone()
+            self._fts_mode = "trigram" if row is not None and "trigram" in (row["sql"] or "") else "plain"
+        else:
+            if version == 1:
+                # Add the added-at columns and fill them from stored raw_json before
+                # the schema script runs: CREATE TABLE IF NOT EXISTS cannot add a
+                # column, and the script would advance the version past this repair.
                 self._upgrade_items_added_at()
-            except Exception:
-                self.close()
-                raise
-        try:
             self.connection.executescript(SCHEMA)
-        except Exception:
-            self.close()
-            raise
-        self._fts_mode = self._prepare_search_index()
+            self._fts_mode = self._prepare_search_index()
         self._window_functions = self._supports_window_functions()
 
     def _upgrade_items_added_at(self):
@@ -366,7 +403,9 @@ class BookmarkIndex:
         return True
 
     def close(self):
-        self.connection.close()
+        if self.connection is not None:
+            self.connection.close()
+            self.connection = None
 
     def __enter__(self):
         return self
@@ -374,8 +413,21 @@ class BookmarkIndex:
     def __exit__(self, *_):
         self.close()
 
+    def resolve_source_id(self, source_id):
+        """Resolve retired source IDs to their surviving merge target."""
+        current, seen = source_id, set()
+        while current not in seen:
+            seen.add(current)
+            row = self.connection.execute(
+                "SELECT new_source_id FROM source_redirects WHERE old_source_id=?", (current,)).fetchone()
+            if row is None:
+                return current
+            current = row[0]
+        raise ValueError("Source redirect cycle: " + str(source_id))
+
     def _source(self, source_id):
-        row = self.connection.execute("SELECT * FROM sources WHERE source_id=?", (source_id,)).fetchone()
+        resolved = self.resolve_source_id(source_id)
+        row = self.connection.execute("SELECT * FROM sources WHERE source_id=?", (resolved,)).fetchone()
         if row is None:
             raise ValueError("Unknown source_id: " + str(source_id))
         return dict(row)
@@ -443,8 +495,13 @@ class BookmarkIndex:
         else:
             if anchor:
                 raise ValueError("Only permanent sections can be copy anchors")
-            section_id = obj.get("id")
             label = obj.get("label", "unknown")
+            section_id = obj.get("id")
+            if not section_id:
+                # The canvas right-click export omits ids; derive a stable one so
+                # re-importing the same file keeps refs and duplicate groups.
+                digest = hashlib.sha256(("temporary|" + str(label) + "|" + _string(obj.get("title", ""), "section title") + "|" + file_path).encode("utf-8")).hexdigest()[:12]
+                section_id = "temp-section-export-" + digest
         _string(section_id, "section id", True)
         _string(label, "section label", True)
         return section_id, {
@@ -491,7 +548,14 @@ class BookmarkIndex:
             for position, node in enumerate(nodes):
                 if not isinstance(node, dict):
                     raise ValueError("Items must be JSON objects")
-                item_id = _string(node.get("id"), "item id", True)
+                if node.get("id") is None:
+                    # Exported items carry no id; derive one from position and
+                    # content so re-imports stay stable and duplicates still group.
+                    seed = "|".join([section_id, parent_id or "", str(position),
+                                     str(node.get("url") or ""), str(node.get("title") or "")])
+                    item_id = ("tempId_" if not permanent else "persistId_") + hashlib.sha256(seed.encode("utf-8")).hexdigest()[:16]
+                else:
+                    item_id = _string(node.get("id"), "item id", True)
                 if item_id in seen:
                     raise ValueError("Duplicate item id in section: " + item_id)
                 seen.add(item_id)
@@ -500,8 +564,9 @@ class BookmarkIndex:
                 if not permanent and node.get("sectionId", section_id) != section_id:
                     raise ValueError("Temporary item sectionId mismatch: " + item_id)
                 title = _string(node.get("title", ""), "item title")
-                url = _string(node.get("url", ""), "item URL")
-                item_type = ("bookmark" if url else "folder") if permanent else node.get("type")
+                url = _string(node.get("url") or "", "item URL")
+                item_type = (("bookmark" if url else "folder") if permanent
+                             else (node.get("type") or ("bookmark" if url else "folder")))
                 if item_type not in ("bookmark", "folder") or (item_type == "bookmark" and not url):
                     raise ValueError("Invalid bookmark/folder type: " + item_id)
                 children = node.get("children", [])
@@ -531,8 +596,12 @@ class BookmarkIndex:
                     "tag_text": "\n".join(tag["text"] for tag in normalized),
                     "path_json": _dump(list(titles)), "folder_path": " / ".join(titles),
                     "ancestor_ids_json": _dump(list(ancestor_ids)),
-                    # Full child arrays also survive in files/sections.raw_json.
-                    "raw_json": _dump({key: value for key, value in node.items() if key != "children"}),
+                    # Keep tree structure in parent_id/path/position and in the
+                    # source-file snapshot; duplicating descendants here makes
+                    # every ancestor appear changed when one child is edited.
+                    "raw_json": _dump({key: value for key, value in node.items()
+                                       if key != "children"
+                                       and not (item_type == "folder" and key == "url" and value == "")}),
                     "metadata_json": _dump(item_meta if permanent else {key: node[key] for key in ("tags", "note", "noteColor") if key in node}),
                     "added_at": added[0], "added_at_source": added[1], "added_at_confidence": added[2],
                 }))
@@ -597,6 +666,8 @@ class BookmarkIndex:
     @contextmanager
     def transaction(self):
         """Serialize writers; allow an importer to publish metadata atomically."""
+        if self.readonly:
+            raise sqlite3.OperationalError("attempt to write a readonly bookmark index")
         nested = self.connection.in_transaction
         if nested:
             self.connection.execute("SAVEPOINT bookmark_sync")
@@ -850,7 +921,8 @@ class BookmarkIndex:
         return rows[0]
 
     def _scope(self, source_id, section=None, group_id=None, folder_id=None, tags=None, tag_colors=None,
-               item_types=None):
+               item_types=None, has_note=None, has_tags=None, metadata_match="all"):
+        source_id = self.resolve_source_id(source_id)
         self._source(source_id)
         selected_types = list(item_types) if item_types else ["bookmark"]
         clauses = ["i.source_id=?", "i.item_type IN (%s)" % ",".join("?" for _ in selected_types)]
@@ -892,15 +964,189 @@ class BookmarkIndex:
             _string(color, "tag color", True)
             clauses.append("EXISTS (SELECT 1 FROM item_tags t WHERE t.item_pk=i.pk AND t.color=?)")
             args.append(color)
+        metadata_clauses = []
+        if has_note is not None:
+            metadata_clauses.append("trim(i.note)<>''" if has_note else "trim(i.note)=''")
+        if has_tags is not None:
+            metadata_clauses.append("EXISTS (SELECT 1 FROM item_tags t WHERE t.item_pk=i.pk)"
+                                    if has_tags else
+                                    "NOT EXISTS (SELECT 1 FROM item_tags t WHERE t.item_pk=i.pk)")
+        if metadata_clauses:
+            clauses.append("(" + (" OR " if metadata_match == "any" else " AND ").join(metadata_clauses) + ")")
         return clauses, args
 
+    def source_summary(self, source_id):
+        """Recognise a freshly imported package: counts, sections, noise, duplicates."""
+        source_id = self.resolve_source_id(source_id)
+        from url_classify import is_search_result_url
+        counts = {}
+        for row in self.connection.execute(
+                "SELECT item_type, COUNT(*) AS c FROM items WHERE source_id=? GROUP BY item_type",
+                (source_id,)):
+            counts[row["item_type"]] = row["c"]
+        notes = self.connection.execute(
+            "SELECT COUNT(*) FROM items WHERE source_id=? AND note!=''", (source_id,)).fetchone()[0]
+        notes_bookmarks = self.connection.execute(
+            "SELECT COUNT(*) FROM items WHERE source_id=? AND note!='' AND item_type='bookmark'",
+            (source_id,)).fetchone()[0]
+        tagged = self.connection.execute(
+            "SELECT COUNT(*) FROM items WHERE source_id=? AND tag_text!=''", (source_id,)).fetchone()[0]
+        section_counts = {row["section_id"]: row["c"] for row in self.connection.execute(
+            "SELECT section_id, COUNT(*) AS c FROM items WHERE source_id=? GROUP BY section_id",
+            (source_id,))}
+        sections = []
+        for row in self.connection.execute(
+                "SELECT section_id, label, title, section_type, file_path, raw_json FROM sections"
+                " WHERE source_id=? ORDER BY section_id", (source_id,)):
+            try:
+                raw = json.loads(row["raw_json"]) if row["raw_json"] else {}
+            except ValueError:
+                raw = {}
+            sections.append({**dict(row), "items": section_counts.get(row["section_id"], 0),
+                             "source": raw.get("source") or "",
+                             "temp_kind": raw.get("tempKind") or ""})
+            sections[-1].pop("raw_json", None)
+        urls = [row["url"] for row in self.connection.execute(
+            "SELECT DISTINCT url FROM items WHERE source_id=? AND url!=''", (source_id,))]
+        search_pages = sum(1 for url in urls if is_search_result_url(url))
+        from junk import is_junk
+        junk_items = 0
+        for row in self.connection.execute(
+                "SELECT title, url FROM items WHERE source_id=? AND item_type='bookmark'",
+                (source_id,)):
+            if is_junk(row["title"], row["url"]):
+                junk_items += 1
+        duplicates = self.duplicate_urls(source_id, include_search=False, limit=1)
+        return {"source_id": source_id,
+                "counts": {"items": sum(counts.values()), "bookmarks": counts.get("bookmark", 0),
+                           "folders": counts.get("folder", 0)},
+                "notes": notes, "notes_bookmarks": notes_bookmarks,
+                "notes_folders": notes - notes_bookmarks, "tagged": tagged, "unique_urls": len(urls),
+                "search_pages": search_pages, "junk_items": junk_items,
+                "duplicate_url_groups": duplicates["total_groups"],
+                "sections": sections}
+
+    def duplicate_urls(self, source_id, min_count=2, limit=50, offset=0, include_search=False):
+        """Group bookmarks that point at the same URL and suggest which copy to keep.
+
+        The suggestion prefers an instance with a note, then the earliest added
+        copy, then the shallowest folder path.  It is a proposal, never an edit.
+        """
+        source_id = self.resolve_source_id(source_id)
+        if isinstance(min_count, bool) or not isinstance(min_count, int) or min_count < 2:
+            raise ValueError("min_count must be an integer of at least 2")
+        if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 500:
+            raise ValueError("limit must be an integer from 1 to 500")
+        if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
+            raise ValueError("offset must be a non-negative integer")
+        if type(include_search) is not bool:
+            raise ValueError("include_search must be a boolean")
+        from junk import is_junk
+        from url_classify import is_search_result_url
+        base = ("FROM items WHERE source_id=? AND item_type='bookmark'"
+                " AND url IS NOT NULL AND url!=''")
+        rows = self.connection.execute(
+            "SELECT url, COUNT(*) AS copies, MIN(added_at) AS first_added " + base +
+            " GROUP BY url HAVING copies>=? ORDER BY copies DESC, first_added, url",
+            (source_id, min_count)).fetchall()
+        groups = [dict(row) for row in rows]
+        # Placeholder rows (divider searches) are not real duplicates; drop them
+        # before counting groups so the report stays honest.
+        junk_seen = 0
+        kept_groups = []
+        for group in groups:
+            if is_junk("", group["url"]):
+                junk_seen += group["copies"]
+                continue
+            kept_groups.append(group)
+        groups = kept_groups
+        if not include_search:
+            groups = [group for group in groups if not is_search_result_url(group["url"])]
+        urls = [group["url"] for group in groups]
+        by_url = {}
+        if urls:
+            placeholders = ",".join("?" for _ in urls)
+            instance_rows = self.connection.execute(
+                "SELECT section_id, item_id, url, title, note, note_color, folder_path, added_at, position "
+                + base + " AND url IN (" + placeholders + ") ORDER BY url, added_at, folder_path, item_id",
+                [source_id, *urls]).fetchall()
+            for row in instance_rows:
+                by_url.setdefault(row["url"], []).append(dict(row))
+        kept = []
+        for group in groups:
+            copies = []
+            for instance in by_url.get(group["url"], []):
+                if is_junk(instance.get("title"), instance.get("url")):
+                    junk_seen += 1
+                    continue
+                instance["depth"] = (instance.get("folder_path") or "").count("/") + 1
+                copies.append(instance)
+            if len(copies) < min_count:
+                continue
+            keep = sorted(copies, key=lambda item: (
+                0 if item.get("note") else 1, item.get("added_at") or "9999",
+                item["depth"], item["section_id"], item["item_id"]))[0]
+            group["copies"] = len(copies)
+            group["keep"] = {key: keep[key] for key in ("section_id", "item_id", "title",
+                                                        "folder_path", "note", "added_at")}
+            group["instances"] = copies
+            kept.append(group)
+        total_groups = len(kept)
+        page = kept[offset:offset + limit]
+        shown = offset + len(page)
+        return {"source_id": source_id, "total_groups": total_groups, "groups": page,
+                "limit": limit, "offset": offset,
+                "next_offset": shown if shown < total_groups else None,
+                "include_search": include_search, "junk_instances_excluded": junk_seen}
+
+    def folder_children(self, source_id, folder_id, section=None, tags=None, tag_colors=None,
+                        has_note=None, has_tags=None, metadata_match="all", limit=8, offset=0):
+        """List one folder's direct bookmark/folder children with stable pagination."""
+        source_id = self.resolve_source_id(source_id)
+        if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 1000:
+            raise ValueError("limit must be an integer from 1 to 1000")
+        if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
+            raise ValueError("offset must be a non-negative integer")
+        self._source(source_id)
+        rows = self.connection.execute(
+            "SELECT * FROM items WHERE source_id=? AND item_type='folder' AND item_id=?",
+            (source_id, folder_id)).fetchall()
+        if section is not None:
+            selected_sections = {row["canonical_id"] or row["section_id"]
+                                 for row in self._sections(source_id, section)}
+            rows = [row for row in rows if row["section_id"] in selected_sections]
+        unique = {(row["section_id"], row["item_id"]): row for row in rows}
+        if not unique:
+            raise ValueError("Unknown folder ID: " + str(folder_id))
+        if len(unique) > 1:
+            raise ValueError("Ambiguous folder ID; pass section to select one: " + str(folder_id))
+        folder = next(iter(unique.values()))
+        clauses, parameters = self._scope(
+            source_id, section=folder["section_id"], tags=tags, tag_colors=tag_colors,
+            item_types=["bookmark", "folder"], has_note=has_note, has_tags=has_tags,
+            metadata_match=metadata_match)
+        clauses.append("i.parent_id=?")
+        parameters.append(folder_id)
+        where = " AND ".join(clauses)
+        total = self.connection.execute("SELECT count(*) FROM items i WHERE " + where, parameters).fetchone()[0]
+        children = self.connection.execute(
+            "SELECT i.* FROM items i WHERE " + where +
+            " ORDER BY i.position,i.preorder,i.item_id LIMIT ? OFFSET ?",
+            parameters + [limit, offset]).fetchall()
+        anchors = self._section_anchors(source_id)
+        return {"source_id": source_id, "folder": self._item_result(folder, anchors),
+                "total": total, "limit": limit, "offset": offset,
+                "results": [self._item_result(row, anchors) for row in children]}
+
     @staticmethod
     def plan_terms(question):
         """Question text into literal search terms; no semantics, no guessing.
 
         ASCII runs of two or more characters are kept whole. A CJK run of four
-        or fewer characters is kept whole as well; a longer run contributes its
-        bigrams instead, so 本地部署 still reaches a card that only says 部署.
+        or fewer characters is kept whole as well, with boundary-aligned bigrams
+        when exactly four characters. A longer run contributes boundary-aligned
+        bigrams without cross-word fragments, so 本地部署 still reaches a card
+        that only says 部署.
         A single ASCII character is dropped, because it would match almost
         everything; a single CJK character is kept, because it is often the
         whole entity. The order is the order of appearance and the list is
@@ -918,14 +1164,26 @@ class BookmarkIndex:
             if run[0].isascii():
                 candidates = [run] if len(run) >= 2 else []
             else:
-                candidates = [run] if len(run) <= 4 else []
-                candidates.extend(run[index:index + 2] for index in range(len(run) - 1))
+                candidates = []
+                sub_segments = [s for s in PLAN_CJK_SPLITTERS.split(run) if s]
+                for segment in sub_segments:
+                    if len(segment) <= 4:
+                        candidates.append(segment)
+                        if len(segment) == 4:
+                            candidates.extend([segment[:2], segment[2:]])
+                    else:
+                        if len(segment) % 2 == 0:
+                            candidates.extend(segment[i:i + 2] for i in range(0, len(segment) - 1, 2))
+                        else:
+                            for i in range(0, len(segment) - 1, 2):
+                                candidates.append(segment[i:i + 2])
+                            candidates.append(segment[-2:])
             for candidate in candidates:
                 if candidate not in terms:
                     terms.append(candidate)
         return terms[:PLAN_MAX_TERMS]
 
-    def plan_scope(self, source_id, question, max_cards=3, max_urls=8, homepage_limit=0):
+    def plan_scope(self, source_id, question, max_cards=3, max_urls=8, homepage_limit=0, field_score=False):
         """Rank the cards a question touches and the bookmarks inside them.
 
         Local only: literal matching over indexed metadata, no fetch and no
@@ -942,6 +1200,7 @@ class BookmarkIndex:
         cards, deduplicated by URL, with the terms each URL's own rows matched
         and the source-wide instance count, which is not the selected scope.
         """
+        source_id = self.resolve_source_id(source_id)
         source = self._source(source_id)
         package = self._input_path(source_id, source["package_path"])
         # Partial imports keep cards whose file disappeared. They stay indexed,
@@ -956,6 +1215,8 @@ class BookmarkIndex:
             raise ValueError("max_urls must be an integer from 1 to 8")
         if type(homepage_limit) is not int or not 0 <= homepage_limit <= 8:
             raise ValueError("homepage_limit must be an integer from 0 to 8")
+        if type(field_score) is not bool:
+            raise ValueError("field_score must be a boolean")
         terms = self.plan_terms(question)
         result = {"source_id": source_id, "terms": terms, "term_cards": {}, "cards": [], "urls": [],
                   "truncated": False, "errors": []}
@@ -1029,8 +1290,20 @@ class BookmarkIndex:
         # ascending tie-breaks, the second orders by score, hits and freshness,
         # and equal rows keep the order the first pass gave them.
         ordered = sorted(hits, key=lambda value: (sections[value]["file_path"], value))
-        ordered.sort(key=lambda value: (scores[value] * density[value], hits[value],
-                                        (newest.get(value) or (None, None))[0] or ""), reverse=True)
+        if field_score:
+            card_field_scores = {}
+            for sid in scores:
+                sec_items = [dict(r) for r in self.connection.execute(
+                    "SELECT * FROM items i WHERE i.source_id=? AND i.section_id=? AND i.item_type='bookmark' AND (" + where + ")",
+                    (source_id, sid, *parameters)).fetchall()]
+                total_f = sum(BookmarkIndex.score_item(r, terms)[0] for r in sec_items)
+                card_field_scores[sid] = total_f / (bookmarks.get(sid) or 1)
+            ordered.sort(key=lambda value: (card_field_scores.get(value, 0.0) * density[value],
+                                            scores[value] * density[value], hits[value],
+                                            (newest.get(value) or (None, None))[0] or ""), reverse=True)
+        else:
+            ordered.sort(key=lambda value: (scores[value] * density[value], hits[value],
+                                            (newest.get(value) or (None, None))[0] or ""), reverse=True)
         trees, ranked = set(), []
         for section_id in ordered:
             row = sections[section_id]
@@ -1152,8 +1425,10 @@ class BookmarkIndex:
     def _item_result(self, row, anchors=None, section_cache=None):
         keys = row.keys()
         result = {key: row[key] for key in ("source_id", "section_id", "item_id", "parent_id", "item_type", "title",
-                                            "url", "note", "note_color", "position")
+                                            "note", "note_color", "position")
                   + (("added_at", "added_at_source", "added_at_confidence") if "added_at" in keys else ())}
+        if row["item_type"] != "folder" or row["url"]:
+            result["url"] = row["url"]
         result["added_at_precision"] = ADDED_AT_SOURCES.get(result.get("added_at_source"), (None,))[0]
         result.update(source=row["source_id"], section=row["section_id"], item=row["item_id"],
             path=json.loads(row["path_json"]), folder_path=row["folder_path"], tags=json.loads(row["tags_json"]),
@@ -1177,15 +1452,93 @@ class BookmarkIndex:
         result["section_label"], result["file_path"] = section[0], section[1]
         return result
 
+    @staticmethod
+    def score_item(row, targets=None):
+        """Compute non-binding weighted scoring hints for a candidate item.
+
+        Scoring tiers:
+        - title_prefix: 140
+        - title_contains: 135
+        - url: 120
+        - tag: 110
+        - note: 90
+        - folder_path: inherited evidence only; it is not an item score
+        Time confidence tier bonus:
+        - high: +10
+        - medium: +5
+        - low: +2
+        """
+        targets = targets or []
+        keys = set(row.keys()) if hasattr(row, "keys") else set()
+        title = (row["title"] if "title" in keys and row["title"] else "").casefold()
+        note = (row["note"] if "note" in keys and row["note"] else "").casefold()
+        folder_path = (row["folder_path"] if "folder_path" in keys and row["folder_path"] else "").casefold()
+        url = (row["url"] if "url" in keys and row["url"] else "").casefold()
+        tag_text = (row["tag_text"] if "tag_text" in keys and row["tag_text"] else "").casefold()
+
+        field_hits = []
+        score = 0
+        for target in targets:
+            t = (target or "").casefold().strip()
+            if not t:
+                continue
+            if title.startswith(t):
+                if "title_prefix" not in field_hits:
+                    field_hits.append("title_prefix")
+                    score += 140
+            elif t in title:
+                if "title_contains" not in field_hits:
+                    field_hits.append("title_contains")
+                    score += 135
+            if t in tag_text:
+                if "tag" not in field_hits:
+                    field_hits.append("tag")
+                    score += 110
+            if t in note:
+                if "note" not in field_hits:
+                    field_hits.append("note")
+                    score += 90
+            if t in folder_path and "folder_path" not in field_hits:
+                field_hits.append("folder_path")
+            if t in url:
+                if "url" not in field_hits:
+                    field_hits.append("url")
+                    score += 120
+
+        conf = row["added_at_confidence"] if "added_at_confidence" in keys else None
+        if conf == "high":
+            score += 10
+            field_hits.append("time_high")
+        elif conf == "medium":
+            score += 5
+            field_hits.append("time_medium")
+        elif conf == "low":
+            score += 2
+            field_hits.append("time_low")
+
+        return score, field_hits
+
     def search(self, source_id, targets=None, section=None, group_id=None, folder_id=None, tags=None,
-               limit=20, offset=0, tag_colors=None, item_types=None, count_only=False, compact=False):
+               limit=20, offset=0, tag_colors=None, item_types=None, count_only=False, compact=False,
+               score=False, has_note=None, has_tags=None, metadata_match="all", sort_by="default"):
         """Literal structured search. ``limit=0`` or ``count_only`` returns counts only."""
+        source_id = self.resolve_source_id(source_id)
         if isinstance(limit, bool) or not isinstance(limit, int) or not 0 <= limit <= 1000:
             raise ValueError("limit must be an integer from 0 to 1000; 0 returns counts only")
         if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
             raise ValueError("offset must be a non-negative integer")
         if type(compact) is not bool:
             raise ValueError("compact must be a boolean")
+        if type(score) is not bool:
+            raise ValueError("score must be a boolean")
+        if has_note is not None and type(has_note) is not bool:
+            raise ValueError("has_note must be a boolean or null")
+        if has_tags is not None and type(has_tags) is not bool:
+            raise ValueError("has_tags must be a boolean or null")
+        if metadata_match not in ("all", "any"):
+            raise ValueError("metadata_match must be all or any")
+        if sort_by not in ("default", "added_at_desc", "added_at_asc"):
+            raise ValueError("sort_by must be default, added_at_desc or added_at_asc")
         if isinstance(targets, str) or isinstance(tags, str) or isinstance(tag_colors, str):
             raise ValueError("targets, tags and tag_colors must be lists of strings")
         targets = list(dict.fromkeys(targets or []))
@@ -1198,7 +1551,8 @@ class BookmarkIndex:
                     or any(value not in ("bookmark", "folder") for value in item_types)
                     or len(set(item_types)) != len(item_types)):
                 raise ValueError("item_types must select bookmark and/or folder without duplicates")
-        clauses, args = self._scope(source_id, section, group_id, folder_id, tags, tag_colors, item_types)
+        clauses, args = self._scope(source_id, section, group_id, folder_id, tags, tag_colors, item_types,
+                                    has_note, has_tags, metadata_match)
         page_size = 0 if count_only else limit
         anchors_by_section = self._section_anchors(source_id)
         section_cache = {}
@@ -1212,34 +1566,94 @@ class BookmarkIndex:
             where = " AND ".join(predicates)
             rows, total = [], None
             if page and page_size:
-                if self._window_functions:
+                if score:
+                    total = self.connection.execute(
+                        "SELECT count(*) FROM items i WHERE " + where, parameters).fetchone()[0]
+                    candidate_limit = max(1000, offset + page_size * 5)
+                    if total > candidate_limit:
+                        all_rows = self.connection.execute(
+                            "SELECT i.* FROM items i WHERE " + where +
+                            " ORDER BY (i.added_at IS NULL), i.added_at DESC, i.section_id, i.preorder LIMIT ?",
+                            parameters + [candidate_limit]).fetchall()
+                    else:
+                        all_rows = self.connection.execute(
+                            "SELECT i.* FROM items i WHERE " + where, parameters).fetchall()
+                    scored_rows = []
+                    for r in all_rows:
+                        s_val, s_hits = BookmarkIndex.score_item(r, chosen)
+                        scored_rows.append((r, s_val, s_hits))
+                    scored_rows.sort(key=lambda t: (t[0]["section_id"], t[0]["preorder"], t[0]["item_id"]))
+                    if sort_by == "default":
+                        scored_rows.sort(key=lambda t: t[0]["added_at"] or "", reverse=True)
+                        scored_rows.sort(key=lambda t: 0 if t[0]["added_at"] is not None else 1)
+                        scored_rows.sort(key=lambda t: t[1], reverse=True)
+                    else:
+                        scored_rows.sort(key=lambda t: t[1], reverse=True)
+                        scored_rows.sort(key=lambda t: t[0]["added_at"] or "",
+                                         reverse=sort_by == "added_at_desc")
+                        scored_rows.sort(key=lambda t: 0 if t[0]["added_at"] is not None else 1)
+
+                    page_slice = scored_rows[offset : offset + page_size]
+                    rows_result = []
+                    for r, s_val, s_hits in page_slice:
+                        item = self._item_result(r, anchors_by_section, section_cache)
+                        item["score"] = s_val
+                        item["score_hint"] = True
+                        item["field_hits"] = s_hits
+                        rows_result.append(item)
+                elif self._window_functions:
                     # One statement returns the page and its pre-LIMIT count, so a
                     # batch of targets does not pay a second scan per target.
+                    order = ("(i.added_at IS NULL),i.added_at " + ("DESC" if sort_by == "added_at_desc" else "ASC")
+                             if sort_by != "default" else "i.section_id,i.preorder,i.item_id")
                     rows = self.connection.execute(
                         "SELECT i.*, count(*) OVER () AS computed_total FROM items i WHERE " + where +
-                        " ORDER BY i.section_id,i.preorder,i.item_id LIMIT ? OFFSET ?",
+                        " ORDER BY " + order + (",i.section_id,i.preorder,i.item_id" if sort_by != "default" else "") +
+                        " LIMIT ? OFFSET ?",
                         parameters + [page_size, offset]).fetchall()
                     total = rows[0]["computed_total"] if rows else self.connection.execute(
                         "SELECT count(*) FROM items i WHERE " + where, parameters).fetchone()[0]
+                    rows_result = [self._item_result(row, anchors_by_section, section_cache) for row in rows]
                 else:
                     total = self.connection.execute("SELECT count(*) FROM items i WHERE " + where, parameters).fetchone()[0]
+                    order = ("(i.added_at IS NULL),i.added_at " + ("DESC" if sort_by == "added_at_desc" else "ASC")
+                             if sort_by != "default" else "i.section_id,i.preorder,i.item_id")
                     rows = self.connection.execute(
-                        "SELECT i.* FROM items i WHERE " + where + " ORDER BY i.section_id,i.preorder,i.item_id LIMIT ? OFFSET ?",
+                        "SELECT i.* FROM items i WHERE " + where + " ORDER BY " + order +
+                        (",i.section_id,i.preorder,i.item_id" if sort_by != "default" else "") + " LIMIT ? OFFSET ?",
                         parameters + [page_size, offset]).fetchall()
+                    rows_result = [self._item_result(row, anchors_by_section, section_cache) for row in rows]
             else:
                 total = self.connection.execute("SELECT count(*) FROM items i WHERE " + where, parameters).fetchone()[0]
+                rows_result = []
             unique = self.connection.execute(
                 "SELECT count(DISTINCT i.url) FROM items i WHERE " + where + " AND i.url<>''", parameters).fetchone()[0]
             return {"total": total, "unique_urls": unique, "limit": limit, "offset": offset,
-                    "results": [self._item_result(row, anchors_by_section, section_cache) for row in rows]}
+                    "results": rows_result}
 
         result = query(targets)
+        if score:
+            result["score_mode"] = "hint"
+            result["score_candidate_pool"] = {
+                "truncated": result["total"] > max(1000, offset + page_size * 5),
+                "selection": "newest_added_at_first",
+                "candidate_limit": max(1000, offset + page_size * 5),
+            }
+            result["score_note"] = (
+                "Item scores are non-binding ranking hints based on field weights "
+                "(title_prefix: 140, title_contains: 135, url: 120, tag: 110, note: 90) "
+                "plus time confidence bonus (high: +10, medium: +5, low: +2). folder_path is inherited evidence "
+                "for folder aggregation and does not increase an item's score. Filter matching remains literal substring. "
+                "When score_candidate_pool.truncated is true, only the newest-added candidates were ranked."
+            )
         section_rows = self._sections(source_id, section) if section is not None else []
         anchors = {row["section_id"]: row["canonical_id"] for row in section_rows if row["is_anchor"]}
         result.update(source_id=source_id,
             query_scope={"section": section, "group_id": group_id, "folder_id": folder_id,
                          "tags": list(tags or []), "tag_colors": list(tag_colors or []),
-                         "item_types": list(item_types) if item_types else ["bookmark"]},
+                         "item_types": list(item_types) if item_types else ["bookmark"],
+                         "has_note": has_note, "has_tags": has_tags,
+                         "metadata_match": metadata_match, "sort_by": sort_by},
             match_mode="literal_substring", match_index="fts_trigram+like" if self._fts_mode == "trigram" else "like",
             searched_fields=["title", "url", "note", "tags.text", "folder_path"])
         result["counts"] = {"instances": result["total"], "unique_urls": result["unique_urls"],
@@ -1306,6 +1720,7 @@ class BookmarkIndex:
         return matches
 
     def context(self, source_id, section=None, group_id=None, item_id=None):
+        source_id = self.resolve_source_id(source_id)
         source = self._source(source_id)
         sections = self._sections(source_id, section)
         nodes = [dict(row) for row in self.connection.execute("SELECT * FROM nodes WHERE source_id=? ORDER BY canvas_path,position", (source_id,))]
@@ -1405,6 +1820,352 @@ class BookmarkIndex:
                 "instances": instances, "unique_urls": len(urls),
                 "note": "Section totals, not item or folder subtree totals. Sum of counted_sections only; copy anchors share another card's tree and are excluded."}
 
+    def lookup_url(self, url, source_id=None, limit=100, offset=0):
+        """Find all instances of a bookmark URL across sections and sources.
+
+        Returns matching item instances with source, section, folder path,
+        chronological added_at, and associated copy anchor cards.
+        """
+        if not isinstance(url, str) or not url.strip():
+            raise ValueError("url must be a non-empty string")
+        url = url.strip()
+        if source_id is not None:
+            _string(source_id, "source_id", True)
+            source_id = self.resolve_source_id(source_id)
+            self._source(source_id)
+        if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 1000:
+            raise ValueError("limit must be an integer between 1 and 1000")
+        if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
+            raise ValueError("offset must be a non-negative integer")
+
+        clauses = ["i.item_type = 'bookmark'", "i.url = ?"]
+        params = [url]
+        if source_id is not None:
+            clauses.append("i.source_id = ?")
+            params.append(source_id)
+        where = " AND ".join(clauses)
+
+        total = self.connection.execute(f"SELECT count(*) FROM items i WHERE {where}", params).fetchone()[0]
+        distinct_sources = [r[0] for r in self.connection.execute(
+            f"SELECT DISTINCT i.source_id FROM items i WHERE {where} ORDER BY i.source_id", params).fetchall()]
+
+        first_seen_row = self.connection.execute(
+            f"SELECT min(i.added_at) FROM items i WHERE {where} AND i.added_at IS NOT NULL", params).fetchone()
+        first_seen = first_seen_row[0] if first_seen_row and first_seen_row[0] else None
+
+        if source_id is not None:
+            anchors_by_section = self._section_anchors(source_id)
+        else:
+            anchors_by_section = {}
+            for row in self.connection.execute(
+                    "SELECT source_id, section_id, canonical_id FROM sections WHERE is_anchor=1 AND canonical_id IS NOT NULL AND canonical_id != section_id"):
+                anchors_by_section.setdefault((row["source_id"], row["canonical_id"]), []).append(row["section_id"])
+
+        order_by = "ORDER BY (i.added_at IS NULL), i.added_at ASC, i.source_id ASC, i.section_id ASC, i.preorder ASC, i.item_id ASC"
+        query_sql = f"""
+            SELECT i.source_id, i.section_id, i.item_id, i.parent_id, i.title, i.url,
+                   i.note, i.folder_path, i.path_json, i.added_at, i.added_at_source,
+                   i.added_at_confidence, s.label AS section_label, s.title AS section_title,
+                   s.file_path, s.is_anchor, s.canonical_id
+            FROM items i
+            LEFT JOIN sections s ON i.source_id = s.source_id AND i.section_id = s.section_id
+            WHERE {where}
+            {order_by}
+            LIMIT ? OFFSET ?
+        """
+        rows = self.connection.execute(query_sql, params + [limit, offset]).fetchall()
+
+        instances = []
+        for row in rows:
+            sid = row["source_id"]
+            sec_id = row["section_id"]
+            if source_id is not None:
+                anchors = anchors_by_section.get(sec_id, [])
+            else:
+                anchors = anchors_by_section.get((sid, sec_id), [])
+            try:
+                path = json.loads(row["path_json"]) if row["path_json"] else []
+            except (ValueError, TypeError):
+                path = []
+            instances.append({
+                "source_id": sid,
+                "section_id": sec_id,
+                "section_label": row["section_label"] or "",
+                "section_title": row["section_title"] or "",
+                "file_path": row["file_path"] or "",
+                "item_id": row["item_id"],
+                "parent_id": row["parent_id"],
+                "title": row["title"],
+                "url": row["url"],
+                "folder_path": row["folder_path"],
+                "path": path,
+                "added_at": row["added_at"],
+                "added_at_source": row["added_at_source"],
+                "added_at_confidence": row["added_at_confidence"],
+                "shown_in_anchors": sorted(anchors),
+            })
+
+        next_offset = (offset + len(instances)) if (offset + len(instances)) < total else None
+        return {
+            "url": url,
+            "source_filter": source_id,
+            "total_instances": total,
+            "unique_sources": len(distinct_sources),
+            "sources": distinct_sources,
+            "first_seen": first_seen,
+            "instances": instances,
+            "limit": limit,
+            "offset": offset,
+            "next_offset": next_offset,
+        }
+
+    def traverse(self, source_id, section=None, group_id=None, item_id=None, node_id=None,
+                 max_depth=2, direction="all", limit=100, offset=0):
+        """Multi-hop graph traversal starting from nodes, sections, groups, or items.
+
+        Traverses along canvas edges and group memberships up to max_depth hops.
+        Returns the impact surface: affected sections, groups, text cards,
+        and traversed paths with deterministic sorting and pagination.
+        """
+        source_id = self.resolve_source_id(source_id)
+        source = self._source(source_id)
+        if isinstance(max_depth, bool) or not isinstance(max_depth, int) or not 1 <= max_depth <= 10:
+            raise ValueError("max_depth must be an integer between 1 and 10")
+        if direction not in ("all", "out", "in"):
+            raise ValueError("direction must be 'all', 'out', or 'in'")
+        if isinstance(limit, bool) or not isinstance(limit, int) or not 1 <= limit <= 1000:
+            raise ValueError("limit must be an integer between 1 and 1000")
+        if isinstance(offset, bool) or not isinstance(offset, int) or offset < 0:
+            raise ValueError("offset must be a non-negative integer")
+
+        all_nodes = [dict(row) for row in self.connection.execute(
+            "SELECT * FROM nodes WHERE source_id=? ORDER BY canvas_path,position", (source_id,))]
+        all_edges = [dict(row) for row in self.connection.execute(
+            "SELECT * FROM edges WHERE source_id=? ORDER BY canvas_path,edge_id", (source_id,))]
+        all_memberships = [dict(row) for row in self.connection.execute(
+            "SELECT * FROM memberships WHERE source_id=? ORDER BY canvas_path,group_id,node_id", (source_id,))]
+        sections_map = {row["section_id"]: dict(row) for row in self._sections(source_id)}
+
+        for edge in all_edges:
+            edge["direction"] = ("both" if edge["from_end"] == edge["to_end"] == "arrow" else
+                                 "forward" if edge["to_end"] == "arrow" else
+                                 "reverse" if edge["from_end"] == "arrow" else "none")
+            try:
+                edge["raw_json"] = json.loads(edge["raw_json"])
+            except (ValueError, TypeError):
+                edge["raw_json"] = {}
+            edge["added_at"] = _added_at_from_edge_id(edge["edge_id"])
+
+        for node in all_nodes:
+            try:
+                node["raw_json"] = json.loads(node["raw_json"])
+            except (ValueError, TypeError):
+                node["raw_json"] = {}
+
+        node_map = {(n["canvas_path"], n["node_id"]): n for n in all_nodes}
+        start_nodes = []
+        start_scope = {}
+
+        if item_id is not None:
+            start_scope["item_id"] = item_id
+            item_rows = self.connection.execute(
+                "SELECT section_id FROM items WHERE source_id=? AND item_id=?", (source_id, item_id)).fetchall()
+            if not item_rows:
+                raise ValueError("Unknown item in requested scope: " + str(item_id))
+            item_sections = {r[0] for r in item_rows}
+            start_nodes = [n for n in all_nodes if n["section_id"] in item_sections]
+        elif group_id is not None:
+            start_scope["group_id"] = group_id
+            group = self._group(source_id, group_id)
+            start_nodes = [n for n in all_nodes if n["canvas_path"] == group["canvas_path"] and n["node_id"] == group["node_id"]]
+        elif node_id is not None:
+            start_scope["node_id"] = node_id
+            start_nodes = [n for n in all_nodes if n["node_id"] == node_id]
+        elif section is not None:
+            start_scope["section"] = section
+            matched_sections = self._sections(source_id, section)
+            if not matched_sections:
+                raise ValueError("Unknown section: " + str(section))
+            sec_ids = {r["section_id"] for r in matched_sections}
+            start_nodes = [n for n in all_nodes if n["section_id"] in sec_ids]
+            # Match get_context: include enclosing group containers in start nodes (depth 0)
+            group_ids = {m["group_id"] for m in all_memberships if any(
+                n["canvas_path"] == m["canvas_path"] and n["node_id"] == m["node_id"] for n in start_nodes)}
+            for n in all_nodes:
+                if n["node_type"] == "group" and n["node_id"] in group_ids and n not in start_nodes:
+                    start_nodes.append(n)
+        else:
+            raise ValueError("Specify at least one of section, group_id, item_id, or node_id")
+
+        if not start_nodes:
+            raise ValueError("No matching canvas nodes found for the specified start target")
+
+        adj = {}
+        for edge in all_edges:
+            cp = edge["canvas_path"]
+            u = (cp, edge["from_node"])
+            v = (cp, edge["to_node"])
+            edir = edge["direction"]
+            adj.setdefault(u, []).append(("edge", v, edge, "out", edir))
+            adj.setdefault(v, []).append(("edge", u, edge, "in", edir))
+
+        for m in all_memberships:
+            cp = m["canvas_path"]
+            grp = (cp, m["group_id"])
+            mbr = (cp, m["node_id"])
+            adj.setdefault(grp, []).append(("membership", mbr, m, "contains", None))
+            adj.setdefault(mbr, []).append(("membership", grp, m, "member_of", None))
+
+        start_keys = {(n["canvas_path"], n["node_id"]) for n in start_nodes}
+        visited = {key: {"depth": 0, "node": node_map[key], "path": []} for key in start_keys if key in node_map}
+        queue = [(key, 0, []) for key in visited]
+
+        traversed_edges = {}
+        traversed_memberships = set()
+
+        while queue:
+            curr_key, curr_d, curr_path = queue.pop(0)
+            if curr_d >= max_depth:
+                continue
+
+            for hop_type, neighbor_key, obj, rel_or_dir, edge_dir in adj.get(curr_key, []):
+                if neighbor_key not in node_map:
+                    continue
+
+                if hop_type == "edge":
+                    if direction == "out":
+                        if rel_or_dir == "out" and edge_dir not in ("forward", "both", "none"):
+                            continue
+                        if rel_or_dir == "in" and edge_dir not in ("reverse", "both", "none"):
+                            continue
+                    elif direction == "in":
+                        if rel_or_dir == "out" and edge_dir not in ("reverse", "both", "none"):
+                            continue
+                        if rel_or_dir == "in" and edge_dir not in ("forward", "both", "none"):
+                            continue
+
+                    traversed_edges[obj["edge_id"]] = obj
+                    step = {
+                        "hop": curr_d + 1,
+                        "type": "edge",
+                        "edge_id": obj["edge_id"],
+                        "label": obj.get("label", ""),
+                        "direction": obj["direction"],
+                        "from_node": obj["from_node"],
+                        "to_node": obj["to_node"],
+                    }
+                else:
+                    traversed_memberships.add((obj["canvas_path"], obj["group_id"], obj["node_id"]))
+                    step = {
+                        "hop": curr_d + 1,
+                        "type": "membership",
+                        "relation": rel_or_dir,
+                        "group_id": obj["group_id"],
+                        "node_id": obj["node_id"],
+                    }
+
+                if neighbor_key not in visited:
+                    new_path = curr_path + [step]
+                    visited[neighbor_key] = {
+                        "depth": curr_d + 1,
+                        "node": node_map[neighbor_key],
+                        "path": new_path,
+                    }
+                    queue.append((neighbor_key, curr_d + 1, new_path))
+
+        seen_affected_sections = {}
+        affected_groups = []
+        affected_text_cards = []
+        all_visited_nodes = []
+
+        max_depth_reached = max(v["depth"] for v in visited.values()) if visited else 0
+
+        sorted_nodes = sorted(
+            visited.values(),
+            key=lambda item: (item["depth"], item["node"]["canvas_path"], item["node"]["position"], item["node"]["node_id"])
+        )
+
+        for entry in sorted_nodes:
+            d = entry["depth"]
+            n = entry["node"]
+            node_repr = {
+                "depth": d,
+                "canvas_path": n["canvas_path"],
+                "node_id": n["node_id"],
+                "node_type": n["node_type"],
+                "label": n["label"],
+                "text": n["text"],
+                "section_id": n["section_id"],
+                "path": entry["path"],
+            }
+            if n["section_id"] and n["section_id"] in sections_map:
+                s = sections_map[n["section_id"]]
+                target_sec_id = s.get("canonical_id") or s["section_id"]
+                b_count = self.connection.execute(
+                    "SELECT count(*) FROM items WHERE source_id=? AND section_id=? AND item_type='bookmark'",
+                    (source_id, target_sec_id)).fetchone()[0]
+                node_repr["section"] = {
+                    "section_id": s["section_id"],
+                    "label": s["label"],
+                    "title": s["title"],
+                    "file_path": s["file_path"],
+                    "is_anchor": bool(s["is_anchor"]),
+                    "canonical_id": s["canonical_id"],
+                    "bookmark_count": b_count,
+                }
+                if d > 0:
+                    sec_id = s["section_id"]
+                    if sec_id not in seen_affected_sections:
+                        seen_affected_sections[sec_id] = dict(depth=d, **node_repr["section"])
+                    elif d < seen_affected_sections[sec_id]["depth"]:
+                        seen_affected_sections[sec_id]["depth"] = d
+
+            if d > 0:
+                if n["node_type"] == "group":
+                    affected_groups.append({"depth": d, "canvas_path": n["canvas_path"], "node_id": n["node_id"], "label": n["label"]})
+                elif n["node_type"] == "text":
+                    affected_text_cards.append({"depth": d, "canvas_path": n["canvas_path"], "node_id": n["node_id"], "text": n["text"]})
+
+            all_visited_nodes.append(node_repr)
+
+        affected_sections = list(seen_affected_sections.values())
+
+        reached_only = [n for n in all_visited_nodes if n["depth"] > 0]
+        total_reached = len(reached_only)
+        paged_nodes = reached_only[offset : offset + limit]
+        next_offset = (offset + len(paged_nodes)) if (offset + len(paged_nodes)) < total_reached else None
+
+        return {
+            "source_id": source_id,
+            "start_scope": start_scope,
+            "start_nodes": [{"node_id": n["node_id"], "canvas_path": n["canvas_path"], "node_type": n["node_type"], "label": n["label"], "section_id": n["section_id"]} for n in start_nodes],
+            "max_depth": max_depth,
+            "direction": direction,
+            "depth_reached": max_depth_reached,
+            "counts": {
+                "start_nodes": len(start_nodes),
+                "total_visited": len(visited),
+                "reached_nodes": total_reached,
+                "affected_sections": len(affected_sections),
+                "affected_groups": len(affected_groups),
+                "affected_text_cards": len(affected_text_cards),
+                "traversed_edges": len(traversed_edges),
+                "traversed_memberships": len(traversed_memberships),
+            },
+            "affected_sections": affected_sections,
+            "affected_groups": affected_groups,
+            "affected_text_cards": affected_text_cards,
+            "traversed_edges": sorted(traversed_edges.values(), key=lambda e: (e["canvas_path"], e["edge_id"])),
+            "traversed_memberships": [{"canvas_path": cp, "group_id": gid, "node_id": nid} for cp, gid, nid in sorted(traversed_memberships)],
+            "nodes": paged_nodes,
+            "pagination": {
+                "limit": limit,
+                "offset": offset,
+                "total": total_reached,
+                "next_offset": next_offset,
+            }
+        }
+
     def inventory(self, source_ids):
         """Freeze the complete indexed input, independently of query previews.
 
@@ -1415,7 +2176,7 @@ class BookmarkIndex:
         """
         if not isinstance(source_ids, list) or not source_ids:
             raise ValueError("source_ids must contain indexed package IDs")
-        identifiers = sorted(set(_string(value, "source_id", True) for value in source_ids))
+        identifiers = sorted(set(self.resolve_source_id(_string(value, "source_id", True)) for value in source_ids))
         from archive import SourceArchive
         result = {"schema_version": 1, "index_state": "last_synchronized",
                   "source_ids": identifiers, "sources": [], "files": [],
@@ -1498,6 +2259,7 @@ class BookmarkIndex:
         self.connection.execute("SAVEPOINT input_version_snapshot")
         try:
             for source_id in sorted(set(source_ids)):
+                source_id = self.resolve_source_id(source_id)
                 self._source(source_id)
                 records.extend(dict(row) for row in self.connection.execute(
                     "SELECT source_id,file_path,kind,sha256 FROM files WHERE source_id=? ORDER BY file_path", (source_id,)))
@@ -1510,6 +2272,8 @@ class BookmarkIndex:
             ids = [row[0] for row in self.connection.execute("SELECT source_id FROM sources ORDER BY source_id")]
             return {"database": str(self.db_path) if self.db_path else ":memory:", "schema_version": 1,
                 "sources": [self.status(identifier) for identifier in ids]}
+        requested_source_id = source_id
+        source_id = self.resolve_source_id(source_id)
         result = self._source(source_id)
         counts = dict(self.connection.execute("""SELECT count(*) AS items,
             coalesce(sum(item_type='bookmark'),0) AS bookmarks,
@@ -1524,6 +2288,9 @@ class BookmarkIndex:
             counts[table] = self.connection.execute("SELECT count(*) FROM " + table + " WHERE source_id=?", (source_id,)).fetchone()[0]
         result["counts"] = counts
         result.update(bookmarks=counts["bookmarks"], unique_urls=counts["unique_urls"])
+        if requested_source_id != source_id:
+            result["resolved_via_redirect"] = True
+            result["requested_source_id"] = requested_source_id
         package = Path(result["package_path"])
         result["package_exists"] = package.is_dir()
         result["retained_missing_files"] = [row[0] for row in self.connection.execute("SELECT file_path FROM files WHERE source_id=? ORDER BY file_path", (source_id,)) if not (package / row[0]).is_file()]