@thenavidm/apple-photos-mcp-cli 1.0.0

package/src/apple_photos_mcp/search.py

@@ -0,0 +1,283 @@
+"""Ranking assets against a natural-language phrase.
+
+The useful thing about a Photos library is that Apple has already done the hard
+part: every asset carries scene labels, OCR text, an activity guess, a venue
+type and a place. What is missing is a way to ask for them in one phrase.
+
+So a query is split into words, and each word is scored against each field with
+a weight that reflects how much that field means. A word matching a person's
+name is worth far more than the same word appearing in OCR noise. Assets that
+match more of the distinct query words rank above assets that match one word
+many times, which is what stops a single spammy field from winning.
+"""
+
+from __future__ import annotations
+
+import difflib
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
+
+from .library import Asset, PhotosLibrary
+
+#: Words carrying no discriminating power. Kept short on purpose: over-filtering
+#: hurts more than it helps once field weighting is doing the real work.
+STOPWORDS = {
+    "a", "an", "and", "any", "are", "at", "be", "can", "did", "do", "does", "find",
+    "for", "from", "get", "give", "has", "have", "i", "in", "is", "it", "its", "me",
+    "my", "of", "on", "or", "our", "photo", "photos", "pic", "pics", "picture",
+    "pictures", "show", "some", "that", "the", "their", "them", "there", "they",
+    "this", "to", "was", "were", "what", "when", "where", "which", "with", "you",
+    "your",
+}
+
+#: Field weights. Tuned so an explicit human signal (a face, a place you named,
+#: a caption you wrote) always outranks an incidental one (a word Apple's OCR
+#: happened to read in the background of a receipt).
+WEIGHTS: list[tuple[str, float, Callable[[Asset], list[str]]]] = [
+    ("person", 10.0, lambda a: a.persons),
+    ("title", 8.0, lambda a: [a.title] if a.title else []),
+    ("keyword", 7.0, lambda a: a.keywords),
+    ("description", 6.0, lambda a: [a.description] if a.description else []),
+    ("album", 5.0, lambda a: a.albums),
+    ("place", 5.0, lambda a: [x for x in (a.place, a.city, a.state, a.country) if x]),
+    ("label", 4.0, lambda a: a.labels),
+    ("activity", 3.5, lambda a: a.activities),
+    ("venue", 3.0, lambda a: a.venues),
+    ("filename", 2.0, lambda a: [a.filename]),
+    ("text", 1.2, lambda a: a.text),
+]
+
+
+@dataclass
+class Filters:
+    """Structured narrowing applied before ranking."""
+
+    kind: str | None = None           # "photo" | "video"
+    favorite: bool | None = None
+    person: str | None = None
+    album: str | None = None
+    place: str | None = None
+    year: int | None = None
+    date_from: str | None = None      # ISO date, inclusive
+    date_to: str | None = None        # ISO date, inclusive
+    screenshots: bool | None = None   # None = include, False = exclude, True = only
+    include_hidden: bool = False
+    downloaded_only: bool = False
+
+    def keep(self, a: Asset) -> bool:
+        if a.hidden and not self.include_hidden:
+            return False
+        if self.kind == "photo" and a.is_video:
+            return False
+        if self.kind == "video" and not a.is_video:
+            return False
+        if self.favorite is not None and a.favorite is not self.favorite:
+            return False
+        if self.screenshots is True and not a.screenshot:
+            return False
+        if self.screenshots is False and a.screenshot:
+            return False
+        if self.downloaded_only and not a.local:
+            return False
+        if self.person:
+            needle = self.person.lower()
+            if not any(needle in p.lower() for p in a.persons):
+                return False
+        if self.album:
+            needle = self.album.lower()
+            if not any(needle in al.lower() for al in a.albums):
+                return False
+        if self.place:
+            needle = self.place.lower()
+            hay = " ".join(x for x in (a.place, a.city, a.state, a.country) if x).lower()
+            if needle not in hay:
+                return False
+        if self.year is not None and (not a.date or not a.date.startswith(str(self.year))):
+            return False
+        if self.date_from and (not a.date or a.date[:10] < self.date_from):
+            return False
+        if self.date_to and (not a.date or a.date[:10] > self.date_to):
+            return False
+        return True
+
+
+def tokenize(query: str) -> list[str]:
+    words = re.findall(r"[a-z0-9']+", query.lower())
+    kept = [w for w in words if w not in STOPWORDS and len(w) > 1]
+    # A query made entirely of stopwords ("show me the photos") should still
+    # return the filtered set rather than nothing at all.
+    return kept
+
+
+def _field_score(word: str, values: list[str], weight: float) -> float:
+    """Exact token match scores full weight; a prefix match scores half.
+
+    Substring-anywhere matching is deliberately not used: it makes "art" match
+    "heart" and "Sparta", which is the single fastest way to make a search feel
+    broken.
+    """
+    best = 0.0
+    for value in values:
+        if not value:
+            continue
+        for token in re.findall(r"[a-z0-9']+", value.lower()):
+            if token == word:
+                return weight
+            if len(word) > 3 and token.startswith(word):
+                best = max(best, weight * 0.5)
+    return best
+
+
+#: Words that mean the user is deliberately after screenshots, documents or
+#: text, rather than a photograph of something.
+TEXTUAL_INTENT = {
+    "screenshot", "screenshots", "screen", "document", "documents", "receipt",
+    "receipts", "invoice", "invoices", "ticket", "tickets", "text", "note",
+    "notes", "email", "message", "chat", "page", "pdf", "form", "passport",
+    "id", "card", "whiteboard", "slide", "slides", "quote", "tweet", "post",
+}
+
+
+def score(asset: Asset, words: list[str]) -> tuple[float, list[str]]:
+    """Return the asset's score and which fields earned it."""
+    total = 0.0
+    matched_words = 0
+    why: set[str] = set()
+    text_only = True
+    for word in words:
+        best = 0.0
+        best_field = ""
+        for field_name, weight, getter in WEIGHTS:
+            s = _field_score(word, getter(asset), weight)
+            if s > best:
+                best, best_field = s, field_name
+        if best > 0:
+            total += best
+            matched_words += 1
+            why.add(best_field)
+            if best_field != "text":
+                text_only = False
+    if not matched_words:
+        return 0.0, []
+
+    # Covering more of the query is worth more than scoring high on one word,
+    # but only mildly, a strong bonus per extra word lets a screenshot full of
+    # OCR noise beat a photograph that actually shows the thing asked for.
+    coverage = matched_words / len(words)
+    total *= 0.4 + 0.6 * coverage
+    total *= 1 + 0.2 * (matched_words - 1)
+
+    wants_text = bool(TEXTUAL_INTENT & set(words))
+
+    # A screenshot is rarely what "find my photo of X" means. Unless the query
+    # itself is textual, it competes at a discount.
+    if asset.screenshot and not wants_text:
+        total *= 0.45
+
+    # Matching purely on words Apple's OCR read is the weakest evidence there
+    # is: it means the phrase appeared *written inside* the image, not that the
+    # image depicts it. Only trust it when the query was asking about text.
+    if text_only and not wants_text:
+        total *= 0.35
+
+    return total, sorted(why)
+
+
+def search(
+    lib: PhotosLibrary,
+    query: str = "",
+    filters: Filters | None = None,
+    limit: int = 12,
+) -> dict[str, Any]:
+    filters = filters or Filters()
+    words = tokenize(query)
+    candidates = [a for a in lib.assets if filters.keep(a)]
+
+    if not words:
+        # Pure browse: newest first is the only ordering that makes sense.
+        ranked = sorted(candidates, key=lambda a: a.date or "", reverse=True)
+        rows = [a.summary() for a in ranked[:limit]]
+        return {
+            "query": query,
+            "matched": len(candidates),
+            "returned": len(rows),
+            "results": rows,
+            "note": "No search terms, showing the most recent matches for the filters.",
+        }
+
+    scored: list[tuple[float, list[str], Asset]] = []
+    for a in candidates:
+        s, why = score(a, words)
+        if s > 0:
+            scored.append((s, why, a))
+    # Highest score first; newest first among ties.
+    scored.sort(key=lambda t: (t[0], t[2].date or ""), reverse=True)
+
+    rows = []
+    for _score, why, a in scored[:limit]:
+        row = a.summary()
+        row["matched_on"] = why
+        rows.append(row)
+
+    out: dict[str, Any] = {
+        "query": query,
+        "searched": len(candidates),
+        "matched": len(scored),
+        "returned": len(rows),
+        "results": rows,
+    }
+
+    # Apple's label vocabulary is a fixed set of about 1,500 words that its
+    # on-device classifier was trained on. "smiling", "cosy" and "aesthetic" are
+    # not in it, and no amount of rephrasing the same idea will find them. Words
+    # that matched nothing are far more useful to report than a bare zero, so
+    # every unmatched word gets checked against the vocabulary that does exist.
+    unmatched = [w for w in words if not _understood(w, lib)]
+    if unmatched:
+        suggestions = suggest_terms(lib, unmatched)
+        if suggestions:
+            out["did_you_mean"] = suggestions
+        out["unmatched_terms"] = unmatched
+
+    if not rows:
+        out["hint"] = (
+            "Nothing matched. Apple indexes what a photo looks like, not what you "
+            "call it, try a scene word ('sunset', 'document', 'beach'), a place, a "
+            "person's name, or a word that would literally appear inside the image. "
+            "Call list_vocabulary to see the words this library actually knows."
+        )
+    return out
+
+
+def _understood(word: str, lib: PhotosLibrary) -> bool:
+    """Does this library have a *visual* or *structured* meaning for the word?
+
+    Appearing in OCR text does not count. "smiling" turns up inside the text of
+    some screenshot in almost any library, and treating that as understanding is
+    exactly how a search convinces someone it looked when it did not.
+    """
+    if word in lib.structured_terms():
+        return True
+    return any(
+        word == v or v.startswith(word + " ") or (" " + word) in v
+        for v in lib.vocabulary()
+    )
+
+
+def suggest_terms(lib: PhotosLibrary, words: list[str], per_word: int = 4) -> dict[str, list[str]]:
+    """For each word Apple has never heard of, offer the closest words it has.
+
+    This is what turns a dead end into a next step: the agent learns that this
+    library knows "Crowd" and "Audience" but not "keynote", and can ask again.
+    """
+    vocab = lib.vocabulary()
+    out: dict[str, list[str]] = {}
+    for word in words:
+        near = difflib.get_close_matches(word, vocab, n=per_word, cutoff=0.72)
+        prefix = [v for v in vocab if v.startswith(word) and v not in near][:per_word]
+        combined = (near + prefix)[:per_word]
+        if combined:
+            out[word] = combined
+    return out

package/src/apple_photos_mcp/server.py

@@ -0,0 +1,352 @@
+"""The MCP server: tool definitions and their wiring.
+
+The tool surface is deliberately small. A large surface reads well in a table
+and behaves badly in practice, because the model has to guess which of five
+similar tools it wants. What matters here is that `search_photos` is genuinely
+good and that `look_at_photos` exists, so the model can check its own answer
+before giving it.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+from typing import Any
+
+# The Python MCP SDK renamed FastMCP to MCPServer in 2.0. Both are supported so
+# the package keeps working whichever version a user's environment resolves.
+try:  # SDK >= 2.0
+    from mcp.server.mcpserver import Image
+    from mcp.server.mcpserver import MCPServer as _Server
+except ModuleNotFoundError:  # SDK 1.x
+    from mcp.server.fastmcp import FastMCP as _Server  # type: ignore[assignment]
+    from mcp.server.fastmcp import Image  # type: ignore[assignment]
+
+from . import doctor as doctor_mod
+from .config import Config
+from .library import PhotosLibrary
+from .previews import PreviewRenderer
+from .safety import DESTRUCTIVE_WRITE, READ_ONLY_TOOL, REVERSIBLE_WRITE
+from .search import Filters
+from .search import search as run_search
+from .writes import Writer
+
+log = logging.getLogger(__name__)
+
+INSTRUCTIONS = """\
+This is the user's own Apple Photos library, read directly on this Mac. Nothing is
+uploaded anywhere.
+
+How to actually find something:
+
+1. `search_photos` first. It searches Apple's own on-device index, what a photo
+   looks like (scene labels), text Apple read inside it, the activity, the venue,
+   the place, and any faces the user has named.
+2. Then `look_at_photos` on the top few results. Search returns candidates, not
+   answers, and the filenames tell you nothing. Look before you answer.
+3. Only then reply, naming the file and the date so the user can find it.
+
+Two things will save you from confident wrong answers:
+
+* Apple's visual vocabulary is closed, about 1,500 words. If a result carries
+  `unmatched_terms`, Apple has never heard of that word and no rephrasing of the
+  same idea will help. Read `did_you_mean`, or call `list_vocabulary`.
+* Almost every asset in a modern library lives in iCloud, not on the Mac.
+  Previews still work. Exporting an original downloads it first, which is slow.
+
+Nothing here can delete a photo. macOS does not permit it. `archive_photos` moves
+items into an album for the user to empty by hand, and it is the one tool that
+asks for `confirm: true` before it acts.
+"""
+
+
+def build_server(config: Config | None = None) -> _Server:
+    config = config or Config.from_env()
+    lib = PhotosLibrary(config)
+    previews = PreviewRenderer(config, lib)
+    writer = Writer(config, lib)
+
+    mcp = _Server("apple-photos", instructions=INSTRUCTIONS)
+
+    def _err(exc: Exception) -> str:
+        return json.dumps({"error": str(exc)}, indent=2)
+
+    def _json(data: Any) -> str:
+        return json.dumps(data, indent=2, default=str)
+
+    # ------------------------------------------------------------------ reads
+
+    @mcp.tool(annotations=READ_ONLY_TOOL)
+    def search_photos(
+        query: str = "",
+        limit: int = 12,
+        kind: str | None = None,
+        person: str | None = None,
+        album: str | None = None,
+        place: str | None = None,
+        year: int | None = None,
+        date_from: str | None = None,
+        date_to: str | None = None,
+        favorites_only: bool = False,
+        screenshots: str = "include",
+        include_hidden: bool = False,
+    ) -> str:
+        """Search your entire Apple Photos library using Apple's own on-device index.
+
+        Natural visual phrases work best: 'sunset beach', 'receipt', 'dinner in
+        Stockholm', 'whiteboard'. Filters narrow before ranking.
+
+        Args:
+            query: What to find. Leave empty to browse by filter alone.
+            limit: Maximum results (1-100).
+            kind: 'photo' or 'video'.
+            person: Only photos with this named face.
+            album: Only photos in albums whose name contains this.
+            place: Only photos taken somewhere matching this.
+            year: Only photos from this calendar year.
+            date_from: ISO date, inclusive lower bound (YYYY-MM-DD).
+            date_to: ISO date, inclusive upper bound (YYYY-MM-DD).
+            favorites_only: Only items hearted in Photos.
+            screenshots: 'include', 'exclude', or 'only'.
+            include_hidden: Include items in the Hidden album.
+        """
+        try:
+            filters = Filters(
+                kind=kind if kind in ("photo", "video") else None,
+                favorite=True if favorites_only else None,
+                person=person,
+                album=album,
+                place=place,
+                year=year,
+                date_from=date_from,
+                date_to=date_to,
+                screenshots={"include": None, "exclude": False, "only": True}.get(screenshots),
+                include_hidden=include_hidden,
+            )
+            return _json(run_search(lib, query, filters, max(1, min(limit, 100))))
+        except Exception as exc:
+            log.exception("search failed")
+            return _err(exc)
+
+    @mcp.tool(annotations=READ_ONLY_TOOL)
+    def look_at_photos(refs: list[str], size: int = 640) -> list[str | Image]:
+        """Actually look at photos: renders each one and returns it as an image.
+
+        Use this on the top few results of every search before answering a "find
+        my photo of X" question. Works even for photos that live only in iCloud,
+        because it reads Apple's own local thumbnails rather than the original.
+
+        Args:
+            refs: Photo uuids from search results, or exact filenames.
+            size: Longest edge in pixels (128-2048).
+        """
+        out: list[str | Image] = []
+        size = max(128, min(size, 2048))
+        for ref in refs[: config.preview_max]:
+            asset = lib.get(ref)
+            if asset is None:
+                out.append(f"{ref}: not found in this library")
+                continue
+            preview = previews.render(asset, px=size)
+            caption = f"{asset.filename} · {(asset.date or '')[:10]} · uuid {asset.uuid}"
+            if asset.place:
+                caption += f" · {asset.place}"
+            out.append(caption)
+            if preview.ok and preview.path is not None:
+                out.append(Image(path=str(preview.path)))
+            else:
+                out.append(f"  (no preview: {preview.error})")
+        if len(refs) > config.preview_max:
+            out.append(
+                f"Only the first {config.preview_max} of {len(refs)} were rendered. "
+                f"Call again for the rest."
+            )
+        return out
+
+    @mcp.tool(annotations=READ_ONLY_TOOL)
+    def photo_info(refs: list[str]) -> str:
+        """Everything known about specific photos: metadata, ML labels, place,
+        albums, faces, and any text Apple read inside the image.
+
+        Args:
+            refs: Photo uuids or exact filenames.
+        """
+        try:
+            rows = []
+            for ref in refs[:50]:
+                asset = lib.get(ref)
+                rows.append(
+                    asset.details() if asset else {"ref": ref, "error": "not found"}
+                )
+            return _json({"count": len(rows), "photos": rows})
+        except Exception as exc:
+            return _err(exc)
+
+    @mcp.tool(annotations=READ_ONLY_TOOL)
+    def library_stats() -> str:
+        """Size and shape of the library: totals, albums, named people, how much
+        is indexed, and how much lives only in iCloud."""
+        try:
+            return _json(lib.stats())
+        except Exception as exc:
+            return _err(exc)
+
+    @mcp.tool(annotations=READ_ONLY_TOOL)
+    def list_vocabulary(starts_with: str = "", limit: int = 200) -> str:
+        """The visual words this library actually knows.
+
+        Apple's classifier has a closed vocabulary. When a search finds nothing,
+        this is how to discover the word it does understand instead.
+
+        Args:
+            starts_with: Only terms beginning with this prefix.
+            limit: Maximum terms to return.
+        """
+        try:
+            counts = lib.label_counts()
+            terms = lib.vocabulary()
+            if starts_with:
+                terms = [t for t in terms if t.startswith(starts_with.lower())]
+            ranked = sorted(terms, key=lambda t: -counts.get(t.title(), 0))[:limit]
+            return _json(
+                {
+                    "total_terms": len(lib.vocabulary()),
+                    "returned": len(ranked),
+                    "terms": ranked,
+                }
+            )
+        except Exception as exc:
+            return _err(exc)
+
+    @mcp.tool(annotations=REVERSIBLE_WRITE)
+    def export_originals(refs: list[str], directory: str | None = None) -> str:
+        """Export full-quality originals to a folder on this Mac.
+
+        Slow for iCloud-only assets: each one is downloaded first. Nothing is
+        uploaded anywhere.
+
+        Args:
+            refs: Photo uuids or exact filenames.
+            directory: Absolute destination folder. Defaults to ~/Downloads/Photos Exports.
+        """
+        try:
+            import osxphotos
+
+            dest = Path(directory).expanduser() if directory else config.export_dir
+            dest.mkdir(parents=True, exist_ok=True)
+            uuids = []
+            missing = []
+            for ref in refs[: config.write_batch_max]:
+                asset = lib.get(ref)
+                (uuids.append(asset.uuid) if asset else missing.append(ref))
+            if not uuids:
+                return _json({"error": "none of those refs resolved", "unresolved": missing})
+
+            db = osxphotos.PhotosDB(str(lib._library_path()))
+            written: list[str] = []
+            failed: list[dict[str, str]] = []
+            for photo in db.photos(uuid=uuids):
+                try:
+                    written.extend(photo.export(str(dest), use_photos_export=not photo.path))
+                except Exception as exc:
+                    failed.append({"uuid": photo.uuid, "reason": str(exc)})
+            return _json(
+                {
+                    "directory": str(dest),
+                    "exported": len(written),
+                    "files": written,
+                    "failed": failed,
+                    "unresolved": missing,
+                }
+            )
+        except Exception as exc:
+            log.exception("export failed")
+            return _err(exc)
+
+    @mcp.tool(annotations=READ_ONLY_TOOL)
+    def doctor() -> str:
+        """Diagnose setup: macOS, Full Disk Access, the library, the index, and
+        whether writes are enabled. Run this first when anything misbehaves."""
+        try:
+            return _json(doctor_mod.run(config, lib))
+        except Exception as exc:
+            return _err(exc)
+
+    # ----------------------------------------------------------------- writes
+    #
+    # In read-only mode these are never registered, so they do not appear in the
+    # tool list at all. A model cannot call a tool it cannot see, and an error
+    # message is an invitation to retry differently.
+    if config.read_only:
+        return mcp
+
+    @mcp.tool(annotations=REVERSIBLE_WRITE)
+    def favorite_photos(refs: list[str], favorite: bool = True) -> str:
+        """Heart photos in Photos, or remove the heart.
+
+        Args:
+            refs: Photo uuids or exact filenames.
+            favorite: False to un-favorite.
+        """
+        try:
+            return _json(writer.set_favorite(refs, favorite))
+        except Exception as exc:
+            return _err(exc)
+
+    @mcp.tool(annotations=REVERSIBLE_WRITE)
+    def set_photo_title(ref: str, title: str) -> str:
+        """Set one photo's title."""
+        try:
+            return _json(writer.set_title(ref, title))
+        except Exception as exc:
+            return _err(exc)
+
+    @mcp.tool(annotations=REVERSIBLE_WRITE)
+    def set_photo_description(ref: str, description: str) -> str:
+        """Set one photo's description/caption."""
+        try:
+            return _json(writer.set_description(ref, description))
+        except Exception as exc:
+            return _err(exc)
+
+    @mcp.tool(annotations=REVERSIBLE_WRITE)
+    def add_keywords(refs: list[str], keywords: list[str]) -> str:
+        """Add keywords to photos, keeping the ones already there."""
+        try:
+            return _json(writer.add_keywords(refs, keywords))
+        except Exception as exc:
+            return _err(exc)
+
+    @mcp.tool(annotations=REVERSIBLE_WRITE)
+    def add_to_album(album: str, refs: list[str]) -> str:
+        """Add photos to an album, creating it if needed.
+
+        Args:
+            album: Album name.
+            refs: Photo uuids or exact filenames.
+        """
+        try:
+            return _json(writer.add_to_album(album, refs))
+        except Exception as exc:
+            return _err(exc)
+
+    @mcp.tool(annotations=DESTRUCTIVE_WRITE)
+    def archive_photos(refs: list[str], confirm: bool = False) -> str:
+        """The closest thing to deleting that macOS allows.
+
+        Moves photos into an archive album so they leave the main library view.
+        No app, including this one, can permanently delete photos by script, so
+        the user empties that album by hand.
+
+        Args:
+            refs: Photo uuids or exact filenames.
+            confirm: The user reads this as deleting their photos. Set true only
+                when they have actually asked for these specific items to go.
+        """
+        try:
+            return _json(writer.archive(refs, confirm=confirm))
+        except Exception as exc:
+            return _err(exc)
+
+    return mcp