bookmark-research 0.6.0 → 0.7.0

package/src/cli.py

@@ -122,8 +122,8 @@ def _parser():
     configure.add_argument("--search-fallback-provider", action="append", choices=Settings.PROVIDERS)
     configure.add_argument("--search-strategy", choices=("hedged", "staggered", "sequential", "wait_all"))
     configure.add_argument("--fetch-strategy", choices=("sequential", "staggered", "hedged"))
-    configure.add_argument("--fetch-provider", choices=Settings.PROVIDERS)
-    configure.add_argument("--fetch-fallback-provider", action="append", choices=Settings.PROVIDERS)
+    configure.add_argument("--fetch-provider", choices=Settings.FETCH_PROVIDERS)
+    configure.add_argument("--fetch-fallback-provider", action="append", choices=Settings.FETCH_PROVIDERS)
     configure.add_argument("--depth", choices=("auto", "quick", "agentic", "deep"),
                            help="Default research depth; auto picks one from each request")
     configure.add_argument("--max-characters", type=int)
@@ -159,6 +159,8 @@ def _parser():
     watch.add_argument("--interval", type=float, default=1.0)
     watch.add_argument("--debounce", type=float, default=2.0)
     watch.add_argument("--deletion-grace", type=float, default=5.0)
+    watch.add_argument("--verify-hashes", action="store_true",
+                       help="Always read and verify full byte hashes instead of pre-screening by mtime and size")
     search = commands.add_parser("search", help="Literal structured bookmark search")
     search.add_argument("source")
     search.add_argument("--compact", action="store_true", help="Return shared rows with ordered page references; preserve extra fields")
@@ -171,9 +173,41 @@ def _parser():
     search.add_argument("--item-type", action="append", choices=("bookmark", "folder"), default=[],
                         help="Restrict to bookmark and/or folder rows; default bookmark, use folder to enumerate folders")
     search.add_argument("--count-only", action="store_true", help="Return counts without result rows")
+    search.add_argument("--score", action="store_true", help="Rank results by weighted score hints (title prefix, title contains, tag, note, path, url + temporal bonus)")
     search.add_argument("--limit", type=int, default=20)
     search.add_argument("--offset", type=int, default=0)
     search.add_argument("--no-refresh", action="store_true", help="Read the last synchronized index without refreshing")
+    lookup_url = commands.add_parser("lookup-url", help="Reverse index: find all instances of a bookmark URL across sections and sources")
+    lookup_url.add_argument("url")
+    lookup_url.add_argument("--source-id")
+    lookup_url.add_argument("--limit", type=int, default=100)
+    lookup_url.add_argument("--offset", type=int, default=0)
+    locate = commands.add_parser("locate", help="Locate Canvas bookmarks by text/URL or list a folder's direct children")
+    locate.add_argument("--text")
+    locate.add_argument("--url")
+    locate.add_argument("--in", dest="in_sources", action="append", choices=("canvas", "wiki", "archive"),
+                        help="Search one saved scope: Canvas bookmarks, Wiki revisions, or archive captures")
+    locate.add_argument("--source-id")
+    locate.add_argument("--section")
+    locate.add_argument("--folder-id", help="Search within a folder, or with no text/URL list its direct children")
+    locate.add_argument("--tag", dest="tags", action="append")
+    locate.add_argument("--tag-color", dest="tag_colors", action="append")
+    note_filter = locate.add_mutually_exclusive_group()
+    note_filter.add_argument("--has-note", dest="has_note", action="store_true")
+    note_filter.add_argument("--without-note", dest="has_note", action="store_false")
+    locate.set_defaults(has_note=None)
+    tags_filter = locate.add_mutually_exclusive_group()
+    tags_filter.add_argument("--has-tags", dest="has_tags", action="store_true")
+    tags_filter.add_argument("--without-tags", dest="has_tags", action="store_false")
+    locate.set_defaults(has_tags=None)
+    locate.add_argument("--metadata-match", choices=("all", "any"), default="all")
+    locate.add_argument("--limit", type=int, default=8)
+    locate.add_argument("--offset", type=int, default=0)
+    locate.add_argument("--handle", help="Continue a previous locate response")
+    refresh = locate.add_mutually_exclusive_group()
+    refresh.add_argument("--refresh", dest="refresh", action="store_true")
+    refresh.add_argument("--no-refresh", dest="refresh", action="store_false")
+    locate.set_defaults(refresh=False)
     context = commands.add_parser("context")
     context.add_argument("source")
     context.add_argument("--section")
@@ -183,6 +217,17 @@ def _parser():
     context.add_argument("--collection", choices=("sections", "items", "nodes", "groups", "edges", "related_nodes", "memberships"))
     context.add_argument("--offset", type=int, default=0)
     context.add_argument("--limit", type=int, default=200, help="Rows per context collection")
+    traverse = commands.add_parser("traverse", help="Multi-hop graph traversal along canvas edges and memberships")
+    traverse.add_argument("source")
+    traverse.add_argument("--section")
+    traverse.add_argument("--group")
+    traverse.add_argument("--item")
+    traverse.add_argument("--node")
+    traverse.add_argument("--depth", type=int, default=2)
+    traverse.add_argument("--direction", choices=("all", "out", "in"), default="all")
+    traverse.add_argument("--limit", type=int, default=100)
+    traverse.add_argument("--offset", type=int, default=0)
+    traverse.add_argument("--no-refresh", action="store_true")
     providers = commands.add_parser("providers")
     providers.add_argument("--probe", action="store_true", help="Connect to selected official MCP endpoints")
     providers.add_argument("--provider", action="append")
@@ -195,7 +240,7 @@ def _parser():
     web.add_argument("--timeout", type=int)
     fetch = commands.add_parser("fetch-web")
     fetch.add_argument("urls", nargs="+")
-    fetch.add_argument("--provider", choices=Settings.PROVIDERS)
+    fetch.add_argument("--provider", choices=Settings.FETCH_PROVIDERS)
     fetch.add_argument("--timeout", type=int, help="Per provider HTTP request timeout; discovery and fallback can take longer overall")
     fetch.add_argument("--max-characters", type=int)
     fetch.add_argument("--max-total-characters", type=int, default=None)
@@ -215,6 +260,11 @@ def _parser():
     raw.add_argument("--occurrence-offset", type=int, default=0, help="Continue a body's occurrence list here")
     merge = commands.add_parser("merge-results", help="Fuse normalized batches from existing MCPs")
     merge.add_argument("input", help="JSON input file, or '-' for stdin")
+    merge.add_argument("--archive", action="store_true", help="Save merged extracts to knowledge archive")
+    ingest = commands.add_parser("ingest", help="Fuse a search payload from a JSON file (no inline MCP size limit)")
+    ingest.add_argument("input_path", nargs="?", help="JSON input file, or '-' for stdin")
+    ingest.add_argument("--input", dest="input_option", help="JSON input file, or '-' for stdin")
+    ingest.add_argument("--archive", action="store_true", help="Save merged extracts to knowledge archive")
     research = commands.add_parser("research", help="Durable host-led deep research, evidence and reports")
     research.add_argument("--directory", help="Research root outside source packages and plugin; default in the data directory")
     research_commands = research.add_subparsers(dest="research_command", required=True)
@@ -305,7 +355,7 @@ def _parser():
     refresh = wiki_commands.add_parser("refresh", help="Capture cited webpages for review without rewriting the Wiki")
     refresh.add_argument("page_id")
     refresh.add_argument("--expected-revision", type=int)
-    refresh.add_argument("--provider", choices=Settings.PROVIDERS)
+    refresh.add_argument("--provider", choices=Settings.FETCH_PROVIDERS)
     refresh.add_argument("--max-characters", type=int)
     refresh.add_argument("--offset", type=int, default=0)
     refresh.add_argument("--limit", type=int, default=8)
@@ -317,17 +367,125 @@ def _parser():
             view.add_argument("--source-url", help="Filter direct citations and linked dependents")
         view.add_argument("--offset", type=int, default=0)
         view.add_argument("--limit", type=int, default=20)
+    dupes = commands.add_parser("dupes", help="Report bookmarks whose URL appears more than once")
+    dupes.add_argument("--source-id")
+    dupes.add_argument("--min-count", type=int, default=2)
+    dupes.add_argument("--limit", type=int, default=50)
+    dupes.add_argument("--offset", type=int, default=0)
+    dupes.add_argument("--include-search", action="store_true",
+                       help="Include search-engine result pages (excluded by default)")
+    job = commands.add_parser("job", help="Submit and observe durable local jobs")
+    job_commands = job.add_subparsers(dest="job_command", required=True)
+    submit = job_commands.add_parser("submit")
+    submit.add_argument("--kind", choices=("fetch", "ingest", "research"), required=True)
+    submit.add_argument("--input", required=True, help="Payload JSON file, or - for stdin")
+    submit.add_argument("--idempotency-key")
+    submit.add_argument("--priority", type=int, default=0)
+    listing = job_commands.add_parser("list")
+    listing.add_argument("--state", choices=("queued", "running", "succeeded", "failed", "cancelled"))
+    listing.add_argument("--kind", choices=("fetch", "ingest", "research"))
+    listing.add_argument("--limit", type=int, default=20)
+    listing.add_argument("--offset", type=int, default=0)
+    status = job_commands.add_parser("status")
+    status.add_argument("job_id")
+    result = job_commands.add_parser("result")
+    result.add_argument("job_id")
+    cancel = job_commands.add_parser("cancel")
+    cancel.add_argument("job_id")
+    worker = job_commands.add_parser("worker", help="Run the singleton job worker")
+    worker.add_argument("--once", action="store_true", help="Claim and run at most one job")
+    worker.add_argument("--max-jobs", type=int, default=None)
+    worker.add_argument("--concurrency", type=int, default=2)
+    worker.add_argument("--poll-seconds", type=float, default=0.5)
+    worker.add_argument("--lease-seconds", type=float, default=120.0)
     commands.add_parser("serve", help="Run the stdio MCP tool server")
+    bench = commands.add_parser("bench", help="Prepare reproducible host comparison artifacts")
+    bench_commands = bench.add_subparsers(dest="bench_command", required=True)
+    ladder = bench_commands.add_parser(
+        "ladder", help="Create a 15-question host capture template; host traces are recorded separately")
+    ladder.add_argument("--preset", choices=("legacy", "quick"), default="quick",
+                        help="Record which advertised tool surface the host run must use")
+    ladder.add_argument("--file", default=".bench/ladder.json", help="Template output path")
+    ladder.add_argument("--force", action="store_true", help="Replace an existing ladder file")
+    def add_output_options(command_parser):
+        command_parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON (the default format)")
+        command_parser.add_argument("--fields", help="Project comma-separated dotted fields from object results")
+        command_parser.add_argument("--out", help="Write the complete result to this file")
+        command_parser.add_argument("--quiet", action="store_true", help="With --out, print only a compact write receipt")
+        for action in command_parser._actions:
+            if isinstance(action, argparse._SubParsersAction):
+                for child in action.choices.values():
+                    add_output_options(child)
+    for command_parser in commands.choices.values():
+        add_output_options(command_parser)
     return parser
 
 
+def _project_fields(value, fields):
+    if not fields:
+        return value
+    selected = {}
+    for path in (part.strip() for part in fields.split(",")):
+        if not path:
+            continue
+        current = value
+        found = True
+        for segment in path.split("."):
+            if isinstance(current, dict) and segment in current:
+                current = current[segment]
+            else:
+                found = False
+                break
+        if found:
+            selected[path] = current
+    return selected
+
+
+def _emit_result(result, args):
+    projected = _project_fields(result, args.fields)
+    rendered = json.dumps(projected, ensure_ascii=False, indent=None if args.json else 2,
+                          separators=(",", ":") if args.json else None) + "\n"
+    if args.out:
+        destination = Path(args.out).expanduser()
+        destination.parent.mkdir(parents=True, exist_ok=True)
+        destination.write_text(rendered, encoding="utf-8")
+        if args.quiet:
+            print(json.dumps({"written": str(destination), "characters": len(rendered.rstrip("\n"))},
+                             ensure_ascii=False, separators=(",", ":") if args.json else None))
+            return
+    print(rendered, end="")
+
+
 def main(argv=None):
     from textio import utf8_stdio
     utf8_stdio()
     args = _parser().parse_args(argv)
     try:
         settings = Settings(args.config)
-        if args.command == "serve":
+        if args.command == "bench":
+            destination = Path(args.file).expanduser()
+            if destination.exists() and not args.force:
+                raise ValueError("Ladder file already exists; choose --file or pass --force: " + str(destination))
+            scripts_directory = Path(__file__).resolve().parent.parent / "scripts"
+            sys.path.insert(0, str(scripts_directory))
+            from evaluate_ladder import template as ladder_template
+            from bench_tools_list import LEGACY_TOOLS
+            from mcp_server import _legacy_compatibility_expired, release_version
+            if args.preset == "legacy" and _legacy_compatibility_expired(release_version()):
+                raise ValueError("The legacy ladder preset ended at 0.8.0")
+            surface = "legacy-25" if args.preset == "legacy" else "quick-10"
+            artifact = ladder_template()
+            artifact["baseline_id"] = "2026-11-" + args.preset
+            artifact["tool_surface"] = surface
+            artifact["required_tool_scope"] = (list(LEGACY_TOOLS) if args.preset == "legacy"
+                                                else list(Settings.QUICK_TOOLS))
+            destination.parent.mkdir(parents=True, exist_ok=True)
+            destination.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+            result = {"template": str(destination), "questions": 15, "tool_surface": surface,
+                      "runs_required": 3, "host_traces_recorded": False}
+            _emit_result(result, args)
+            return 0
+        elif args.command == "serve":
             from mcp_server import serve
             # Concurrency is read once at startup; an unreadable settings file
             # falls back to the conservative defaults instead of failing to serve.
@@ -343,7 +501,7 @@ def main(argv=None):
         if args.command == "watch":
             from source_watcher import SourceWatcher
             watcher = SourceWatcher(_database(args.db), interval=args.interval, debounce=args.debounce,
-                deletion_grace=args.deletion_grace,
+                deletion_grace=args.deletion_grace, verify_hashes=args.verify_hashes,
                 on_change=lambda value: print(json.dumps(value, ensure_ascii=False), flush=True))
             try:
                 watcher.run()
@@ -457,6 +615,48 @@ def main(argv=None):
                 result = store.search(args.query, args.offset, args.limit)
             else:
                 result = store.lint(args.page_id)
+        elif args.command == "job":
+            from jobs import JobStore, JobWorker, default_handlers
+            store = JobStore()
+            try:
+                if args.job_command == "submit":
+                    raw = sys.stdin.read() if args.input == "-" else Path(args.input).expanduser().read_text(encoding="utf-8")
+                    payload = json.loads(raw)
+                    job = store.submit(args.kind, payload, idempotency_key=args.idempotency_key,
+                                       priority=args.priority)
+                    _emit_result({"job": job}, args)
+                elif args.job_command == "list":
+                    _emit_result({"jobs": store.list(state=args.state, kind=args.kind,
+                                                     limit=args.limit, offset=args.offset)}, args)
+                elif args.job_command == "status":
+                    job = store.get(args.job_id)
+                    if job is None:
+                        raise ValueError("Unknown job: " + args.job_id)
+                    _emit_result({"job": job}, args)
+                elif args.job_command == "result":
+                    job = store.get(args.job_id)
+                    if job is None:
+                        raise ValueError("Unknown job: " + args.job_id)
+                    if job["result"] is None:
+                        _emit_result({"job_id": job["job_id"], "state": job["state"],
+                                      "note": "no result yet"}, args)
+                    else:
+                        _emit_result(job["result"], args)
+                elif args.job_command == "cancel":
+                    _emit_result({"job": store.cancel(args.job_id)}, args)
+                elif args.job_command == "worker":
+                    worker = JobWorker(store, handlers=default_handlers(),
+                                       concurrency=args.concurrency,
+                                       lease_seconds=args.lease_seconds,
+                                       poll_seconds=args.poll_seconds)
+                    if args.once:
+                        _emit_result({"processed": worker.run_once()}, args)
+                    else:
+                        worker.serve(max_jobs=args.max_jobs)
+                        _emit_result({"worker": "stopped"}, args)
+            finally:
+                store.close()
+            return 0
         elif args.command == "config":
             if args.config_command == "show":
                 result = settings.describe()
@@ -523,7 +723,12 @@ def main(argv=None):
                 else:
                     if args.offset:
                         raise ValueError("offset requires capture_id so continued reads use the same saved evidence")
-                    result = engine.fetch(args.urls, args.provider, archive=args.archive, max_characters=args.max_characters)
+                    if args.raw:
+                        result = engine._fetch(args.urls, args.provider, archive=args.archive,
+                                               max_characters=args.max_characters)
+                    else:
+                        result = engine.fetch(args.urls, args.provider, archive=args.archive,
+                                              max_characters=args.max_characters)
                     if not args.raw:
                         result = SourceArchive.compact_fetch(
                             result, max_total_characters=total, fair_share_min=fair)
@@ -540,24 +745,96 @@ def main(argv=None):
             result = RawLibrary(settings=settings).search(args.query, args.limit, args.offset, args.url,
                                                           args.occurrence_limit, args.occurrence_offset)
         elif args.command == "merge-results":
-            result = fuse_results(_read_json(args.input))
+            result = fuse_results(_read_json(args.input), archive=args.archive, settings=settings)
+        elif args.command == "ingest":
+            # The MCP primitive deliberately caps inline payloads.  A file
+            # keeps larger host search batches out of the model context while
+            # preserving the same deterministic fusion implementation.
+            if bool(args.input_path) == bool(args.input_option):
+                raise ValueError("provide exactly one ingest input path, positionally or with --input")
+            result = fuse_results(_read_json(args.input_option or args.input_path),
+                                  archive=args.archive, settings=settings)
         else:
             from source_manager import SourceManager
-            with BookmarkIndex(str(_database(args.db))) as index:
+            locate_readonly = not args.refresh if args.command == "locate" else True
+            if args.command == "locate" and args.handle:
+                from locate import continuation_refresh_requested
+                locate_readonly = not continuation_refresh_requested(args.handle, _database(args.db))
+            readonly = (args.command in ("status", "history", "lookup-url")
+                        or (args.command == "locate" and locate_readonly)
+                        or (args.command == "search" and args.no_refresh)
+                        or (args.command in ("traverse", "context") and args.no_refresh)
+                        or args.command == "dupes")
+            with BookmarkIndex(str(_database(args.db)), readonly=readonly) as index:
                 sources = SourceManager(index)
-                if args.command == "sync":
+                if args.command == "dupes":
+                    source_ids = [args.source_id] if args.source_id else sources.live_sources()
+                    if len(source_ids) != 1:
+                        raise ValueError("source_required: dupes requires --source-id when the index has zero or multiple live sources")
+                    result = index.duplicate_urls(source_ids[0], min_count=args.min_count,
+                                                  limit=args.limit, offset=args.offset,
+                                                  include_search=args.include_search)
+                elif args.command == "sync":
                     result = sources.sync(args.package, args.source_id, mode=args.mode, completeness=args.completeness)
                 elif args.command == "status":
                     result = sources.status(args.source)
                 elif args.command == "history":
                     result = sources.history(args.source, limit=args.limit, offset=args.offset)
+                elif args.command == "lookup-url":
+                    with index.read_snapshot():
+                        result = index.lookup_url(args.url, source_id=args.source_id, limit=args.limit, offset=args.offset)
+                elif args.command == "locate":
+                    from locate import locate as locate_core
+                    from locate import continuation_scope
+                    locate_scope = args.in_sources or ["canvas"]
+                    if args.handle:
+                        locate_scope = continuation_scope(args.handle, _database(args.db))
+                    wiki = None
+                    raw = None
+                    if locate_scope == ["wiki"]:
+                        from research import ResearchSessions
+                        from wiki import WikiStore
+                        sessions = ResearchSessions(settings=settings, db_path=_database(args.db))
+                        wiki = WikiStore(directory=WikiStore.selected_directory(settings), settings=settings,
+                                         research_sessions=sessions)
+                    elif locate_scope == ["archive"]:
+                        from raw_library import RawLibrary
+                        raw = RawLibrary(settings=settings)
+                    elif args.handle:
+                        # The continuation handle is opaque; provide both
+                        # adapters so the core can restore its original scope.
+                        from research import ResearchSessions
+                        from raw_library import RawLibrary
+                        from wiki import WikiStore
+                        sessions = ResearchSessions(settings=settings, db_path=_database(args.db))
+                        wiki = WikiStore(directory=WikiStore.selected_directory(settings), settings=settings,
+                                         research_sessions=sessions)
+                        raw = RawLibrary(settings=settings)
+                    result = locate_core(index, sources, {
+                        "text": args.text, "url": args.url,
+                        "in": locate_scope,
+                        "source_id": args.source_id, "section": args.section,
+                        "folder_id": args.folder_id, "tags": args.tags,
+                        "tag_colors": args.tag_colors, "has_note": args.has_note,
+                        "has_tags": args.has_tags, "metadata_match": args.metadata_match,
+                        "limit": args.limit, "offset": args.offset,
+                        "handle": args.handle, "refresh": args.refresh,
+                    }, wiki=wiki, raw=raw)
                 elif args.command == "source":
                     if args.source_command == "remove":
                         result = sources.remove(args.source_id, confirm=args.confirm)
                     else:
                         result = sources.merge(args.source_id, args.into_source_id, confirm=args.confirm)
                 else:
-                    refresh = None if args.no_refresh else sources.refresh(args.source)
+                    if args.no_refresh:
+                        refresh = None
+                    elif index.readonly:
+                        # The index degraded to read-only (shared file, read-only
+                        # mount, sandboxed host): answer from the saved snapshot
+                        # instead of failing on the refresh write.
+                        refresh = {"state": "skipped", "reason": "read-only index"}
+                    else:
+                        refresh = sources.refresh(args.source)
                     if refresh and refresh["state"] in ("error", "unavailable"):
                         raise ValueError(refresh["error"] + "; use --no-refresh to read the saved index")
                     with index.read_snapshot():
@@ -566,7 +843,12 @@ def main(argv=None):
                                                   group_id=args.group, folder_id=args.folder, tags=args.tag,
                                                   limit=args.limit, offset=args.offset, tag_colors=args.tag_color,
                                                   item_types=args.item_type or None, count_only=args.count_only,
-                                                  compact=args.compact)
+                                                  compact=args.compact, score=args.score)
+                        elif args.command == "traverse":
+                            result = index.traverse(args.source, section=args.section, group_id=args.group,
+                                                    item_id=args.item, node_id=args.node,
+                                                    max_depth=args.depth, direction=args.direction,
+                                                    limit=args.limit, offset=args.offset)
                         else:
                             from context_output import page_context
                             result = index.context(args.source, section=args.section, group_id=args.group,
@@ -575,7 +857,7 @@ def main(argv=None):
                         result["source"] = sources.source_status(args.source)
                     result["refresh"] = refresh
                     result["stored_snapshot_only"] = args.no_refresh
-        print(json.dumps(result, ensure_ascii=False, indent=2))
+        _emit_result(result, args)
         if args.command == "providers" and args.probe and result["successful_provider_count"] == 0:
             return 1
         if args.command == "search-web" and result.get("successful_provider_count") == 0:
@@ -586,6 +868,8 @@ def main(argv=None):
             return 1
         if args.command == "wiki" and args.wiki_command == "refresh" and result.get("status") == "error":
             return 1
+        if isinstance(result, dict) and result.get("status") == "partial":
+            return 3
         return 0
     except KeyboardInterrupt:
         print(json.dumps({"cancelled": True, "error": "Setup cancelled; previously saved preferences and keys are retained."}))
@@ -598,7 +882,7 @@ def main(argv=None):
             message = ("Cannot write the data directory: %s. Set BOOKMARK_RESEARCH_DATA_DIR to a writable directory "
                        "(and BOOKMARK_RESEARCH_CONFIG for settings), or fix that directory's permissions. "
                        "Run doctor for a storage check." % message)
-        print(json.dumps({"error": message, "type": type(error).__name__}, ensure_ascii=False))
+        print(json.dumps({"error": message, "type": type(error).__name__}, ensure_ascii=False), file=sys.stderr)
         return 1
 
 

package/src/credentials.py

@@ -10,7 +10,8 @@ from settings import Settings
 
 
 class Credentials:
-    NAMES = ("EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", "JINA_API_KEY", "DEEPSEEK_API_KEY", "OPENAI_API_KEY")
+    NAMES = ("EXA_API_KEY", "PARALLEL_API_KEY", "TAVILY_API_KEY", "JINA_API_KEY", "DEEPSEEK_API_KEY",
+             "PERPLEXITY_API_KEY", "XAI_API_KEY", "CUSTOM_AGENTIC_API_KEY", "OPENAI_API_KEY")
 
     def __init__(self, settings=None):
         settings = settings if settings is not None else Settings()

package/src/fetch_cache.py

@@ -0,0 +1,60 @@
+"""Small TTL cache for identical fetch batches.
+
+Repeating the same research within the window should not pay the provider twice.
+The cache stores the compact result the caller would have received, and callers
+mark reused rows so provenance stays explicit.
+"""
+import hashlib
+import json
+import sqlite3
+import time
+from pathlib import Path
+
+
+class FetchCache:
+    def __init__(self, path=None):
+        from settings import Settings
+        self.path = Path(path or (Settings.data_directory() / "fetch-cache.sqlite3")).expanduser()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.connection = sqlite3.connect(str(self.path), timeout=5.0)
+        self.connection.row_factory = sqlite3.Row
+        self.connection.execute("PRAGMA journal_mode=WAL")
+        self.connection.execute(
+            "CREATE TABLE IF NOT EXISTS fetch_cache ("
+            " cache_key TEXT PRIMARY KEY, urls TEXT NOT NULL, provider TEXT,"
+            " result TEXT NOT NULL, stored_at REAL NOT NULL, retrieved_at TEXT NOT NULL)")
+        self.connection.commit()
+
+    def close(self):
+        self.connection.close()
+
+    @staticmethod
+    def key(urls, provider, archive, max_characters):
+        payload = json.dumps({"urls": list(urls), "provider": provider,
+                              "archive": bool(archive), "max_characters": max_characters},
+                             ensure_ascii=False, sort_keys=True)
+        return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+    def get(self, cache_key, ttl_seconds):
+        if not ttl_seconds or ttl_seconds <= 0:
+            return None
+        row = self.connection.execute(
+            "SELECT result, stored_at FROM fetch_cache WHERE cache_key=?", (cache_key,)).fetchone()
+        if row is None:
+            return None
+        age = time.time() - row["stored_at"]
+        if age > ttl_seconds:
+            return None
+        try:
+            return json.loads(row["result"]), age
+        except ValueError:
+            return None
+
+    def put(self, cache_key, urls, provider, result):
+        self.connection.execute(
+            "INSERT OR REPLACE INTO fetch_cache (cache_key, urls, provider, result, stored_at, retrieved_at)"
+            " VALUES (?,?,?,?,?,?)",
+            (cache_key, json.dumps(list(urls), ensure_ascii=False), provider,
+             json.dumps(result, ensure_ascii=False, allow_nan=False),
+             time.time(), result.get("retrieved_at") or ""))
+        self.connection.commit()