bookmark-research 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/.agents/plugins/marketplace.json +1 -1
  2. package/.codex-plugin/plugin.json +4 -1
  3. package/README.md +15 -9
  4. package/README.zh.md +16 -9
  5. package/config/providers.json +26 -0
  6. package/config/settings.example.json +36 -0
  7. package/docs/bookmark-research-architecture.md +2 -2
  8. package/docs/canvas-tree-reading-acceptance.md +98 -0
  9. package/docs/configuration.en.md +70 -0
  10. package/docs/configuration.md +70 -0
  11. package/docs/details.en.md +10 -6
  12. package/docs/details.md +3 -3
  13. package/docs/harness-compatibility.md +1 -1
  14. package/docs/installation.en.md +4 -2
  15. package/docs/installation.md +4 -2
  16. package/docs/instructions.en.md +1 -0
  17. package/docs/instructions.md +1 -0
  18. package/docs/user-guide.en.md +1 -1
  19. package/docs/user-guide.md +1 -1
  20. package/hosts/dsh/plugin.js +3 -0
  21. package/package.json +1 -1
  22. package/scripts/bench_sync.py +191 -0
  23. package/scripts/bench_tools_list.py +129 -0
  24. package/scripts/evaluate_ladder.py +149 -0
  25. package/scripts/export_bundle.py +2 -0
  26. package/scripts/host_install.py +4 -1
  27. package/scripts/install.py +120 -28
  28. package/skills/bookmark-research/SKILL.md +28 -9
  29. package/skills/bookmark-research/references/package-semantics.md +56 -6
  30. package/skills/bookmark-research/references/research-workflow.md +5 -2
  31. package/skills/bookmark-research/references/settings-and-archive.md +5 -3
  32. package/skills/bookmark-research/references/wiki.md +3 -1
  33. package/skills/bookmark-research/references/zh/package-semantics.md +56 -6
  34. package/skills/bookmark-research/references/zh/research-workflow.md +5 -2
  35. package/skills/bookmark-research/references/zh/settings-and-archive.md +5 -3
  36. package/skills/bookmark-research/references/zh/skill-guide.md +20 -8
  37. package/skills/bookmark-research/references/zh/wiki.md +3 -1
  38. package/src/archive.py +100 -0
  39. package/src/bookmark_index.py +808 -41
  40. package/src/browser_client.py +342 -0
  41. package/src/cli.py +298 -14
  42. package/src/credentials.py +2 -1
  43. package/src/fetch_cache.py +60 -0
  44. package/src/grok_search.py +272 -0
  45. package/src/jobs.py +349 -0
  46. package/src/junk.py +33 -0
  47. package/src/locate.py +862 -0
  48. package/src/mcp_dispatch.py +22 -6
  49. package/src/mcp_server.py +573 -23
  50. package/src/netscape.py +124 -0
  51. package/src/onboarding.py +59 -9
  52. package/src/perplexity.py +78 -0
  53. package/src/raw_library.py +183 -45
  54. package/src/readiness.py +40 -5
  55. package/src/ref_codec.py +70 -0
  56. package/src/research.py +125 -12
  57. package/src/research_plan.py +11 -1
  58. package/src/routing.py +5 -2
  59. package/src/search_results.py +80 -2
  60. package/src/settings.py +61 -16
  61. package/src/source_inputs.py +109 -3
  62. package/src/source_manager.py +71 -12
  63. package/src/source_versions.py +20 -2
  64. package/src/source_watcher.py +4 -2
  65. package/src/storage.py +60 -0
  66. package/src/url_classify.py +45 -0
  67. package/src/web_search.py +106 -10
  68. package/src/wiki.py +269 -18
  69. package/src/wiki_search.py +119 -27
package/src/web_search.py CHANGED
@@ -4,6 +4,7 @@ import concurrent.futures
4
4
  import copy
5
5
  import json
6
6
  import re
7
+ import sqlite3
7
8
  import threading
8
9
  import time
9
10
  import uuid
@@ -116,6 +117,13 @@ class SearchProviders:
116
117
  state.client = DeepSeekClient(headers=headers, timeout=timeout,
117
118
  search_url=info.get("search_url"),
118
119
  fetch_url=info.get("fetch_url"))
120
+ elif info.get("client") == "perplexity":
121
+ from perplexity import PerplexityClient
122
+ state.client = PerplexityClient(headers=headers, timeout=timeout,
123
+ search_url=info.get("search_url"))
124
+ elif info.get("client") == "browser":
125
+ from browser_client import BrowserClient
126
+ state.client = BrowserClient(settings=self.settings, timeout=timeout)
119
127
  else:
120
128
  from jina import JinaClient
121
129
  state.client = JinaClient(headers=headers, timeout=timeout)
@@ -201,10 +209,21 @@ class SearchProviders:
201
209
  client, before, usage = None, None, self._empty_usage()
202
210
  try:
203
211
  if self.registry["providers"][provider].get("transport") == "http":
212
+ if provider == "browser":
213
+ enabled = self.settings.load().get("browser", {}).get("enabled", False)
214
+ capabilities = {"search": {"status": "unsupported", "execution_verified": False},
215
+ "fetch": {"status": "supported" if enabled else "disabled", "execution_verified": False}}
216
+ return {"provider": provider, "status": "ok" if enabled else "not_configured", "tools": ["browser_render"] if enabled else [],
217
+ "network_probe_performed": False, "catalog_source": "local_browser_driver",
218
+ "capabilities": capabilities,
219
+ "authentication_state": {"configured": enabled, "credential_configured": enabled, "credential_validity_verified": False, "oauth_implemented": False},
220
+ "usage": usage}
221
+ capabilities = {purpose: {"status": "supported" if self.registry["providers"][provider].get(purpose + "_tools") else "unsupported",
222
+ "execution_verified": False}
223
+ for purpose in ("search", "fetch")}
204
224
  return {"provider": provider, "status": "not_probed", "tools": [],
205
225
  "network_probe_performed": False, "catalog_source": "documented_http_api",
206
- "capabilities": {purpose: {"status": "supported", "execution_verified": False}
207
- for purpose in ("search", "fetch")},
226
+ "capabilities": capabilities,
208
227
  "authentication_state": self._authentication(provider), "usage": usage}
209
228
  client = self._client(provider)
210
229
  before = self._snapshot(client)
@@ -373,6 +392,8 @@ class SearchProviders:
373
392
  skipped = True
374
393
  raise state.cooldown_error
375
394
  if self.registry["providers"][provider].get("transport") == "http":
395
+ if not self.registry["providers"][provider].get(purpose + "_tools", []):
396
+ raise McpError("Provider does not support " + purpose, "capability_unavailable")
376
397
  tool = {"name": self.registry["providers"][provider][purpose + "_tools"][0]}
377
398
  arguments = {"query": query} if purpose == "search" else {"urls": urls}
378
399
  if purpose == "search":
@@ -871,7 +892,9 @@ class SearchProviders:
871
892
  attempts, coverage, ranks = [], {}, {}
872
893
 
873
894
  def quality(row):
874
- return (row["status"] == "ok", row.get("characters", 0) > 0,
895
+ usable = row.get("content_state") not in ("js_shell_suspected", "challenge_blocked", "unavailable")
896
+ return (row["status"] == "ok" and usable,
897
+ row["status"] == "ok", row.get("characters", 0) > 0,
875
898
  row["extraction_status"] not in ("request_failed", "unrecognized_or_not_returned",
876
899
  "no_content_returned", "content_present_unparsed"))
877
900
 
@@ -899,11 +922,18 @@ class SearchProviders:
899
922
  coverage[row["url"]] = {**row, "provider": fetched["provider"]}
900
923
  ranks[row["url"]] = rank
901
924
 
902
- def unresolved():
903
- return [url for url in urls if coverage.get(url, {}).get("status") != "ok"]
925
+ def unresolved(remaining_providers=()):
926
+ def needs_work(u):
927
+ current = coverage.get(u)
928
+ if not current or current.get("status") != "ok":
929
+ return True
930
+ if current.get("content_state") in ("js_shell_suspected", "challenge_blocked") and any(p == "browser" for p in remaining_providers):
931
+ return True
932
+ return False
933
+ return [url for url in urls if needs_work(url)]
904
934
 
905
935
  def assemble(abandoned=()):
906
- pending = unresolved()
936
+ pending = [url for url in urls if coverage.get(url, {}).get("status") != "ok"]
907
937
  successful = len(urls) - len(pending)
908
938
  return {"mode": "waterfall", "urls": urls, "providers": providers, "attempts": attempts,
909
939
  "status": "ok" if not pending else "partial" if successful else "error",
@@ -918,7 +948,7 @@ class SearchProviders:
918
948
 
919
949
  if strategy == "sequential":
920
950
  for stage, names in enumerate((providers[:1], providers[1:])):
921
- pending = [url for url in urls if coverage.get(url, {}).get("status") != "ok"]
951
+ pending = unresolved(names)
922
952
  if not pending or not names:
923
953
  break
924
954
  if stage and reserve_fallback is not None:
@@ -1003,15 +1033,15 @@ class SearchProviders:
1003
1033
  # One provider finished with work left: reserve and hand the
1004
1034
  # pending URLs to every remaining provider at once, exactly
1005
1035
  # like the sequential fallback stage.
1006
- pending = unresolved()
1007
1036
  names = queue[:]
1037
+ pending = unresolved(names)
1008
1038
  if names and reserve_fallback is not None:
1009
1039
  names = list(reserve_fallback(names, pending))
1010
1040
  queue[:] = []
1011
1041
  start(names, pending)
1012
1042
  elif queue and last_dispatch is not None and time.monotonic() >= last_dispatch + grace:
1013
- pending = unresolved()
1014
1043
  names = [queue.pop(0)]
1044
+ pending = unresolved(names)
1015
1045
  if reserve_fallback is not None:
1016
1046
  names = list(reserve_fallback(names, pending))
1017
1047
  start(names, pending)
@@ -1025,7 +1055,73 @@ class SearchProviders:
1025
1055
  pool.shutdown(wait=not abandoned)
1026
1056
  return assemble(abandoned)
1027
1057
 
1028
- def fetch(self, urls, provider=None, archive=None, max_characters=None, probe=True):
1058
+ def fetch(self, urls, provider=None, archive=None, max_characters=None, probe=True, use_cache=True):
1059
+ """Route known URLs: skip search-result pages, reuse a fresh cache hit, else fetch."""
1060
+ from url_classify import is_search_result_url
1061
+ preferences = self.settings.load()
1062
+ if not isinstance(urls, list):
1063
+ return self._fetch(urls, provider, archive, max_characters, probe)
1064
+ skipped = [url for url in urls if isinstance(url, str) and is_search_result_url(url)]
1065
+ fetchable = [url for url in urls if url not in skipped]
1066
+ skipped_rows = [{"url": url, "status": "skipped", "content_state": "unavailable",
1067
+ "extraction_status": "search_result_page", "characters": 0,
1068
+ "complete_page_verified": False,
1069
+ "error": "search-engine result page; search again instead of fetching it"}
1070
+ for url in skipped]
1071
+ ttl = (preferences["fetch"].get("reuse_seconds") or 0) if use_cache else 0
1072
+ cache = None
1073
+ cache_key = None
1074
+ if ttl > 0 and fetchable:
1075
+ try:
1076
+ from fetch_cache import FetchCache
1077
+ cache = FetchCache()
1078
+ effective_archive = (preferences["archive"]["enabled"]
1079
+ if archive is None else archive)
1080
+ effective_max = (preferences["fetch"]["max_characters"]
1081
+ if max_characters is None else max_characters)
1082
+ cache_key = FetchCache.key(fetchable, provider, effective_archive, effective_max)
1083
+ hit = cache.get(cache_key, ttl)
1084
+ except (OSError, ValueError, sqlite3.Error):
1085
+ hit = None
1086
+ if hit is not None:
1087
+ cached, age = hit
1088
+ cached = dict(cached)
1089
+ cached["cached"] = True
1090
+ cached["cache_age_seconds"] = round(age, 1)
1091
+ if skipped_rows:
1092
+ cached["per_url"] = list(cached.get("per_url") or []) + skipped_rows
1093
+ cached["skipped_url_count"] = len(skipped_rows)
1094
+ if cache is not None:
1095
+ cache.close()
1096
+ return cached
1097
+ try:
1098
+ if not fetchable and skipped_rows:
1099
+ return {"mode": "skipped", "urls": skipped, "providers": [], "attempts": [],
1100
+ "status": "error", "provider": None, "per_url": skipped_rows,
1101
+ "successful_url_count": 0, "skipped_url_count": len(skipped_rows),
1102
+ "retrieved_at": datetime.now(timezone.utc).isoformat(),
1103
+ "archive": {"status": "disabled"},
1104
+ "freshness": "Search-result pages were not fetched; run a search instead."}
1105
+ result = self._fetch(fetchable, provider, archive, max_characters, probe)
1106
+ if skipped_rows:
1107
+ result = dict(result)
1108
+ result["per_url"] = list(result.get("per_url") or []) + skipped_rows
1109
+ result["skipped_url_count"] = len(skipped_rows)
1110
+ if result.get("status") == "ok" and result.get("successful_url_count") == 0:
1111
+ result["status"] = "partial"
1112
+ if cache is not None and cache_key is not None and use_cache:
1113
+ usable = any(row.get("status") == "ok" for row in result.get("per_url") or [])
1114
+ if usable:
1115
+ try:
1116
+ cache.put(cache_key, fetchable, provider, result)
1117
+ except sqlite3.Error:
1118
+ pass
1119
+ return result
1120
+ finally:
1121
+ if cache is not None:
1122
+ cache.close()
1123
+
1124
+ def _fetch(self, urls, provider=None, archive=None, max_characters=None, probe=True):
1029
1125
  preferences = self.settings.load()
1030
1126
  automatic = provider is None
1031
1127
  provider = preferences["fetch"]["provider"] if provider is None else provider
package/src/wiki.py CHANGED
@@ -16,7 +16,7 @@ import shutil
16
16
  import sqlite3
17
17
  import uuid
18
18
  from contextlib import ExitStack, contextmanager
19
- from datetime import datetime, timezone
19
+ from datetime import datetime, timedelta, timezone
20
20
  from pathlib import Path
21
21
 
22
22
  from research import ResearchSessions
@@ -427,7 +427,8 @@ class WikiStore:
427
427
  prepared = {}
428
428
  for source in sources:
429
429
  if not isinstance(source, dict) or set(source) - {
430
- "id", "url", "text", "capture_id", "title", "retrieved_at", "kind"}:
430
+ "id", "url", "text", "capture_id", "title", "retrieved_at", "kind",
431
+ "bookmark_refs", "inventory_ids"}:
431
432
  raise ValueError("Invalid Wiki source fields")
432
433
  identifier = ResearchSessions._identifier(source.get("id"), "Source id")
433
434
  if identifier in prepared:
@@ -438,6 +439,85 @@ class WikiStore:
438
439
  raise ValueError("Invalid Wiki source kind")
439
440
  if kind == "page" or (kind == "local_document" and not url.startswith("file://")):
440
441
  SourceArchive._key(url)
442
+ bookmark_refs = source.get("bookmark_refs")
443
+ inventory_ids = source.get("inventory_ids")
444
+ if bookmark_refs is None and (inventory_ids is not None or url):
445
+ db_exists = (str(self.research.db_path) == ":memory:") or (
446
+ self.research.db_path and Path(self.research.db_path).is_file()
447
+ )
448
+ if db_exists:
449
+ from archive import SourceArchive
450
+ if str(self.research.db_path) == ":memory:":
451
+ from bookmark_index import BookmarkIndex
452
+ index_context = BookmarkIndex(self.research.db_path)
453
+ connection = index_context.connection
454
+ else:
455
+ index_context = None
456
+ connection = sqlite3.connect(self.research.db_path.as_uri() + "?mode=ro", uri=True)
457
+ connection.row_factory = sqlite3.Row
458
+ try:
459
+ rows = connection.execute(
460
+ "SELECT source_id,section_id,item_id,url FROM items "
461
+ "WHERE item_type='bookmark' AND url<>'' ORDER BY source_id,section_id,preorder"
462
+ ).fetchall()
463
+ wanted = set(inventory_ids or [])
464
+ matches = []
465
+ for row in rows:
466
+ try:
467
+ matches_url = SourceArchive._key(row["url"]) == SourceArchive._key(url)
468
+ except (TypeError, ValueError):
469
+ matches_url = row["url"] == url
470
+ inventory_id = "u-" + hashlib.sha256(row["url"].encode("utf-8")).hexdigest()[:32]
471
+ if matches_url and (not wanted or inventory_id in wanted):
472
+ matches.append({"source_id": row["source_id"], "section_id": row["section_id"],
473
+ "item_id": row["item_id"]})
474
+ if matches:
475
+ bookmark_refs = matches
476
+ except sqlite3.Error:
477
+ pass
478
+ finally:
479
+ if index_context is None:
480
+ connection.close()
481
+ else:
482
+ index_context.close()
483
+ if bookmark_refs is not None:
484
+ if not isinstance(bookmark_refs, list):
485
+ raise ValueError("bookmark_refs must be a list")
486
+ for ref in bookmark_refs:
487
+ if not isinstance(ref, dict) or set(ref) != {"source_id", "section_id", "item_id"}:
488
+ raise ValueError("Invalid bookmark_ref structure; needs source_id, section_id, item_id")
489
+ ResearchSessions._identifier(ref["source_id"], "Bookmark ref source_id")
490
+ ResearchSessions._identifier(ref["section_id"], "Bookmark ref section_id")
491
+ ResearchSessions._identifier(ref["item_id"], "Bookmark ref item_id")
492
+ db_exists = (str(self.research.db_path) == ":memory:") or (
493
+ self.research.db_path and Path(self.research.db_path).is_file()
494
+ )
495
+ if not db_exists:
496
+ raise ValueError("Bookmark database is not available to validate bookmark_refs")
497
+ from bookmark_index import BookmarkIndex
498
+ with BookmarkIndex(self.research.db_path) as index:
499
+ for ref in bookmark_refs:
500
+ row = index.connection.execute(
501
+ "SELECT url FROM items WHERE source_id=? AND section_id=? AND item_id=?",
502
+ (ref["source_id"], ref["section_id"], ref["item_id"]),
503
+ ).fetchone()
504
+ if row is None:
505
+ raise ValueError("Bookmark reference does not exist: %s/%s/%s" % (
506
+ ref["source_id"], ref["section_id"], ref["item_id"]))
507
+ item_url = row[0]
508
+ if item_url != url:
509
+ try:
510
+ if SourceArchive._key(item_url) != SourceArchive._key(url):
511
+ raise ValueError("Bookmark reference %s belongs to %s, not %s" % (
512
+ ref["item_id"], item_url, url))
513
+ except Exception:
514
+ raise ValueError("Bookmark reference %s belongs to %s, not %s" % (
515
+ ref["item_id"], item_url, url))
516
+ if inventory_ids is not None:
517
+ if not isinstance(inventory_ids, list):
518
+ raise ValueError("inventory_ids must be a list")
519
+ for inv_id in inventory_ids:
520
+ ResearchSessions._identifier(inv_id, "Inventory id")
441
521
  capture_id = source.get("capture_id")
442
522
  if capture_id and source.get("text") is not None:
443
523
  raise ValueError("Use capture_id or source text, not both")
@@ -454,9 +534,14 @@ class WikiStore:
454
534
  retrieved_at=saved.get("retrieved_at"))
455
535
  elif source.get("retrieved_at") is not None:
456
536
  provenance["retrieved_at"] = source["retrieved_at"]
457
- prepared[identifier] = {"url": url, "text": self._text(text, "Source text", 2000000),
458
- "title": self._text(source.get("title") or url, "Source title", 2000),
459
- "provenance": provenance}
537
+ entry = {"url": url, "text": self._text(text, "Source text", 2000000),
538
+ "title": self._text(source.get("title") or url, "Source title", 2000),
539
+ "provenance": provenance}
540
+ if bookmark_refs is not None:
541
+ entry["bookmark_refs"] = bookmark_refs
542
+ if inventory_ids is not None:
543
+ entry["inventory_ids"] = inventory_ids
544
+ prepared[identifier] = entry
460
545
  value = copy.deepcopy(page)
461
546
  if not isinstance(value, dict) or not isinstance(value.get("sections"), list):
462
547
  raise ValueError("page needs authored sections")
@@ -529,10 +614,11 @@ class WikiStore:
529
614
  shutil.rmtree(path)
530
615
 
531
616
  def write(self, page_id, page, change_note, expected_revision=0, reviewed_input_version=None,
532
- acknowledge_all=False, sources=None):
617
+ acknowledge_all=False, sources=None, ignore_duplicate_warning=False):
533
618
  if sources is None:
534
619
  return self._write_page(page_id, page, change_note, expected_revision,
535
- reviewed_input_version, acknowledge_all)
620
+ reviewed_input_version, acknowledge_all,
621
+ ignore_duplicate_warning=ignore_duplicate_warning)
536
622
  page_id = self._slug(page_id)
537
623
  self._text(change_note, "Change note")
538
624
  ResearchSessions._integer(expected_revision, "expected_revision", 0, 10000)
@@ -544,12 +630,14 @@ class WikiStore:
544
630
  with ExitStack() as cleanup:
545
631
  compiled = self._compile_page(page_id, page, sources, cleanup)
546
632
  result = self._write_page(page_id, compiled, change_note, expected_revision,
547
- reviewed_input_version, acknowledge_all)
633
+ reviewed_input_version, acknowledge_all,
634
+ sources=sources,
635
+ ignore_duplicate_warning=ignore_duplicate_warning)
548
636
  cleanup.pop_all()
549
637
  return result
550
638
 
551
639
  def _write_page(self, page_id, page, change_note, expected_revision=0, reviewed_input_version=None,
552
- acknowledge_all=False):
640
+ acknowledge_all=False, sources=None, ignore_duplicate_warning=False):
553
641
  """Create at revision 0, or update the explicit revision read by the caller.
554
642
 
555
643
  ``reviewed_input_version`` records that the page was checked against that
@@ -569,6 +657,18 @@ class WikiStore:
569
657
  index = self._index()
570
658
  cache = {}
571
659
  evidence = [self._evidence(*reference, cache) for reference in references]
660
+ from archive import SourceArchive
661
+ raw_urls = []
662
+ if sources:
663
+ raw_urls.extend(s.get("url") for s in sources if isinstance(s, dict) and s.get("url"))
664
+ for ev in evidence:
665
+ for cit in ev.get("citations", []):
666
+ if cit.get("url"):
667
+ raw_urls.append(cit["url"])
668
+ canonical_urls = {SourceArchive._key(u) for u in raw_urls if u}
669
+ duplicate_warnings = []
670
+ if not ignore_duplicate_warning:
671
+ duplicate_warnings = self._detect_duplicate_concepts(page_id, page.get("title", ""), canonical_urls, index)
572
672
  previous = index["pages"].get(page_id)
573
673
  if expected_revision != (previous["revision"] if previous else 0):
574
674
  raise ValueError("Wiki revision changed; read the current page before updating")
@@ -633,12 +733,17 @@ class WikiStore:
633
733
  WikiSearchIndex(self._path("search.sqlite3")).update(page_id, pointer, record)
634
734
  except (OSError, ValueError, sqlite3.Error):
635
735
  search_index = "unavailable"
736
+ self._log_metric("wiki_write", page_id=page_id, hit_count=1)
636
737
  from wiki_work import WikiWork
637
- return {"page_id": page_id, **pointer, "status": "written", "directory": str(self.directory),
638
- "artifacts": self._artifacts(pointer), "semantic_review": "caller_declared",
639
- "acknowledged": acknowledged, "search_index": search_index,
640
- "work_receipt": WikiWork.receipt(record, pointer),
641
- "catalog": {key: str(self._path(key)) for key in ("index.md", "log.md")}}
738
+ receipt = {"page_id": page_id, **pointer, "status": "written", "directory": str(self.directory),
739
+ "artifacts": self._artifacts(pointer), "semantic_review": "caller_declared",
740
+ "acknowledged": acknowledged, "search_index": search_index,
741
+ "work_receipt": WikiWork.receipt(record, pointer),
742
+ "catalog": {key: str(self._path(key)) for key in ("index.md", "log.md")}}
743
+ if duplicate_warnings and not ignore_duplicate_warning:
744
+ receipt["duplicate_concept_warning"] = duplicate_warnings
745
+ receipt["warnings"] = duplicate_warnings
746
+ return receipt
642
747
 
643
748
  def acknowledge(self, page_id, note, expected_revision=None):
644
749
  """Re-publish a page unchanged, recording that its sources were re-reviewed.
@@ -743,12 +848,67 @@ class WikiStore:
743
848
  raise ValueError("Wiki revision belongs to a different page")
744
849
  if revision is None or record["revision"] == revision:
745
850
  issues = self._issues(record, index, resolve_paths=True)
851
+ self._log_metric("wiki_get", page_id=page_id, hit_count=1, reused=True)
746
852
  return {**record, "artifacts": self._artifacts(pointer), "validation": self._validation(issues)}
747
853
  if record["previous"] is not None and record["previous"]["revision"] >= record["revision"]:
748
854
  raise ValueError("Invalid Wiki revision history")
749
855
  pointer = record["previous"]
750
856
  raise ValueError("Unknown Wiki page or revision")
751
857
 
858
+ def _detect_duplicate_concepts(self, page_id, title, urls, index):
859
+ try:
860
+ return WikiSearchIndex(self._path("search.sqlite3")).detect_duplicates(
861
+ index, self._revision, page_id, title, urls
862
+ )
863
+ except (OSError, ValueError, sqlite3.Error):
864
+ from archive import SourceArchive
865
+ norm_new = WikiSearchIndex.normalize(title) if title else ""
866
+ tokens_new = set(re.findall(r"\w+", title.lower())) if title else set()
867
+ duplicates = []
868
+ for other_id, pointer in index.get("pages", {}).items():
869
+ if other_id == page_id:
870
+ continue
871
+ other_title = pointer.get("title", "")
872
+ norm_other = WikiSearchIndex.normalize(other_title) if other_title else ""
873
+ tokens_other = set(re.findall(r"\w+", other_title.lower())) if other_title else set()
874
+ reasons = []
875
+ similar_title = False
876
+ if norm_new and norm_other:
877
+ if norm_new == norm_other:
878
+ similar_title = True
879
+ elif (norm_new in norm_other or norm_other in norm_new) and min(len(norm_new), len(norm_other)) >= 4:
880
+ similar_title = True
881
+ elif tokens_new and tokens_other:
882
+ union = tokens_new | tokens_other
883
+ if union and (len(tokens_new & tokens_other) / len(union)) >= 0.7:
884
+ similar_title = True
885
+ if similar_title:
886
+ reasons.append("similar_title")
887
+ matched_urls = []
888
+ if urls:
889
+ try:
890
+ record = self._revision(pointer)
891
+ rec_urls = {
892
+ SourceArchive._key(c["url"]) for ev in record.get("evidence", [])
893
+ for c in ev.get("citations", [])
894
+ if c.get("url") and (c.get("provenance") or {}).get("kind") not in ("local_document", "external_report")
895
+ }
896
+ matched_urls = sorted(urls & rec_urls)
897
+ if matched_urls:
898
+ reasons.append("overlapping_urls")
899
+ except Exception:
900
+ pass
901
+ if reasons:
902
+ duplicates.append({
903
+ "page_id": other_id,
904
+ "title": other_title,
905
+ "reasons": reasons,
906
+ "overlapping_urls": matched_urls,
907
+ "candidate_merge_target": other_id,
908
+ "suggestion": f"Consider reviewing or merging with existing Wiki page '{other_id}' ({other_title})."
909
+ })
910
+ return duplicates
911
+
752
912
  def _affected(self, index, urls):
753
913
  try:
754
914
  return WikiSearchIndex(self._path("search.sqlite3")).affected(index, self._revision, urls)
@@ -907,7 +1067,8 @@ class WikiStore:
907
1067
  query = self._text(query, "Wiki query", 500)
908
1068
  ResearchSessions._integer(offset, "offset", 0, 10000000)
909
1069
  ResearchSessions._integer(limit, "limit", 1, 100)
910
- groups = WikiSearchIndex.terms(query)
1070
+ extra_aliases = self.settings.load().get("wiki", {}).get("aliases", [])
1071
+ groups = WikiSearchIndex.terms(query, extra_aliases=extra_aliases)
911
1072
  index, matches, excluded, cache = self._index(), [], [], {}
912
1073
  search_index = "indexed"
913
1074
  try:
@@ -922,16 +1083,24 @@ class WikiStore:
922
1083
  except (OSError, ValueError, KeyError, TypeError):
923
1084
  invalid.append(page_id)
924
1085
  excluded.extend(invalid)
1086
+ doc_freq = {}
1087
+ for candidate in candidates:
1088
+ cand_text = (candidate["title"] + "\n" + candidate["body"] + "\n" + candidate.get("aliases", "")).casefold()
1089
+ cand_squashed = WikiSearchIndex.normalize(cand_text)
1090
+ for variants in groups:
1091
+ matched_variant = next((variant for variant in variants if WikiSearchIndex.normalize(variant) in cand_squashed), None)
1092
+ if matched_variant:
1093
+ doc_freq[matched_variant] = doc_freq.get(matched_variant, 0) + 1
925
1094
  checked = 0
926
1095
  for candidate in candidates:
927
- match = WikiSearchIndex.match(candidate, query, groups)
1096
+ match = WikiSearchIndex.match(candidate, query, groups, doc_freq=doc_freq)
928
1097
  if match is None:
929
1098
  continue
930
1099
  page_id = candidate["page_id"]
931
1100
  pointer = index["pages"][page_id]
932
1101
  try:
933
1102
  record = self._revision(pointer)
934
- match = WikiSearchIndex.match(WikiSearchIndex.row(page_id, pointer, record), query, groups)
1103
+ match = WikiSearchIndex.match(WikiSearchIndex.row(page_id, pointer, record), query, groups, doc_freq=doc_freq)
935
1104
  if match is None:
936
1105
  continue
937
1106
  checked += 1
@@ -949,6 +1118,7 @@ class WikiStore:
949
1118
  except (OSError, ValueError, KeyError, TypeError):
950
1119
  excluded.append(page_id)
951
1120
  matches.sort(key=lambda row: (-row[0], row[1]["page_id"]))
1121
+ self._log_metric("wiki_search", hit_count=len(matches))
952
1122
  return {"query": query, "results": [row[1] for row in matches[offset:offset + limit]], "total": len(matches),
953
1123
  "next_offset": offset + limit if offset + limit < len(matches) else None,
954
1124
  "excluded_stale_pages": len(excluded), "excluded_page_ids": excluded[:100],
@@ -1052,7 +1222,88 @@ class WikiStore:
1052
1222
  lines.extend(["", "[Saved text](%s) · [Manifest](%s) · [Response](%s) · SHA-256: `%s`" %
1053
1223
  (artifacts["body_file"], artifacts["manifest_file"], artifacts["response_file"],
1054
1224
  citation["source_sha256"]), ""])
1055
- lines.extend(["## Update record", "", record["change_note"], ""])
1056
1225
  if record["previous"]:
1057
1226
  lines.extend(["[Previous revision](%s)" % link(self._path(record["previous"]["markdown_file"])), ""])
1058
1227
  return "\n".join(lines) + "\n"
1228
+
1229
+ def _log_metric(self, action, page_id=None, hit_count=None, reused=None, page_ids=None):
1230
+ try:
1231
+ path = self._path("metrics.jsonl")
1232
+ entry = {
1233
+ "timestamp": datetime.now(timezone.utc).isoformat(),
1234
+ "action": action,
1235
+ "page_id": page_id,
1236
+ "hit_count": hit_count,
1237
+ "reused": reused,
1238
+ }
1239
+ if page_ids:
1240
+ entry["page_ids"] = list(dict.fromkeys(page_ids))
1241
+ with path.open("a", encoding="utf-8") as stream:
1242
+ stream.write(json.dumps(entry, ensure_ascii=False) + "\n")
1243
+ except Exception:
1244
+ pass
1245
+
1246
+ def metrics_summary(self, days=30):
1247
+ """Summarize append-only Wiki usage metrics over the past N days."""
1248
+ ResearchSessions._integer(days, "days", 1, 3650)
1249
+ now = datetime.now(timezone.utc)
1250
+ cutoff = now - timedelta(days=days)
1251
+ path = self._path("metrics.jsonl")
1252
+ summary = {
1253
+ "days": days,
1254
+ "since": cutoff.isoformat(),
1255
+ "total_hits": 0,
1256
+ "searches": 0,
1257
+ "reads": 0,
1258
+ "writes": 0,
1259
+ "plan_preflights": 0,
1260
+ "reused_pages": [],
1261
+ "actions": {},
1262
+ }
1263
+ if not path.is_file():
1264
+ return summary
1265
+ reused_set = set()
1266
+ try:
1267
+ with path.open("r", encoding="utf-8") as stream:
1268
+ for line in stream:
1269
+ line = line.strip()
1270
+ if not line:
1271
+ continue
1272
+ try:
1273
+ record = json.loads(line)
1274
+ except Exception:
1275
+ continue
1276
+ ts_str = record.get("timestamp")
1277
+ if not ts_str:
1278
+ continue
1279
+ try:
1280
+ ts = datetime.fromisoformat(ts_str)
1281
+ except Exception:
1282
+ continue
1283
+ if ts.tzinfo is None:
1284
+ ts = ts.replace(tzinfo=timezone.utc)
1285
+ if ts < cutoff:
1286
+ continue
1287
+ action = record.get("action", "unknown")
1288
+ summary["actions"][action] = summary["actions"].get(action, 0) + 1
1289
+ hit_count = record.get("hit_count") or 0
1290
+ if action == "wiki_search":
1291
+ summary["searches"] += 1
1292
+ summary["total_hits"] += hit_count
1293
+ elif action == "wiki_get":
1294
+ summary["reads"] += 1
1295
+ elif action == "wiki_write":
1296
+ summary["writes"] += 1
1297
+ elif action == "plan_research":
1298
+ summary["plan_preflights"] += 1
1299
+ summary["total_hits"] += hit_count
1300
+ if record.get("reused"):
1301
+ if record.get("page_id"):
1302
+ reused_set.add(record["page_id"])
1303
+ for pid in record.get("page_ids", []):
1304
+ reused_set.add(pid)
1305
+ except Exception:
1306
+ pass
1307
+ summary["reused_pages"] = sorted(reused_set)
1308
+ return summary
1309
+