bookmark-research 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +1 -1
- package/.codex-plugin/plugin.json +4 -1
- package/README.md +15 -9
- package/README.zh.md +16 -9
- package/config/providers.json +26 -0
- package/config/settings.example.json +36 -0
- package/docs/bookmark-research-architecture.md +2 -2
- package/docs/canvas-tree-reading-acceptance.md +98 -0
- package/docs/configuration.en.md +70 -0
- package/docs/configuration.md +70 -0
- package/docs/details.en.md +10 -6
- package/docs/details.md +3 -3
- package/docs/harness-compatibility.md +1 -1
- package/docs/installation.en.md +4 -2
- package/docs/installation.md +4 -2
- package/docs/instructions.en.md +1 -0
- package/docs/instructions.md +1 -0
- package/docs/user-guide.en.md +1 -1
- package/docs/user-guide.md +1 -1
- package/hosts/dsh/plugin.js +3 -0
- package/package.json +1 -1
- package/scripts/bench_sync.py +191 -0
- package/scripts/bench_tools_list.py +129 -0
- package/scripts/evaluate_ladder.py +149 -0
- package/scripts/export_bundle.py +2 -0
- package/scripts/host_install.py +4 -1
- package/scripts/install.py +120 -28
- package/skills/bookmark-research/SKILL.md +28 -9
- package/skills/bookmark-research/references/package-semantics.md +56 -6
- package/skills/bookmark-research/references/research-workflow.md +5 -2
- package/skills/bookmark-research/references/settings-and-archive.md +5 -3
- package/skills/bookmark-research/references/wiki.md +3 -1
- package/skills/bookmark-research/references/zh/package-semantics.md +56 -6
- package/skills/bookmark-research/references/zh/research-workflow.md +5 -2
- package/skills/bookmark-research/references/zh/settings-and-archive.md +5 -3
- package/skills/bookmark-research/references/zh/skill-guide.md +20 -8
- package/skills/bookmark-research/references/zh/wiki.md +3 -1
- package/src/archive.py +100 -0
- package/src/bookmark_index.py +808 -41
- package/src/browser_client.py +342 -0
- package/src/cli.py +298 -14
- package/src/credentials.py +2 -1
- package/src/fetch_cache.py +60 -0
- package/src/grok_search.py +272 -0
- package/src/jobs.py +349 -0
- package/src/junk.py +33 -0
- package/src/locate.py +862 -0
- package/src/mcp_dispatch.py +22 -6
- package/src/mcp_server.py +573 -23
- package/src/netscape.py +124 -0
- package/src/onboarding.py +59 -9
- package/src/perplexity.py +78 -0
- package/src/raw_library.py +183 -45
- package/src/readiness.py +40 -5
- package/src/ref_codec.py +70 -0
- package/src/research.py +125 -12
- package/src/research_plan.py +11 -1
- package/src/routing.py +5 -2
- package/src/search_results.py +80 -2
- package/src/settings.py +61 -16
- package/src/source_inputs.py +109 -3
- package/src/source_manager.py +71 -12
- package/src/source_versions.py +20 -2
- package/src/source_watcher.py +4 -2
- package/src/storage.py +60 -0
- package/src/url_classify.py +45 -0
- package/src/web_search.py +106 -10
- package/src/wiki.py +269 -18
- package/src/wiki_search.py +119 -27
package/src/web_search.py
CHANGED
|
@@ -4,6 +4,7 @@ import concurrent.futures
|
|
|
4
4
|
import copy
|
|
5
5
|
import json
|
|
6
6
|
import re
|
|
7
|
+
import sqlite3
|
|
7
8
|
import threading
|
|
8
9
|
import time
|
|
9
10
|
import uuid
|
|
@@ -116,6 +117,13 @@ class SearchProviders:
|
|
|
116
117
|
state.client = DeepSeekClient(headers=headers, timeout=timeout,
|
|
117
118
|
search_url=info.get("search_url"),
|
|
118
119
|
fetch_url=info.get("fetch_url"))
|
|
120
|
+
elif info.get("client") == "perplexity":
|
|
121
|
+
from perplexity import PerplexityClient
|
|
122
|
+
state.client = PerplexityClient(headers=headers, timeout=timeout,
|
|
123
|
+
search_url=info.get("search_url"))
|
|
124
|
+
elif info.get("client") == "browser":
|
|
125
|
+
from browser_client import BrowserClient
|
|
126
|
+
state.client = BrowserClient(settings=self.settings, timeout=timeout)
|
|
119
127
|
else:
|
|
120
128
|
from jina import JinaClient
|
|
121
129
|
state.client = JinaClient(headers=headers, timeout=timeout)
|
|
@@ -201,10 +209,21 @@ class SearchProviders:
|
|
|
201
209
|
client, before, usage = None, None, self._empty_usage()
|
|
202
210
|
try:
|
|
203
211
|
if self.registry["providers"][provider].get("transport") == "http":
|
|
212
|
+
if provider == "browser":
|
|
213
|
+
enabled = self.settings.load().get("browser", {}).get("enabled", False)
|
|
214
|
+
capabilities = {"search": {"status": "unsupported", "execution_verified": False},
|
|
215
|
+
"fetch": {"status": "supported" if enabled else "disabled", "execution_verified": False}}
|
|
216
|
+
return {"provider": provider, "status": "ok" if enabled else "not_configured", "tools": ["browser_render"] if enabled else [],
|
|
217
|
+
"network_probe_performed": False, "catalog_source": "local_browser_driver",
|
|
218
|
+
"capabilities": capabilities,
|
|
219
|
+
"authentication_state": {"configured": enabled, "credential_configured": enabled, "credential_validity_verified": False, "oauth_implemented": False},
|
|
220
|
+
"usage": usage}
|
|
221
|
+
capabilities = {purpose: {"status": "supported" if self.registry["providers"][provider].get(purpose + "_tools") else "unsupported",
|
|
222
|
+
"execution_verified": False}
|
|
223
|
+
for purpose in ("search", "fetch")}
|
|
204
224
|
return {"provider": provider, "status": "not_probed", "tools": [],
|
|
205
225
|
"network_probe_performed": False, "catalog_source": "documented_http_api",
|
|
206
|
-
"capabilities":
|
|
207
|
-
for purpose in ("search", "fetch")},
|
|
226
|
+
"capabilities": capabilities,
|
|
208
227
|
"authentication_state": self._authentication(provider), "usage": usage}
|
|
209
228
|
client = self._client(provider)
|
|
210
229
|
before = self._snapshot(client)
|
|
@@ -373,6 +392,8 @@ class SearchProviders:
|
|
|
373
392
|
skipped = True
|
|
374
393
|
raise state.cooldown_error
|
|
375
394
|
if self.registry["providers"][provider].get("transport") == "http":
|
|
395
|
+
if not self.registry["providers"][provider].get(purpose + "_tools", []):
|
|
396
|
+
raise McpError("Provider does not support " + purpose, "capability_unavailable")
|
|
376
397
|
tool = {"name": self.registry["providers"][provider][purpose + "_tools"][0]}
|
|
377
398
|
arguments = {"query": query} if purpose == "search" else {"urls": urls}
|
|
378
399
|
if purpose == "search":
|
|
@@ -871,7 +892,9 @@ class SearchProviders:
|
|
|
871
892
|
attempts, coverage, ranks = [], {}, {}
|
|
872
893
|
|
|
873
894
|
def quality(row):
|
|
874
|
-
|
|
895
|
+
usable = row.get("content_state") not in ("js_shell_suspected", "challenge_blocked", "unavailable")
|
|
896
|
+
return (row["status"] == "ok" and usable,
|
|
897
|
+
row["status"] == "ok", row.get("characters", 0) > 0,
|
|
875
898
|
row["extraction_status"] not in ("request_failed", "unrecognized_or_not_returned",
|
|
876
899
|
"no_content_returned", "content_present_unparsed"))
|
|
877
900
|
|
|
@@ -899,11 +922,18 @@ class SearchProviders:
|
|
|
899
922
|
coverage[row["url"]] = {**row, "provider": fetched["provider"]}
|
|
900
923
|
ranks[row["url"]] = rank
|
|
901
924
|
|
|
902
|
-
def unresolved():
|
|
903
|
-
|
|
925
|
+
def unresolved(remaining_providers=()):
|
|
926
|
+
def needs_work(u):
|
|
927
|
+
current = coverage.get(u)
|
|
928
|
+
if not current or current.get("status") != "ok":
|
|
929
|
+
return True
|
|
930
|
+
if current.get("content_state") in ("js_shell_suspected", "challenge_blocked") and any(p == "browser" for p in remaining_providers):
|
|
931
|
+
return True
|
|
932
|
+
return False
|
|
933
|
+
return [url for url in urls if needs_work(url)]
|
|
904
934
|
|
|
905
935
|
def assemble(abandoned=()):
|
|
906
|
-
pending =
|
|
936
|
+
pending = [url for url in urls if coverage.get(url, {}).get("status") != "ok"]
|
|
907
937
|
successful = len(urls) - len(pending)
|
|
908
938
|
return {"mode": "waterfall", "urls": urls, "providers": providers, "attempts": attempts,
|
|
909
939
|
"status": "ok" if not pending else "partial" if successful else "error",
|
|
@@ -918,7 +948,7 @@ class SearchProviders:
|
|
|
918
948
|
|
|
919
949
|
if strategy == "sequential":
|
|
920
950
|
for stage, names in enumerate((providers[:1], providers[1:])):
|
|
921
|
-
pending =
|
|
951
|
+
pending = unresolved(names)
|
|
922
952
|
if not pending or not names:
|
|
923
953
|
break
|
|
924
954
|
if stage and reserve_fallback is not None:
|
|
@@ -1003,15 +1033,15 @@ class SearchProviders:
|
|
|
1003
1033
|
# One provider finished with work left: reserve and hand the
|
|
1004
1034
|
# pending URLs to every remaining provider at once, exactly
|
|
1005
1035
|
# like the sequential fallback stage.
|
|
1006
|
-
pending = unresolved()
|
|
1007
1036
|
names = queue[:]
|
|
1037
|
+
pending = unresolved(names)
|
|
1008
1038
|
if names and reserve_fallback is not None:
|
|
1009
1039
|
names = list(reserve_fallback(names, pending))
|
|
1010
1040
|
queue[:] = []
|
|
1011
1041
|
start(names, pending)
|
|
1012
1042
|
elif queue and last_dispatch is not None and time.monotonic() >= last_dispatch + grace:
|
|
1013
|
-
pending = unresolved()
|
|
1014
1043
|
names = [queue.pop(0)]
|
|
1044
|
+
pending = unresolved(names)
|
|
1015
1045
|
if reserve_fallback is not None:
|
|
1016
1046
|
names = list(reserve_fallback(names, pending))
|
|
1017
1047
|
start(names, pending)
|
|
@@ -1025,7 +1055,73 @@ class SearchProviders:
|
|
|
1025
1055
|
pool.shutdown(wait=not abandoned)
|
|
1026
1056
|
return assemble(abandoned)
|
|
1027
1057
|
|
|
1028
|
-
def fetch(self, urls, provider=None, archive=None, max_characters=None, probe=True):
|
|
1058
|
+
def fetch(self, urls, provider=None, archive=None, max_characters=None, probe=True, use_cache=True):
|
|
1059
|
+
"""Route known URLs: skip search-result pages, reuse a fresh cache hit, else fetch."""
|
|
1060
|
+
from url_classify import is_search_result_url
|
|
1061
|
+
preferences = self.settings.load()
|
|
1062
|
+
if not isinstance(urls, list):
|
|
1063
|
+
return self._fetch(urls, provider, archive, max_characters, probe)
|
|
1064
|
+
skipped = [url for url in urls if isinstance(url, str) and is_search_result_url(url)]
|
|
1065
|
+
fetchable = [url for url in urls if url not in skipped]
|
|
1066
|
+
skipped_rows = [{"url": url, "status": "skipped", "content_state": "unavailable",
|
|
1067
|
+
"extraction_status": "search_result_page", "characters": 0,
|
|
1068
|
+
"complete_page_verified": False,
|
|
1069
|
+
"error": "search-engine result page; search again instead of fetching it"}
|
|
1070
|
+
for url in skipped]
|
|
1071
|
+
ttl = (preferences["fetch"].get("reuse_seconds") or 0) if use_cache else 0
|
|
1072
|
+
cache = None
|
|
1073
|
+
cache_key = None
|
|
1074
|
+
if ttl > 0 and fetchable:
|
|
1075
|
+
try:
|
|
1076
|
+
from fetch_cache import FetchCache
|
|
1077
|
+
cache = FetchCache()
|
|
1078
|
+
effective_archive = (preferences["archive"]["enabled"]
|
|
1079
|
+
if archive is None else archive)
|
|
1080
|
+
effective_max = (preferences["fetch"]["max_characters"]
|
|
1081
|
+
if max_characters is None else max_characters)
|
|
1082
|
+
cache_key = FetchCache.key(fetchable, provider, effective_archive, effective_max)
|
|
1083
|
+
hit = cache.get(cache_key, ttl)
|
|
1084
|
+
except (OSError, ValueError, sqlite3.Error):
|
|
1085
|
+
hit = None
|
|
1086
|
+
if hit is not None:
|
|
1087
|
+
cached, age = hit
|
|
1088
|
+
cached = dict(cached)
|
|
1089
|
+
cached["cached"] = True
|
|
1090
|
+
cached["cache_age_seconds"] = round(age, 1)
|
|
1091
|
+
if skipped_rows:
|
|
1092
|
+
cached["per_url"] = list(cached.get("per_url") or []) + skipped_rows
|
|
1093
|
+
cached["skipped_url_count"] = len(skipped_rows)
|
|
1094
|
+
if cache is not None:
|
|
1095
|
+
cache.close()
|
|
1096
|
+
return cached
|
|
1097
|
+
try:
|
|
1098
|
+
if not fetchable and skipped_rows:
|
|
1099
|
+
return {"mode": "skipped", "urls": skipped, "providers": [], "attempts": [],
|
|
1100
|
+
"status": "error", "provider": None, "per_url": skipped_rows,
|
|
1101
|
+
"successful_url_count": 0, "skipped_url_count": len(skipped_rows),
|
|
1102
|
+
"retrieved_at": datetime.now(timezone.utc).isoformat(),
|
|
1103
|
+
"archive": {"status": "disabled"},
|
|
1104
|
+
"freshness": "Search-result pages were not fetched; run a search instead."}
|
|
1105
|
+
result = self._fetch(fetchable, provider, archive, max_characters, probe)
|
|
1106
|
+
if skipped_rows:
|
|
1107
|
+
result = dict(result)
|
|
1108
|
+
result["per_url"] = list(result.get("per_url") or []) + skipped_rows
|
|
1109
|
+
result["skipped_url_count"] = len(skipped_rows)
|
|
1110
|
+
if result.get("status") == "ok" and result.get("successful_url_count") == 0:
|
|
1111
|
+
result["status"] = "partial"
|
|
1112
|
+
if cache is not None and cache_key is not None and use_cache:
|
|
1113
|
+
usable = any(row.get("status") == "ok" for row in result.get("per_url") or [])
|
|
1114
|
+
if usable:
|
|
1115
|
+
try:
|
|
1116
|
+
cache.put(cache_key, fetchable, provider, result)
|
|
1117
|
+
except sqlite3.Error:
|
|
1118
|
+
pass
|
|
1119
|
+
return result
|
|
1120
|
+
finally:
|
|
1121
|
+
if cache is not None:
|
|
1122
|
+
cache.close()
|
|
1123
|
+
|
|
1124
|
+
def _fetch(self, urls, provider=None, archive=None, max_characters=None, probe=True):
|
|
1029
1125
|
preferences = self.settings.load()
|
|
1030
1126
|
automatic = provider is None
|
|
1031
1127
|
provider = preferences["fetch"]["provider"] if provider is None else provider
|
package/src/wiki.py
CHANGED
|
@@ -16,7 +16,7 @@ import shutil
|
|
|
16
16
|
import sqlite3
|
|
17
17
|
import uuid
|
|
18
18
|
from contextlib import ExitStack, contextmanager
|
|
19
|
-
from datetime import datetime, timezone
|
|
19
|
+
from datetime import datetime, timedelta, timezone
|
|
20
20
|
from pathlib import Path
|
|
21
21
|
|
|
22
22
|
from research import ResearchSessions
|
|
@@ -427,7 +427,8 @@ class WikiStore:
|
|
|
427
427
|
prepared = {}
|
|
428
428
|
for source in sources:
|
|
429
429
|
if not isinstance(source, dict) or set(source) - {
|
|
430
|
-
"id", "url", "text", "capture_id", "title", "retrieved_at", "kind"
|
|
430
|
+
"id", "url", "text", "capture_id", "title", "retrieved_at", "kind",
|
|
431
|
+
"bookmark_refs", "inventory_ids"}:
|
|
431
432
|
raise ValueError("Invalid Wiki source fields")
|
|
432
433
|
identifier = ResearchSessions._identifier(source.get("id"), "Source id")
|
|
433
434
|
if identifier in prepared:
|
|
@@ -438,6 +439,85 @@ class WikiStore:
|
|
|
438
439
|
raise ValueError("Invalid Wiki source kind")
|
|
439
440
|
if kind == "page" or (kind == "local_document" and not url.startswith("file://")):
|
|
440
441
|
SourceArchive._key(url)
|
|
442
|
+
bookmark_refs = source.get("bookmark_refs")
|
|
443
|
+
inventory_ids = source.get("inventory_ids")
|
|
444
|
+
if bookmark_refs is None and (inventory_ids is not None or url):
|
|
445
|
+
db_exists = (str(self.research.db_path) == ":memory:") or (
|
|
446
|
+
self.research.db_path and Path(self.research.db_path).is_file()
|
|
447
|
+
)
|
|
448
|
+
if db_exists:
|
|
449
|
+
from archive import SourceArchive
|
|
450
|
+
if str(self.research.db_path) == ":memory:":
|
|
451
|
+
from bookmark_index import BookmarkIndex
|
|
452
|
+
index_context = BookmarkIndex(self.research.db_path)
|
|
453
|
+
connection = index_context.connection
|
|
454
|
+
else:
|
|
455
|
+
index_context = None
|
|
456
|
+
connection = sqlite3.connect(self.research.db_path.as_uri() + "?mode=ro", uri=True)
|
|
457
|
+
connection.row_factory = sqlite3.Row
|
|
458
|
+
try:
|
|
459
|
+
rows = connection.execute(
|
|
460
|
+
"SELECT source_id,section_id,item_id,url FROM items "
|
|
461
|
+
"WHERE item_type='bookmark' AND url<>'' ORDER BY source_id,section_id,preorder"
|
|
462
|
+
).fetchall()
|
|
463
|
+
wanted = set(inventory_ids or [])
|
|
464
|
+
matches = []
|
|
465
|
+
for row in rows:
|
|
466
|
+
try:
|
|
467
|
+
matches_url = SourceArchive._key(row["url"]) == SourceArchive._key(url)
|
|
468
|
+
except (TypeError, ValueError):
|
|
469
|
+
matches_url = row["url"] == url
|
|
470
|
+
inventory_id = "u-" + hashlib.sha256(row["url"].encode("utf-8")).hexdigest()[:32]
|
|
471
|
+
if matches_url and (not wanted or inventory_id in wanted):
|
|
472
|
+
matches.append({"source_id": row["source_id"], "section_id": row["section_id"],
|
|
473
|
+
"item_id": row["item_id"]})
|
|
474
|
+
if matches:
|
|
475
|
+
bookmark_refs = matches
|
|
476
|
+
except sqlite3.Error:
|
|
477
|
+
pass
|
|
478
|
+
finally:
|
|
479
|
+
if index_context is None:
|
|
480
|
+
connection.close()
|
|
481
|
+
else:
|
|
482
|
+
index_context.close()
|
|
483
|
+
if bookmark_refs is not None:
|
|
484
|
+
if not isinstance(bookmark_refs, list):
|
|
485
|
+
raise ValueError("bookmark_refs must be a list")
|
|
486
|
+
for ref in bookmark_refs:
|
|
487
|
+
if not isinstance(ref, dict) or set(ref) != {"source_id", "section_id", "item_id"}:
|
|
488
|
+
raise ValueError("Invalid bookmark_ref structure; needs source_id, section_id, item_id")
|
|
489
|
+
ResearchSessions._identifier(ref["source_id"], "Bookmark ref source_id")
|
|
490
|
+
ResearchSessions._identifier(ref["section_id"], "Bookmark ref section_id")
|
|
491
|
+
ResearchSessions._identifier(ref["item_id"], "Bookmark ref item_id")
|
|
492
|
+
db_exists = (str(self.research.db_path) == ":memory:") or (
|
|
493
|
+
self.research.db_path and Path(self.research.db_path).is_file()
|
|
494
|
+
)
|
|
495
|
+
if not db_exists:
|
|
496
|
+
raise ValueError("Bookmark database is not available to validate bookmark_refs")
|
|
497
|
+
from bookmark_index import BookmarkIndex
|
|
498
|
+
with BookmarkIndex(self.research.db_path) as index:
|
|
499
|
+
for ref in bookmark_refs:
|
|
500
|
+
row = index.connection.execute(
|
|
501
|
+
"SELECT url FROM items WHERE source_id=? AND section_id=? AND item_id=?",
|
|
502
|
+
(ref["source_id"], ref["section_id"], ref["item_id"]),
|
|
503
|
+
).fetchone()
|
|
504
|
+
if row is None:
|
|
505
|
+
raise ValueError("Bookmark reference does not exist: %s/%s/%s" % (
|
|
506
|
+
ref["source_id"], ref["section_id"], ref["item_id"]))
|
|
507
|
+
item_url = row[0]
|
|
508
|
+
if item_url != url:
|
|
509
|
+
try:
|
|
510
|
+
if SourceArchive._key(item_url) != SourceArchive._key(url):
|
|
511
|
+
raise ValueError("Bookmark reference %s belongs to %s, not %s" % (
|
|
512
|
+
ref["item_id"], item_url, url))
|
|
513
|
+
except Exception:
|
|
514
|
+
raise ValueError("Bookmark reference %s belongs to %s, not %s" % (
|
|
515
|
+
ref["item_id"], item_url, url))
|
|
516
|
+
if inventory_ids is not None:
|
|
517
|
+
if not isinstance(inventory_ids, list):
|
|
518
|
+
raise ValueError("inventory_ids must be a list")
|
|
519
|
+
for inv_id in inventory_ids:
|
|
520
|
+
ResearchSessions._identifier(inv_id, "Inventory id")
|
|
441
521
|
capture_id = source.get("capture_id")
|
|
442
522
|
if capture_id and source.get("text") is not None:
|
|
443
523
|
raise ValueError("Use capture_id or source text, not both")
|
|
@@ -454,9 +534,14 @@ class WikiStore:
|
|
|
454
534
|
retrieved_at=saved.get("retrieved_at"))
|
|
455
535
|
elif source.get("retrieved_at") is not None:
|
|
456
536
|
provenance["retrieved_at"] = source["retrieved_at"]
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
537
|
+
entry = {"url": url, "text": self._text(text, "Source text", 2000000),
|
|
538
|
+
"title": self._text(source.get("title") or url, "Source title", 2000),
|
|
539
|
+
"provenance": provenance}
|
|
540
|
+
if bookmark_refs is not None:
|
|
541
|
+
entry["bookmark_refs"] = bookmark_refs
|
|
542
|
+
if inventory_ids is not None:
|
|
543
|
+
entry["inventory_ids"] = inventory_ids
|
|
544
|
+
prepared[identifier] = entry
|
|
460
545
|
value = copy.deepcopy(page)
|
|
461
546
|
if not isinstance(value, dict) or not isinstance(value.get("sections"), list):
|
|
462
547
|
raise ValueError("page needs authored sections")
|
|
@@ -529,10 +614,11 @@ class WikiStore:
|
|
|
529
614
|
shutil.rmtree(path)
|
|
530
615
|
|
|
531
616
|
def write(self, page_id, page, change_note, expected_revision=0, reviewed_input_version=None,
|
|
532
|
-
acknowledge_all=False, sources=None):
|
|
617
|
+
acknowledge_all=False, sources=None, ignore_duplicate_warning=False):
|
|
533
618
|
if sources is None:
|
|
534
619
|
return self._write_page(page_id, page, change_note, expected_revision,
|
|
535
|
-
reviewed_input_version, acknowledge_all
|
|
620
|
+
reviewed_input_version, acknowledge_all,
|
|
621
|
+
ignore_duplicate_warning=ignore_duplicate_warning)
|
|
536
622
|
page_id = self._slug(page_id)
|
|
537
623
|
self._text(change_note, "Change note")
|
|
538
624
|
ResearchSessions._integer(expected_revision, "expected_revision", 0, 10000)
|
|
@@ -544,12 +630,14 @@ class WikiStore:
|
|
|
544
630
|
with ExitStack() as cleanup:
|
|
545
631
|
compiled = self._compile_page(page_id, page, sources, cleanup)
|
|
546
632
|
result = self._write_page(page_id, compiled, change_note, expected_revision,
|
|
547
|
-
reviewed_input_version, acknowledge_all
|
|
633
|
+
reviewed_input_version, acknowledge_all,
|
|
634
|
+
sources=sources,
|
|
635
|
+
ignore_duplicate_warning=ignore_duplicate_warning)
|
|
548
636
|
cleanup.pop_all()
|
|
549
637
|
return result
|
|
550
638
|
|
|
551
639
|
def _write_page(self, page_id, page, change_note, expected_revision=0, reviewed_input_version=None,
|
|
552
|
-
acknowledge_all=False):
|
|
640
|
+
acknowledge_all=False, sources=None, ignore_duplicate_warning=False):
|
|
553
641
|
"""Create at revision 0, or update the explicit revision read by the caller.
|
|
554
642
|
|
|
555
643
|
``reviewed_input_version`` records that the page was checked against that
|
|
@@ -569,6 +657,18 @@ class WikiStore:
|
|
|
569
657
|
index = self._index()
|
|
570
658
|
cache = {}
|
|
571
659
|
evidence = [self._evidence(*reference, cache) for reference in references]
|
|
660
|
+
from archive import SourceArchive
|
|
661
|
+
raw_urls = []
|
|
662
|
+
if sources:
|
|
663
|
+
raw_urls.extend(s.get("url") for s in sources if isinstance(s, dict) and s.get("url"))
|
|
664
|
+
for ev in evidence:
|
|
665
|
+
for cit in ev.get("citations", []):
|
|
666
|
+
if cit.get("url"):
|
|
667
|
+
raw_urls.append(cit["url"])
|
|
668
|
+
canonical_urls = {SourceArchive._key(u) for u in raw_urls if u}
|
|
669
|
+
duplicate_warnings = []
|
|
670
|
+
if not ignore_duplicate_warning:
|
|
671
|
+
duplicate_warnings = self._detect_duplicate_concepts(page_id, page.get("title", ""), canonical_urls, index)
|
|
572
672
|
previous = index["pages"].get(page_id)
|
|
573
673
|
if expected_revision != (previous["revision"] if previous else 0):
|
|
574
674
|
raise ValueError("Wiki revision changed; read the current page before updating")
|
|
@@ -633,12 +733,17 @@ class WikiStore:
|
|
|
633
733
|
WikiSearchIndex(self._path("search.sqlite3")).update(page_id, pointer, record)
|
|
634
734
|
except (OSError, ValueError, sqlite3.Error):
|
|
635
735
|
search_index = "unavailable"
|
|
736
|
+
self._log_metric("wiki_write", page_id=page_id, hit_count=1)
|
|
636
737
|
from wiki_work import WikiWork
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
738
|
+
receipt = {"page_id": page_id, **pointer, "status": "written", "directory": str(self.directory),
|
|
739
|
+
"artifacts": self._artifacts(pointer), "semantic_review": "caller_declared",
|
|
740
|
+
"acknowledged": acknowledged, "search_index": search_index,
|
|
741
|
+
"work_receipt": WikiWork.receipt(record, pointer),
|
|
742
|
+
"catalog": {key: str(self._path(key)) for key in ("index.md", "log.md")}}
|
|
743
|
+
if duplicate_warnings and not ignore_duplicate_warning:
|
|
744
|
+
receipt["duplicate_concept_warning"] = duplicate_warnings
|
|
745
|
+
receipt["warnings"] = duplicate_warnings
|
|
746
|
+
return receipt
|
|
642
747
|
|
|
643
748
|
def acknowledge(self, page_id, note, expected_revision=None):
|
|
644
749
|
"""Re-publish a page unchanged, recording that its sources were re-reviewed.
|
|
@@ -743,12 +848,67 @@ class WikiStore:
|
|
|
743
848
|
raise ValueError("Wiki revision belongs to a different page")
|
|
744
849
|
if revision is None or record["revision"] == revision:
|
|
745
850
|
issues = self._issues(record, index, resolve_paths=True)
|
|
851
|
+
self._log_metric("wiki_get", page_id=page_id, hit_count=1, reused=True)
|
|
746
852
|
return {**record, "artifacts": self._artifacts(pointer), "validation": self._validation(issues)}
|
|
747
853
|
if record["previous"] is not None and record["previous"]["revision"] >= record["revision"]:
|
|
748
854
|
raise ValueError("Invalid Wiki revision history")
|
|
749
855
|
pointer = record["previous"]
|
|
750
856
|
raise ValueError("Unknown Wiki page or revision")
|
|
751
857
|
|
|
858
|
+
def _detect_duplicate_concepts(self, page_id, title, urls, index):
|
|
859
|
+
try:
|
|
860
|
+
return WikiSearchIndex(self._path("search.sqlite3")).detect_duplicates(
|
|
861
|
+
index, self._revision, page_id, title, urls
|
|
862
|
+
)
|
|
863
|
+
except (OSError, ValueError, sqlite3.Error):
|
|
864
|
+
from archive import SourceArchive
|
|
865
|
+
norm_new = WikiSearchIndex.normalize(title) if title else ""
|
|
866
|
+
tokens_new = set(re.findall(r"\w+", title.lower())) if title else set()
|
|
867
|
+
duplicates = []
|
|
868
|
+
for other_id, pointer in index.get("pages", {}).items():
|
|
869
|
+
if other_id == page_id:
|
|
870
|
+
continue
|
|
871
|
+
other_title = pointer.get("title", "")
|
|
872
|
+
norm_other = WikiSearchIndex.normalize(other_title) if other_title else ""
|
|
873
|
+
tokens_other = set(re.findall(r"\w+", other_title.lower())) if other_title else set()
|
|
874
|
+
reasons = []
|
|
875
|
+
similar_title = False
|
|
876
|
+
if norm_new and norm_other:
|
|
877
|
+
if norm_new == norm_other:
|
|
878
|
+
similar_title = True
|
|
879
|
+
elif (norm_new in norm_other or norm_other in norm_new) and min(len(norm_new), len(norm_other)) >= 4:
|
|
880
|
+
similar_title = True
|
|
881
|
+
elif tokens_new and tokens_other:
|
|
882
|
+
union = tokens_new | tokens_other
|
|
883
|
+
if union and (len(tokens_new & tokens_other) / len(union)) >= 0.7:
|
|
884
|
+
similar_title = True
|
|
885
|
+
if similar_title:
|
|
886
|
+
reasons.append("similar_title")
|
|
887
|
+
matched_urls = []
|
|
888
|
+
if urls:
|
|
889
|
+
try:
|
|
890
|
+
record = self._revision(pointer)
|
|
891
|
+
rec_urls = {
|
|
892
|
+
SourceArchive._key(c["url"]) for ev in record.get("evidence", [])
|
|
893
|
+
for c in ev.get("citations", [])
|
|
894
|
+
if c.get("url") and (c.get("provenance") or {}).get("kind") not in ("local_document", "external_report")
|
|
895
|
+
}
|
|
896
|
+
matched_urls = sorted(urls & rec_urls)
|
|
897
|
+
if matched_urls:
|
|
898
|
+
reasons.append("overlapping_urls")
|
|
899
|
+
except Exception:
|
|
900
|
+
pass
|
|
901
|
+
if reasons:
|
|
902
|
+
duplicates.append({
|
|
903
|
+
"page_id": other_id,
|
|
904
|
+
"title": other_title,
|
|
905
|
+
"reasons": reasons,
|
|
906
|
+
"overlapping_urls": matched_urls,
|
|
907
|
+
"candidate_merge_target": other_id,
|
|
908
|
+
"suggestion": f"Consider reviewing or merging with existing Wiki page '{other_id}' ({other_title})."
|
|
909
|
+
})
|
|
910
|
+
return duplicates
|
|
911
|
+
|
|
752
912
|
def _affected(self, index, urls):
|
|
753
913
|
try:
|
|
754
914
|
return WikiSearchIndex(self._path("search.sqlite3")).affected(index, self._revision, urls)
|
|
@@ -907,7 +1067,8 @@ class WikiStore:
|
|
|
907
1067
|
query = self._text(query, "Wiki query", 500)
|
|
908
1068
|
ResearchSessions._integer(offset, "offset", 0, 10000000)
|
|
909
1069
|
ResearchSessions._integer(limit, "limit", 1, 100)
|
|
910
|
-
|
|
1070
|
+
extra_aliases = self.settings.load().get("wiki", {}).get("aliases", [])
|
|
1071
|
+
groups = WikiSearchIndex.terms(query, extra_aliases=extra_aliases)
|
|
911
1072
|
index, matches, excluded, cache = self._index(), [], [], {}
|
|
912
1073
|
search_index = "indexed"
|
|
913
1074
|
try:
|
|
@@ -922,16 +1083,24 @@ class WikiStore:
|
|
|
922
1083
|
except (OSError, ValueError, KeyError, TypeError):
|
|
923
1084
|
invalid.append(page_id)
|
|
924
1085
|
excluded.extend(invalid)
|
|
1086
|
+
doc_freq = {}
|
|
1087
|
+
for candidate in candidates:
|
|
1088
|
+
cand_text = (candidate["title"] + "\n" + candidate["body"] + "\n" + candidate.get("aliases", "")).casefold()
|
|
1089
|
+
cand_squashed = WikiSearchIndex.normalize(cand_text)
|
|
1090
|
+
for variants in groups:
|
|
1091
|
+
matched_variant = next((variant for variant in variants if WikiSearchIndex.normalize(variant) in cand_squashed), None)
|
|
1092
|
+
if matched_variant:
|
|
1093
|
+
doc_freq[matched_variant] = doc_freq.get(matched_variant, 0) + 1
|
|
925
1094
|
checked = 0
|
|
926
1095
|
for candidate in candidates:
|
|
927
|
-
match = WikiSearchIndex.match(candidate, query, groups)
|
|
1096
|
+
match = WikiSearchIndex.match(candidate, query, groups, doc_freq=doc_freq)
|
|
928
1097
|
if match is None:
|
|
929
1098
|
continue
|
|
930
1099
|
page_id = candidate["page_id"]
|
|
931
1100
|
pointer = index["pages"][page_id]
|
|
932
1101
|
try:
|
|
933
1102
|
record = self._revision(pointer)
|
|
934
|
-
match = WikiSearchIndex.match(WikiSearchIndex.row(page_id, pointer, record), query, groups)
|
|
1103
|
+
match = WikiSearchIndex.match(WikiSearchIndex.row(page_id, pointer, record), query, groups, doc_freq=doc_freq)
|
|
935
1104
|
if match is None:
|
|
936
1105
|
continue
|
|
937
1106
|
checked += 1
|
|
@@ -949,6 +1118,7 @@ class WikiStore:
|
|
|
949
1118
|
except (OSError, ValueError, KeyError, TypeError):
|
|
950
1119
|
excluded.append(page_id)
|
|
951
1120
|
matches.sort(key=lambda row: (-row[0], row[1]["page_id"]))
|
|
1121
|
+
self._log_metric("wiki_search", hit_count=len(matches))
|
|
952
1122
|
return {"query": query, "results": [row[1] for row in matches[offset:offset + limit]], "total": len(matches),
|
|
953
1123
|
"next_offset": offset + limit if offset + limit < len(matches) else None,
|
|
954
1124
|
"excluded_stale_pages": len(excluded), "excluded_page_ids": excluded[:100],
|
|
@@ -1052,7 +1222,88 @@ class WikiStore:
|
|
|
1052
1222
|
lines.extend(["", "[Saved text](%s) · [Manifest](%s) · [Response](%s) · SHA-256: `%s`" %
|
|
1053
1223
|
(artifacts["body_file"], artifacts["manifest_file"], artifacts["response_file"],
|
|
1054
1224
|
citation["source_sha256"]), ""])
|
|
1055
|
-
lines.extend(["## Update record", "", record["change_note"], ""])
|
|
1056
1225
|
if record["previous"]:
|
|
1057
1226
|
lines.extend(["[Previous revision](%s)" % link(self._path(record["previous"]["markdown_file"])), ""])
|
|
1058
1227
|
return "\n".join(lines) + "\n"
|
|
1228
|
+
|
|
1229
|
+
def _log_metric(self, action, page_id=None, hit_count=None, reused=None, page_ids=None):
|
|
1230
|
+
try:
|
|
1231
|
+
path = self._path("metrics.jsonl")
|
|
1232
|
+
entry = {
|
|
1233
|
+
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
1234
|
+
"action": action,
|
|
1235
|
+
"page_id": page_id,
|
|
1236
|
+
"hit_count": hit_count,
|
|
1237
|
+
"reused": reused,
|
|
1238
|
+
}
|
|
1239
|
+
if page_ids:
|
|
1240
|
+
entry["page_ids"] = list(dict.fromkeys(page_ids))
|
|
1241
|
+
with path.open("a", encoding="utf-8") as stream:
|
|
1242
|
+
stream.write(json.dumps(entry, ensure_ascii=False) + "\n")
|
|
1243
|
+
except Exception:
|
|
1244
|
+
pass
|
|
1245
|
+
|
|
1246
|
+
def metrics_summary(self, days=30):
|
|
1247
|
+
"""Summarize append-only Wiki usage metrics over the past N days."""
|
|
1248
|
+
ResearchSessions._integer(days, "days", 1, 3650)
|
|
1249
|
+
now = datetime.now(timezone.utc)
|
|
1250
|
+
cutoff = now - timedelta(days=days)
|
|
1251
|
+
path = self._path("metrics.jsonl")
|
|
1252
|
+
summary = {
|
|
1253
|
+
"days": days,
|
|
1254
|
+
"since": cutoff.isoformat(),
|
|
1255
|
+
"total_hits": 0,
|
|
1256
|
+
"searches": 0,
|
|
1257
|
+
"reads": 0,
|
|
1258
|
+
"writes": 0,
|
|
1259
|
+
"plan_preflights": 0,
|
|
1260
|
+
"reused_pages": [],
|
|
1261
|
+
"actions": {},
|
|
1262
|
+
}
|
|
1263
|
+
if not path.is_file():
|
|
1264
|
+
return summary
|
|
1265
|
+
reused_set = set()
|
|
1266
|
+
try:
|
|
1267
|
+
with path.open("r", encoding="utf-8") as stream:
|
|
1268
|
+
for line in stream:
|
|
1269
|
+
line = line.strip()
|
|
1270
|
+
if not line:
|
|
1271
|
+
continue
|
|
1272
|
+
try:
|
|
1273
|
+
record = json.loads(line)
|
|
1274
|
+
except Exception:
|
|
1275
|
+
continue
|
|
1276
|
+
ts_str = record.get("timestamp")
|
|
1277
|
+
if not ts_str:
|
|
1278
|
+
continue
|
|
1279
|
+
try:
|
|
1280
|
+
ts = datetime.fromisoformat(ts_str)
|
|
1281
|
+
except Exception:
|
|
1282
|
+
continue
|
|
1283
|
+
if ts.tzinfo is None:
|
|
1284
|
+
ts = ts.replace(tzinfo=timezone.utc)
|
|
1285
|
+
if ts < cutoff:
|
|
1286
|
+
continue
|
|
1287
|
+
action = record.get("action", "unknown")
|
|
1288
|
+
summary["actions"][action] = summary["actions"].get(action, 0) + 1
|
|
1289
|
+
hit_count = record.get("hit_count") or 0
|
|
1290
|
+
if action == "wiki_search":
|
|
1291
|
+
summary["searches"] += 1
|
|
1292
|
+
summary["total_hits"] += hit_count
|
|
1293
|
+
elif action == "wiki_get":
|
|
1294
|
+
summary["reads"] += 1
|
|
1295
|
+
elif action == "wiki_write":
|
|
1296
|
+
summary["writes"] += 1
|
|
1297
|
+
elif action == "plan_research":
|
|
1298
|
+
summary["plan_preflights"] += 1
|
|
1299
|
+
summary["total_hits"] += hit_count
|
|
1300
|
+
if record.get("reused"):
|
|
1301
|
+
if record.get("page_id"):
|
|
1302
|
+
reused_set.add(record["page_id"])
|
|
1303
|
+
for pid in record.get("page_ids", []):
|
|
1304
|
+
reused_set.add(pid)
|
|
1305
|
+
except Exception:
|
|
1306
|
+
pass
|
|
1307
|
+
summary["reused_pages"] = sorted(reused_set)
|
|
1308
|
+
return summary
|
|
1309
|
+
|