bookmark-research 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/plugins/marketplace.json +1 -1
- package/.codex-plugin/plugin.json +4 -1
- package/README.md +15 -9
- package/README.zh.md +16 -9
- package/config/providers.json +26 -0
- package/config/settings.example.json +36 -0
- package/docs/bookmark-research-architecture.md +2 -2
- package/docs/canvas-tree-reading-acceptance.md +98 -0
- package/docs/configuration.en.md +70 -0
- package/docs/configuration.md +70 -0
- package/docs/details.en.md +10 -6
- package/docs/details.md +3 -3
- package/docs/harness-compatibility.md +1 -1
- package/docs/installation.en.md +4 -2
- package/docs/installation.md +4 -2
- package/docs/instructions.en.md +1 -0
- package/docs/instructions.md +1 -0
- package/docs/user-guide.en.md +1 -1
- package/docs/user-guide.md +1 -1
- package/hosts/dsh/plugin.js +3 -0
- package/package.json +1 -1
- package/scripts/bench_sync.py +191 -0
- package/scripts/bench_tools_list.py +129 -0
- package/scripts/evaluate_ladder.py +149 -0
- package/scripts/export_bundle.py +2 -0
- package/scripts/host_install.py +4 -1
- package/scripts/install.py +120 -28
- package/skills/bookmark-research/SKILL.md +28 -9
- package/skills/bookmark-research/references/package-semantics.md +56 -6
- package/skills/bookmark-research/references/research-workflow.md +5 -2
- package/skills/bookmark-research/references/settings-and-archive.md +5 -3
- package/skills/bookmark-research/references/wiki.md +3 -1
- package/skills/bookmark-research/references/zh/package-semantics.md +56 -6
- package/skills/bookmark-research/references/zh/research-workflow.md +5 -2
- package/skills/bookmark-research/references/zh/settings-and-archive.md +5 -3
- package/skills/bookmark-research/references/zh/skill-guide.md +20 -8
- package/skills/bookmark-research/references/zh/wiki.md +3 -1
- package/src/archive.py +100 -0
- package/src/bookmark_index.py +808 -41
- package/src/browser_client.py +342 -0
- package/src/cli.py +298 -14
- package/src/credentials.py +2 -1
- package/src/fetch_cache.py +60 -0
- package/src/grok_search.py +272 -0
- package/src/jobs.py +349 -0
- package/src/junk.py +33 -0
- package/src/locate.py +862 -0
- package/src/mcp_dispatch.py +22 -6
- package/src/mcp_server.py +573 -23
- package/src/netscape.py +124 -0
- package/src/onboarding.py +59 -9
- package/src/perplexity.py +78 -0
- package/src/raw_library.py +183 -45
- package/src/readiness.py +40 -5
- package/src/ref_codec.py +70 -0
- package/src/research.py +125 -12
- package/src/research_plan.py +11 -1
- package/src/routing.py +5 -2
- package/src/search_results.py +80 -2
- package/src/settings.py +61 -16
- package/src/source_inputs.py +109 -3
- package/src/source_manager.py +71 -12
- package/src/source_versions.py +20 -2
- package/src/source_watcher.py +4 -2
- package/src/storage.py +60 -0
- package/src/url_classify.py +45 -0
- package/src/web_search.py +106 -10
- package/src/wiki.py +269 -18
- package/src/wiki_search.py +119 -27
package/src/mcp_server.py
CHANGED
|
@@ -5,16 +5,22 @@ implements initialization, ping, tools/list and tools/call, not resources,
|
|
|
5
5
|
prompts, sampling, tasks, HTTP transports or arbitrary SQL/command execution.
|
|
6
6
|
"""
|
|
7
7
|
|
|
8
|
+
import copy
|
|
8
9
|
import hashlib
|
|
9
10
|
import json
|
|
10
11
|
import math
|
|
11
12
|
import os
|
|
13
|
+
import re
|
|
12
14
|
import sys
|
|
13
15
|
import threading
|
|
16
|
+
from collections import OrderedDict
|
|
14
17
|
from pathlib import Path
|
|
15
18
|
|
|
16
19
|
from release import release_version
|
|
17
20
|
from settings import Settings
|
|
21
|
+
from credentials import Credentials
|
|
22
|
+
from locate import continuation_scope, locate as locate_core, locate_ref
|
|
23
|
+
from ref_codec import resolve as resolve_ref
|
|
18
24
|
|
|
19
25
|
|
|
20
26
|
PROTOCOLS = ("2024-11-05", "2025-03-26", "2025-06-18", "2025-11-25")
|
|
@@ -47,16 +53,18 @@ SETTINGS_SCHEMA = _object_schema({
|
|
|
47
53
|
"strategy": {"type": "string", "enum": ["hedged", "staggered", "sequential", "wait_all"]},
|
|
48
54
|
"grace_seconds": {"type": ["number", "null"], "minimum": 0, "maximum": 60},
|
|
49
55
|
"deadline_seconds": {"type": ["number", "null"], "minimum": 0, "maximum": 60},
|
|
50
|
-
"cache_seconds": {"type": "integer", "minimum": 0, "maximum": 3600
|
|
51
|
-
|
|
52
|
-
|
|
56
|
+
"cache_seconds": {"type": "integer", "minimum": 0, "maximum": 3600,
|
|
57
|
+
"description": "Search result cache TTL in seconds. 0 disables cache (always fresh network requests). Maximum 3600s (1 hour)."}}),
|
|
58
|
+
"fetch": _object_schema({"provider": {"type": "string", "enum": list(Settings.FETCH_PROVIDERS)},
|
|
59
|
+
"providers": dict(_array_schema({"type": "string", "enum": list(Settings.FETCH_PROVIDERS)}, len(Settings.FETCH_PROVIDERS), 1), uniqueItems=True),
|
|
53
60
|
"max_characters": {"type": "integer", "minimum": 100, "maximum": 100000},
|
|
54
61
|
"max_total_characters": {"type": "integer", "minimum": 100, "maximum": 800000},
|
|
55
62
|
"fair_share_min": {"type": "integer", "minimum": 0, "maximum": 100000},
|
|
56
63
|
"strategy": {"type": "string", "enum": ["sequential", "staggered", "hedged"]},
|
|
57
64
|
"grace_seconds": {"type": ["number", "null"], "minimum": 0, "maximum": 60},
|
|
58
65
|
"deadline_seconds": {"type": ["number", "null"], "minimum": 0, "maximum": 60},
|
|
59
|
-
"reuse_seconds": {"type": "integer", "minimum": 0, "maximum": 86400
|
|
66
|
+
"reuse_seconds": {"type": "integer", "minimum": 0, "maximum": 86400,
|
|
67
|
+
"description": "Fetch page body reuse window in seconds. 0 disables reuse (always re-fetch). Default 0; up to 86400s (24 hours)."}}),
|
|
60
68
|
"server": _object_schema({"network_workers": {"type": "integer", "minimum": 1, "maximum": 16},
|
|
61
69
|
"max_pending_calls": {"type": "integer", "minimum": 1, "maximum": 64}}),
|
|
62
70
|
"archive": _object_schema({"enabled": {"type": "boolean"}, "directory": _text_schema(4096)}),
|
|
@@ -70,9 +78,22 @@ SETTINGS_SCHEMA = _object_schema({
|
|
|
70
78
|
"max_tool_calls": {"type": "integer", "minimum": 1, "maximum": 1000},
|
|
71
79
|
"allow_untested_model": {"type": "boolean"}}),
|
|
72
80
|
"parallel": _object_schema({"processor": _text_schema(64)})}),
|
|
81
|
+
"custom_agentic": _object_schema({
|
|
82
|
+
"enabled": {"type": "boolean"}, "base_url": _text_schema(1024),
|
|
83
|
+
"model": _text_schema(192),
|
|
84
|
+
"api_mode": {"type": "string", "enum": ["chat_completions", "responses"]},
|
|
85
|
+
"search_mode": {"type": "string", "enum": ["prompt", "native"]}}),
|
|
86
|
+
"browser": _object_schema({
|
|
87
|
+
"enabled": {"type": "boolean"},
|
|
88
|
+
"driver": {"type": "string", "enum": ["auto", "playwright", "cdp", "mock"]},
|
|
89
|
+
"cdp_url": _text_schema(1024),
|
|
90
|
+
"headless": {"type": "boolean"},
|
|
91
|
+
"timeout_seconds": {"type": "integer", "minimum": 1, "maximum": 60},
|
|
92
|
+
"max_concurrency": {"type": "integer", "minimum": 1, "maximum": 8}}),
|
|
73
93
|
"wiki": _object_schema({"directory": _text_schema(4096),
|
|
74
94
|
"max_source_age_days": {"type": "integer", "minimum": 1, "maximum": 3650},
|
|
75
|
-
"after_research": {"type": "string", "enum": ["suggest", "auto", "off"]}
|
|
95
|
+
"after_research": {"type": "string", "enum": ["suggest", "auto", "off"]},
|
|
96
|
+
"aliases": _array_schema(_array_schema(_text_schema(100), 50, 1), 50)}),
|
|
76
97
|
"wiki_preflight": {"type": "boolean"},
|
|
77
98
|
"freshness_days": {"type": "integer", "minimum": 1, "maximum": 3650},
|
|
78
99
|
"plan": _object_schema({"max_characters": {"type": "integer", "minimum": 1000, "maximum": 50000},
|
|
@@ -105,11 +126,35 @@ TOOL_SCHEMAS = {
|
|
|
105
126
|
["source_id"]),
|
|
106
127
|
"source_merge": _object_schema({"source_id": IDENTIFIER, "into_source_id": IDENTIFIER,
|
|
107
128
|
"confirm": {"type": "boolean", "default": False}}, ["source_id", "into_source_id"]),
|
|
129
|
+
"locate": _object_schema({
|
|
130
|
+
"text": _text_schema(2000),
|
|
131
|
+
"url": _text_schema(8192),
|
|
132
|
+
"in": _array_schema({"type": "string", "enum": ["canvas", "wiki", "archive"]}, 3, 1),
|
|
133
|
+
"source_id": IDENTIFIER,
|
|
134
|
+
"section": IDENTIFIER,
|
|
135
|
+
"folder_id": {**IDENTIFIER, "description": "Filter matches to a folder subtree; with source_id and no text/url, list this folder's direct children."},
|
|
136
|
+
"package_path": {**IDENTIFIER, "description": "Import this local Canvas package, single section JSON, or bookmark HTML export (snapshot) and query it; omit to use an already-imported source_id."},
|
|
137
|
+
"tags": _array_schema(_text_schema(), 100),
|
|
138
|
+
"tag_colors": _array_schema(_text_schema(), 20),
|
|
139
|
+
"has_note": {"type": "boolean", "description": "Filter for bookmarks with (true) or without (false) a non-empty note. Usable alone, without text/url, to enumerate."},
|
|
140
|
+
"has_tags": {"type": "boolean", "description": "Filter for bookmarks with (true) or without (false) one or more tags. Usable alone, without text/url, to enumerate."},
|
|
141
|
+
"metadata_match": {"type": "string", "enum": ["all", "any"], "default": "all"},
|
|
142
|
+
"limit": {"type": "integer", "minimum": 1, "maximum": 50, "default": 8},
|
|
143
|
+
"offset": {"type": "integer", "minimum": 0, "default": 0},
|
|
144
|
+
"handle": _text_schema(16384),
|
|
145
|
+
"refresh": {"type": "boolean", "default": False},
|
|
146
|
+
}),
|
|
108
147
|
"search_bookmarks": _object_schema({
|
|
109
148
|
"source_id": IDENTIFIER, "targets": _array_schema(_text_schema(), 100),
|
|
110
149
|
"section": IDENTIFIER, "group_id": IDENTIFIER, "folder_id": IDENTIFIER,
|
|
111
150
|
"tags": _array_schema(_text_schema(), 100),
|
|
112
151
|
"tag_colors": _array_schema(_text_schema(), 20),
|
|
152
|
+
"has_note": {"type": "boolean", "description": "Filter for bookmarks with (true) or without (false) a non-empty note. Usable alone, without text/url, to enumerate."},
|
|
153
|
+
"has_tags": {"type": "boolean", "description": "Filter for bookmarks with (true) or without (false) one or more tags. Usable alone, without text/url, to enumerate."},
|
|
154
|
+
"metadata_match": {"type": "string", "enum": ["all", "any"], "default": "all",
|
|
155
|
+
"description": "Combine has_note and has_tags with AND (all) or OR (any)."},
|
|
156
|
+
"sort_by": {"type": "string", "enum": ["default", "added_at_desc", "added_at_asc"], "default": "default",
|
|
157
|
+
"description": "Sort by canvas order or indexed added_at date. Unknown dates remain last."},
|
|
113
158
|
"item_types": {"type": "array", "items": {"type": "string", "enum": ["bookmark", "folder"]},
|
|
114
159
|
"minItems": 1, "maxItems": 2, "uniqueItems": True,
|
|
115
160
|
"description": "Defaults to bookmark; pass [\"folder\"] to enumerate folders."},
|
|
@@ -117,8 +162,16 @@ TOOL_SCHEMAS = {
|
|
|
117
162
|
"offset": {"type": "integer", "minimum": 0, "maximum": 1000000000, "default": 0},
|
|
118
163
|
"count_only": {"type": "boolean", "default": False},
|
|
119
164
|
"compact": {"type": "boolean", "default": False},
|
|
165
|
+
"score": {"type": "boolean", "default": False,
|
|
166
|
+
"description": "Return non-binding ranking score and field_hits hints; literal match set is preserved."},
|
|
120
167
|
"refresh": {"type": "boolean", "default": True},
|
|
121
168
|
}, ["source_id"]),
|
|
169
|
+
"lookup_url": _object_schema({
|
|
170
|
+
"url": _text_schema(8192),
|
|
171
|
+
"source_id": IDENTIFIER,
|
|
172
|
+
"limit": {"type": "integer", "minimum": 1, "maximum": 1000, "default": 100},
|
|
173
|
+
"offset": {"type": "integer", "minimum": 0, "maximum": 1000000000, "default": 0},
|
|
174
|
+
}, ["url"]),
|
|
122
175
|
"get_context": _object_schema({
|
|
123
176
|
"source_id": IDENTIFIER, "section": IDENTIFIER, "group_id": IDENTIFIER,
|
|
124
177
|
"item_id": IDENTIFIER, "refresh": {"type": "boolean", "default": True},
|
|
@@ -127,6 +180,15 @@ TOOL_SCHEMAS = {
|
|
|
127
180
|
"limit": {"type": "integer", "minimum": 1, "maximum": 1000, "default": 200},
|
|
128
181
|
"compact": {"type": "boolean", "default": False},
|
|
129
182
|
}, ["source_id"]),
|
|
183
|
+
"traverse_graph": _object_schema({
|
|
184
|
+
"source_id": IDENTIFIER, "section": IDENTIFIER, "group_id": IDENTIFIER,
|
|
185
|
+
"item_id": IDENTIFIER, "node_id": IDENTIFIER,
|
|
186
|
+
"max_depth": {"type": "integer", "minimum": 1, "maximum": 10, "default": 2},
|
|
187
|
+
"direction": {"type": "string", "enum": ["all", "out", "in"], "default": "all"},
|
|
188
|
+
"limit": {"type": "integer", "minimum": 1, "maximum": 1000, "default": 100},
|
|
189
|
+
"offset": {"type": "integer", "minimum": 0, "maximum": 1000000000, "default": 0},
|
|
190
|
+
"refresh": {"type": "boolean", "default": True},
|
|
191
|
+
}, ["source_id"]),
|
|
130
192
|
"plan_research": _object_schema({
|
|
131
193
|
"source_id": IDENTIFIER, "question": _text_schema(2000),
|
|
132
194
|
"max_cards": {"type": "integer", "minimum": 1, "maximum": 3, "default": 3},
|
|
@@ -153,9 +215,38 @@ TOOL_SCHEMAS = {
|
|
|
153
215
|
"providers": PROVIDERS,
|
|
154
216
|
"limit_per_target": {"type": "integer", "minimum": 1, "maximum": 20},
|
|
155
217
|
}, ["targets"]),
|
|
218
|
+
"merge_search_results": _object_schema({
|
|
219
|
+
"batches": _array_schema(_object_schema({
|
|
220
|
+
"target": _text_schema(2000),
|
|
221
|
+
"provider": _text_schema(100),
|
|
222
|
+
"query": _text_schema(2000),
|
|
223
|
+
"status": {"type": "string", "enum": ["ok", "error"]},
|
|
224
|
+
"results": _array_schema(_object_schema({
|
|
225
|
+
"url": _text_schema(2048),
|
|
226
|
+
"title": {"type": "string", "maxLength": 500},
|
|
227
|
+
"snippet": {"type": "string", "maxLength": 100000},
|
|
228
|
+
"content": {"type": "string", "maxLength": 100000},
|
|
229
|
+
"rank": {"type": "integer", "minimum": 1, "maximum": 100000}
|
|
230
|
+
}, ["url"]), 500),
|
|
231
|
+
"error": {"type": "string", "maxLength": 2000}
|
|
232
|
+
}, ["target", "provider", "query", "status"]), 50, 1),
|
|
233
|
+
"targets": _array_schema(_text_schema(2000), 50),
|
|
234
|
+
"limit_per_target": {"type": "integer", "minimum": 1, "maximum": 50},
|
|
235
|
+
"archive": {"type": "boolean", "default": False}
|
|
236
|
+
}, ["batches"]),
|
|
237
|
+
"agentic_search": _object_schema({
|
|
238
|
+
"query": _text_schema(12000),
|
|
239
|
+
"provider": {"type": "string", "enum": ["grok", "custom"], "default": "grok"},
|
|
240
|
+
"include_x": {"type": "boolean", "default": False,
|
|
241
|
+
"description": "Also search public X posts. This may incur additional provider charges."},
|
|
242
|
+
"rewrite_query": {"type": "boolean", "default": False,
|
|
243
|
+
"description": "Decompose and reformulate search queries before searching."},
|
|
244
|
+
"max_iterations": {"type": "integer", "minimum": 1, "maximum": 5, "default": 1,
|
|
245
|
+
"description": "Budget bound for multi-step agentic refinement iterations (1-5)."}
|
|
246
|
+
}, ["query"]),
|
|
156
247
|
"fetch_web": _object_schema({
|
|
157
248
|
"urls": _array_schema(_text_schema(8192), 16, 1),
|
|
158
|
-
"provider": {"type": "string", "enum": list(
|
|
249
|
+
"provider": {"type": "string", "enum": list(Settings.FETCH_PROVIDERS)},
|
|
159
250
|
"archive": {"type": "boolean"},
|
|
160
251
|
"max_characters": {"type": "integer", "minimum": 100, "maximum": 100000},
|
|
161
252
|
"max_total_characters": {"type": "integer", "minimum": 100, "maximum": 800000,
|
|
@@ -178,6 +269,8 @@ TOOL_SCHEMAS = {
|
|
|
178
269
|
"description": "Occurrences listed per merged body. Each result reports occurrence_count, occurrences_truncated and occurrences_next_offset."},
|
|
179
270
|
"occurrence_offset": {"type": "integer", "minimum": 0, "maximum": 10000000, "default": 0,
|
|
180
271
|
"description": "Continue a body's occurrence list from this offset."},
|
|
272
|
+
"expand_versions": {"type": "boolean", "default": False,
|
|
273
|
+
"description": "When true, expand all version captures by SHA-256 instead of grouping by URL."},
|
|
181
274
|
}, ["query"]),
|
|
182
275
|
}
|
|
183
276
|
|
|
@@ -189,15 +282,20 @@ TOOL_DESCRIPTIONS = {
|
|
|
189
282
|
"source_history": "List saved source versions and managed snapshot paths, including partial-import data. Reimport a snapshot to recover it. Does not refresh sources or rewrite research inventories.",
|
|
190
283
|
"source_remove": "Preview deletion of an imported source, index rows and managed snapshots; confirm=true performs it. Frozen research inventories remain, but item resolution stops and citing Wiki pages need review. For identical duplicates, prefer source_merge.",
|
|
191
284
|
"source_merge": "Preview merging an identical source into into_source_id; confirm=true transfers path aliases and drops the duplicate. Synchronized content versions must match; synchronize legacy sources first.",
|
|
192
|
-
"
|
|
285
|
+
"locate": "Find matches in one local scope: Canvas bookmarks, Wiki revisions, or archived captures. Canvas returns direct title/URL/note/tag matches before inherited folder_path matches; inherited-only items fold into the nearest matching folder with a count and up to three samples. Three query shapes: text or url; folder_id with source_id and no text/url lists that folder's direct children; metadata filters (tags/tag_colors/has_note/has_tags) with no text/url/folder enumerate matching bookmarks newest-first. If total exceeds limit, continue with more.handle and more.offset; handles survive CLI/MCP process restarts. Pass text or url, not both.",
|
|
286
|
+
"search_bookmarks": "Literal all-term search with independent target totals/pages; canvas_matches reports text/group/edge hits separately. compact=true returns shared rows plus ordered result_refs per page; row raw_json/metadata then contain extra/conflicting fields. Default false keeps full rows. score=true returns non-binding ranking score and field_hits hints while preserving literal match set. Keep instances distinct from unique_urls and shared copies. count_only or limit=0 omits rows; item_types=[\"folder\"] includes synthetic roots. Live inputs refresh by default; inspect source.state. refresh=false reads the saved index.",
|
|
287
|
+
"lookup_url": "Reverse index: find all bookmark item instances for an exact URL across sections and sources, with chronological added_at, parent folders and copy anchor cards. source_id is optional.",
|
|
193
288
|
"get_context": "Read section headers, bookmark metadata, folder ancestry, groups and directed edges. Defaults to 200 rows per collection; follow each collection's next_offset with the same scope. Counts cover the full scope. Copy anchors share their primary tree. Live inputs refresh by default. compact=true drops raw_json, description_md and revision; pair it with a collection to reach the smallest budget.",
|
|
289
|
+
"traverse_graph": "Multi-hop graph traversal along canvas edges and group memberships starting from a node, section, group or bookmark item. Returns the impact surface: reached sections, groups, text annotations and edge paths up to max_depth hops with deterministic sorting and pagination.",
|
|
194
290
|
"plan_research": "Read local candidates for a question: matching bookmarks and canvas text/group/edge labels, independently searched Wiki pages and archived sources, and source dates. source_id is optional. max_cards accepts 1–3 (default 3); max_urls accepts 1–8 (default 8). Larger values are rejected. Scores rank candidates without rejecting them; the host decides relevance, depth and whether to search. question_urls lists known URLs to read directly. No network or research task is started.",
|
|
195
291
|
"discovery_bundle": "Assemble the route-② evidence bundle and decide the budget. Pass every source the host collected: only content_state=usable pages become evidence, the rest are listed with the reason they cannot be cited. The bundle is built at the richest level that fits max_characters (excerpts, then digest, then a plain URL list) and always keeps the URL list, so it degrades instead of returning nothing. steps, fetches, variants and elapsed_seconds are the host's own counters, and the plugin owns the decision: stop when budget.continue is false (steps ≤ 6, fetches ≤ 8, variants ≤ 3, 90 seconds). Cite evidence[].id in the answer; never cite a URL that has no evidence entry.",
|
|
196
292
|
"index_status": "Read source modes, snapshots, counts, input availability, pending changes and monitor errors. No refresh or web fetch; unchecked means a live directory needs a recent check.",
|
|
197
293
|
"search_web": "Discover sources for public queries with saved primary providers, then fallbacks for unresolved queries. search.strategy selects hedged (default), staggered, sequential or wait_all. Hedged/staggered primary search has a 10-second default search.deadline_seconds; routing lists providers abandoned at the deadline. Raise this setting or set it to null with update_settings when slower providers need more time. Provider failures remain visible in batches; quota_exhausted providers enter cooldown (using provider retry_after when supplied), while other configured fallbacks are still attempted. Explicit providers restrict the call. Targets accept target/query objects or plain query strings. Results are discovery snippets; pages are not fetched. Each result carries provider_count and corroborated: two or more providers returning one URL means link overlap, never confirmation of its content.",
|
|
294
|
+
"merge_search_results": "Fuse and deduplicate search results from host-native tools or other MCPs. Deduplicates by URL and text SHA-256, computes RRF scores, preserves provider provenance, and optionally archives extracts into knowledge storage.",
|
|
295
|
+
"agentic_search": "Optional model answer/search channel, separate from search_web URL fusion. provider=grok (default) uses xAI Responses and XAI_API_KEY; include_x enables X Search. provider=custom uses the configured custom_agentic OpenAI-compatible endpoint and CUSTOM_AGENTIC_API_KEY; native web search is only requested for Responses mode when explicitly configured. Prompt mode cannot guarantee the upstream actually searches. Citations are not independently verified. Use only when selected by the user or workflow.",
|
|
198
296
|
"fetch_web": "Read known URLs directly (up to 16 per call) with per-page and total text budgets. Continue saved text using capture_id and offset without refetching. Omit provider for the saved fallback order (fetch.strategy overlaps a stalled provider); specifying one restricts it. raw returns provider envelopes; archive=false skips saving. Check page identity before citing. Every page carries content_state, the page's own shape: usable, content_insufficient (under 400 characters or an empty-page marker), js_shell_suspected (a title and navigation but no prose), challenge_blocked, login_required (an SSO or sign-in wall), soft_404 or paywall_suspected; only usable pages are worth citing, and unavailable means no body was obtained. published_at is a day-precision date with published_at_source (provider_metadata, structured_metadata or body_date) and published_at_confidence; a bare date in the text is low confidence and never later than the fetch day. A URL with no body carries liveness: dead only for 404/410, otherwise unknown, and dead_urls lists them.",
|
|
199
|
-
"search_providers": "Describe retrieval capabilities and per-operation auth. probe=true checks MCP catalogs, not successful retrieval
|
|
200
|
-
"search_archive": "Search saved page text across local archives and research tasks using literal terms. No network.
|
|
297
|
+
"search_providers": "Describe configured retrieval capabilities and per-operation auth. Includes Exa, Parallel, Tavily, Jina, DeepSeek and optional Perplexity structured Search. probe=true checks remote MCP catalogs, not successful retrieval; HTTP APIs (Jina, DeepSeek, Perplexity) are reported as not probed. Perplexity is search-only and requires PERPLEXITY_API_KEY. Configure credentials through CLI setup, never chat.",
|
|
298
|
+
"search_archive": "Search saved page text across local archives and research tasks using literal terms. No network. By default, results are grouped by URL, reporting version counts, time span, and distinct versions. Set expand_versions=true to view separate captures merged by body hash. Occurrences have their own pagination. Hash-mismatched bodies are skipped and counted; the index refreshes incrementally.",
|
|
201
299
|
}
|
|
202
300
|
|
|
203
301
|
RESEARCH_ID = _text_schema(80)
|
|
@@ -356,9 +454,13 @@ TOOL_SCHEMAS.update({
|
|
|
356
454
|
"wiki_write": _object_schema({"page_id": RESEARCH_ID, "page": WIKI_PAGE, "change_note": _text_schema(4000),
|
|
357
455
|
"sources": _array_schema(_object_schema({"id": RESEARCH_ID, "url": _text_schema(8192),
|
|
358
456
|
"text": _text_schema(2000000), "capture_id": _text_schema(128), "title": _text_schema(2000),
|
|
359
|
-
"retrieved_at": _text_schema(64), "kind": {"type": "string", "enum": ["page", "local_document", "external_report"]}
|
|
457
|
+
"retrieved_at": _text_schema(64), "kind": {"type": "string", "enum": ["page", "local_document", "external_report"]},
|
|
458
|
+
"bookmark_refs": _array_schema(_object_schema({"source_id": IDENTIFIER, "section_id": IDENTIFIER,
|
|
459
|
+
"item_id": IDENTIFIER}, ["source_id", "section_id", "item_id"]), 100),
|
|
460
|
+
"inventory_ids": dict(_array_schema(RESEARCH_ID, 10000), uniqueItems=True)},
|
|
360
461
|
["id", "url"]), 16, 1),
|
|
361
462
|
"expected_revision": {"type": "integer", "minimum": 0, "maximum": 1000000},
|
|
463
|
+
"ignore_duplicate_warning": {"type": "boolean", "description": "When true, suppresses duplicate concept warnings if URLs or titles overlap with existing pages."},
|
|
362
464
|
"reviewed_input_version": {**_text_schema(200), "description": "The already-synchronized input version this page was checked against; closes source_input_changed for cited tasks at that version."}},
|
|
363
465
|
["page_id", "page", "change_note"]),
|
|
364
466
|
"wiki_acknowledge": _object_schema({"page_id": RESEARCH_ID, "note": _text_schema(4000),
|
|
@@ -383,6 +485,51 @@ TOOL_SCHEMAS.update({
|
|
|
383
485
|
"limit": {"type": "integer", "minimum": 1, "maximum": 100}}, ["query"]),
|
|
384
486
|
"wiki_lint": _object_schema({"page_id": RESEARCH_ID}),
|
|
385
487
|
})
|
|
488
|
+
|
|
489
|
+
# P0b primitive names. The legacy schemas remain available during the
|
|
490
|
+
# compatibility window; these aliases provide one stable surface while the
|
|
491
|
+
# underlying providers and evidence rules are migrated incrementally.
|
|
492
|
+
TOOL_SCHEMAS.update({
|
|
493
|
+
"search": _object_schema({
|
|
494
|
+
"query": _text_schema(2000), "providers": PROVIDERS,
|
|
495
|
+
"limit": {"type": "integer", "minimum": 1, "maximum": 20, "default": 8},
|
|
496
|
+
}, ["query"]),
|
|
497
|
+
"fetch": TOOL_SCHEMAS["fetch_web"],
|
|
498
|
+
"ingest": copy.deepcopy(TOOL_SCHEMAS["merge_search_results"]),
|
|
499
|
+
"spawn": _object_schema({**SERVICE_INPUT, "operation_id": RESEARCH_ID,
|
|
500
|
+
"task": {"type": "string", "enum": ["deep_research", "fetch", "ingest", "research"],
|
|
501
|
+
"default": "deep_research"},
|
|
502
|
+
"args": {"type": "object",
|
|
503
|
+
"description": "Local job payload: fetch {urls, provider?, archive?}, ingest {batches}, research {input, provider?}."},
|
|
504
|
+
"priority": {"type": "integer", "minimum": -100, "maximum": 100, "default": 0}}),
|
|
505
|
+
"job": _object_schema({
|
|
506
|
+
"job_id": _text_schema(128),
|
|
507
|
+
"external_id": RESEARCH_ID,
|
|
508
|
+
"action": {"type": "string", "enum": ["status", "result", "cancel"], "default": "status"},
|
|
509
|
+
"refresh": {"type": "boolean", "default": False},
|
|
510
|
+
"offset": {"type": "integer", "minimum": 0, "maximum": 10000000, "default": 0},
|
|
511
|
+
"limit": {"type": "integer", "minimum": 1, "maximum": 50000, "default": 12000},
|
|
512
|
+
}),
|
|
513
|
+
"health": _object_schema({}),
|
|
514
|
+
"read": copy.deepcopy(TOOL_SCHEMAS["get_context"]),
|
|
515
|
+
"walk": copy.deepcopy(TOOL_SCHEMAS["traverse_graph"]),
|
|
516
|
+
"save": TOOL_SCHEMAS["wiki_write"],
|
|
517
|
+
})
|
|
518
|
+
TOOL_SCHEMAS["ingest"]["properties"]["batches"]["maxItems"] = 16
|
|
519
|
+
# The primitive surfaces accept a short ref as an alternative to repeating
|
|
520
|
+
# source/section/item identifiers returned by locate. Legacy schemas keep
|
|
521
|
+
# their source_id requirement and remain available for compatibility calls.
|
|
522
|
+
for _primitive_name in ("read", "walk"):
|
|
523
|
+
TOOL_SCHEMAS[_primitive_name]["required"] = []
|
|
524
|
+
TOOL_SCHEMAS[_primitive_name]["properties"]["ref"] = _text_schema(128)
|
|
525
|
+
TOOL_SCHEMAS["read"]["properties"].update({
|
|
526
|
+
"research_id": RESEARCH_ID, "evidence_source_id": RESEARCH_ID,
|
|
527
|
+
"evidence_source_ids": dict(_array_schema(RESEARCH_ID, 8, 1), uniqueItems=True),
|
|
528
|
+
"page_id": RESEARCH_ID,
|
|
529
|
+
"revision": {"type": "integer", "minimum": 1, "maximum": 1000000},
|
|
530
|
+
"v": _text_schema(32),
|
|
531
|
+
})
|
|
532
|
+
TOOL_SCHEMAS["read"]["properties"]["collection"]["enum"].append("all")
|
|
386
533
|
TOOL_DESCRIPTIONS.update({
|
|
387
534
|
"research_route": "Describe available quick, agentic or deep routes without starting work. Collaboration and external services are optional; explicit provider is exclusive. failed_routes is cumulative and requires confirmed failures; observe active/unknown runs before choosing another route.",
|
|
388
535
|
"research_services": "Describe optional OpenAI Deep Research and Parallel Task API configuration offline. Configured keys do not prove auth; anonymous Search MCP access is separate.",
|
|
@@ -401,8 +548,44 @@ TOOL_DESCRIPTIONS.update({
|
|
|
401
548
|
"wiki_refresh": "Fetch and archive up to 16 cited webpage URLs from one Wiki revision. Reports changed/unchanged/unverified captures and affected pages; follow continuation for more. Uses retrieval quota. Read saved captures and review new claims before wiki_write; authored text and bookmark packages stay unchanged.",
|
|
402
549
|
"wiki_search": "Find authored Wiki text using an incremental character index, literal/spacing matching and explicit aliases. Validates matching candidates; invalid/retracted evidence is excluded. Freshness warnings remain visible. No embeddings or network calls.",
|
|
403
550
|
"wiki_lint": "Audit all selected Wiki evidence, hashes, source freshness, crosslinks and history. Mechanical integrity and declared semantic review are reported separately; no network.",
|
|
551
|
+
"search": "Search public sources and return discovery results with provider failures and coverage visible. Results are not page正文; fetch known URLs before citing.",
|
|
552
|
+
"fetch": "Fetch known URLs with provider fallback, bounded text budgets, content_state and optional archived capture continuation. A returned body is not automatically a verified citation.",
|
|
553
|
+
"ingest": "Fuse host or provider search batches by canonical URL and body SHA-256, preserving provider provenance, failures and coverage. This is deterministic result ingestion, not Agentic Search planning.",
|
|
554
|
+
"spawn": "Submit work that outlives one tool step. task=deep_research (default) starts a provider-managed Deep Research run and returns an external id; task=fetch/ingest/research enqueues a durable local job and returns job_id, processed by `br job worker`. The host owns iteration and decides when to observe it with job.",
|
|
555
|
+
"job": "Observe or cancel a job. Accepts a local job_id (j-...) or a provider external_id. action=status reads state, result returns the stored result (artifact_path for large output), cancel requests cancellation; no new task is submitted. For local jobs, state=succeeded means the worker finished; read result.status and per_url for the actual fetch outcome.",
|
|
556
|
+
"health": "Return a compact local health summary for indexed sources and the source watcher. It performs no network fetch and is safe to call before research.",
|
|
557
|
+
"read": "Read a target Canvas bookmark and its folder ancestors by default; use collection='all' for full section context. Also reads a pinned Wiki revision, archived capture, or saved research evidence. Canvas refs are source-version-bound; Wiki and archive refs preserve revision/capture identity and content state.",
|
|
558
|
+
"walk": "Traverse Canvas edges and memberships through the shared walk primitive, preserving direction, depth, paths and pagination.",
|
|
559
|
+
"save": "Save a reviewed Wiki revision through the shared save primitive. Citation checks, expected_revision and review requirements remain enforced by WikiStore.",
|
|
404
560
|
})
|
|
405
561
|
|
|
562
|
+
DEPRECATED_TOOL_TARGETS = {
|
|
563
|
+
"search_bookmarks": "locate", "lookup_url": "locate", "search_archive": "locate",
|
|
564
|
+
"wiki_search": "locate", "wiki_list": "locate", "get_context": "read",
|
|
565
|
+
"traverse_graph": "walk", "search_web": "search", "fetch_web": "fetch",
|
|
566
|
+
"merge_search_results": "ingest", "wiki_write": "save", "wiki_acknowledge": "save",
|
|
567
|
+
"research_service_start": "spawn", "research_service_status": "job",
|
|
568
|
+
"research_service_result": "job", "research_service_cancel": "job",
|
|
569
|
+
"research_source": "read", "wiki_get": "read",
|
|
570
|
+
}
|
|
571
|
+
for _legacy_name, _replacement_name in DEPRECATED_TOOL_TARGETS.items():
|
|
572
|
+
TOOL_DESCRIPTIONS[_legacy_name] = "[deprecated→%s] %s" % (
|
|
573
|
+
_replacement_name, TOOL_DESCRIPTIONS[_legacy_name])
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def _legacy_compatibility_expired(version):
|
|
577
|
+
"""Return whether a stable release has reached the 0.8.0 cutoff."""
|
|
578
|
+
match = re.fullmatch(
|
|
579
|
+
r"(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?",
|
|
580
|
+
str(version or ""))
|
|
581
|
+
if match is None:
|
|
582
|
+
return False
|
|
583
|
+
current = tuple(int(match.group(index)) for index in (1, 2, 3))
|
|
584
|
+
cutoff = (0, 8, 0)
|
|
585
|
+
if current != cutoff:
|
|
586
|
+
return current > cutoff
|
|
587
|
+
return match.group(4) is None
|
|
588
|
+
|
|
406
589
|
|
|
407
590
|
class RpcError(Exception):
|
|
408
591
|
def __init__(self, code, message):
|
|
@@ -478,34 +661,109 @@ class StdioMcpServer:
|
|
|
478
661
|
self.protocol = None
|
|
479
662
|
self.ready = False
|
|
480
663
|
self._index = None
|
|
664
|
+
self._read_index = None
|
|
481
665
|
self._source_manager = None
|
|
666
|
+
self._read_source_manager = None
|
|
482
667
|
self._source_watcher = None
|
|
483
668
|
self._providers = None
|
|
484
669
|
self._research_sessions = None
|
|
485
670
|
self._research_services = None
|
|
486
671
|
self._wiki_store = None
|
|
487
672
|
self._raw_library = None
|
|
673
|
+
self._job_store = None
|
|
488
674
|
self._initialization_lock = threading.RLock()
|
|
489
675
|
self._log_lock = threading.Lock()
|
|
490
676
|
self._cancel_pending = None
|
|
677
|
+
self._envelope_versions = OrderedDict()
|
|
678
|
+
self._envelope_handles = {}
|
|
491
679
|
self.settings = settings if settings is not None else Settings()
|
|
492
680
|
|
|
681
|
+
def _compact_source_envelope(self, result):
|
|
682
|
+
"""Collapse repeated source diagnostics while retaining identity per response."""
|
|
683
|
+
if not isinstance(result, dict) or not isinstance(result.get("source"), dict):
|
|
684
|
+
return result
|
|
685
|
+
source = result["source"]
|
|
686
|
+
identity = {key: source[key] for key in ("source_id", "version_id") if key in source}
|
|
687
|
+
diagnostics = {key: value for key, value in source.items()
|
|
688
|
+
if key not in ("source_id", "version_id")}
|
|
689
|
+
refresh = result.get("refresh")
|
|
690
|
+
sync = refresh.get("sync") if isinstance(refresh, dict) else None
|
|
691
|
+
dynamic_refresh = ({key: value for key, value in refresh.items() if key != "sync"}
|
|
692
|
+
if isinstance(refresh, dict) else None)
|
|
693
|
+
payload = {"identity": identity, "source": diagnostics, "sync": sync}
|
|
694
|
+
encoded = json.dumps(payload, ensure_ascii=False, allow_nan=False,
|
|
695
|
+
sort_keys=True, separators=(",", ":"))
|
|
696
|
+
digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
|
697
|
+
handle = self._envelope_handles.get(digest)
|
|
698
|
+
if handle is None:
|
|
699
|
+
prefix = "v-" + digest[:4]
|
|
700
|
+
sequence = 1
|
|
701
|
+
handle = prefix + "-" + str(sequence)
|
|
702
|
+
while handle in self._envelope_versions and self._envelope_versions[handle]["digest"] != digest:
|
|
703
|
+
sequence += 1
|
|
704
|
+
handle = prefix + "-" + str(sequence)
|
|
705
|
+
self._envelope_handles[digest] = handle
|
|
706
|
+
existing = handle in self._envelope_versions
|
|
707
|
+
self._envelope_versions[handle] = {"digest": digest, **payload}
|
|
708
|
+
self._envelope_versions.move_to_end(handle)
|
|
709
|
+
while len(self._envelope_versions) > 128:
|
|
710
|
+
old_handle, old_value = self._envelope_versions.popitem(last=False)
|
|
711
|
+
if self._envelope_handles.get(old_value["digest"]) == old_handle:
|
|
712
|
+
del self._envelope_handles[old_value["digest"]]
|
|
713
|
+
result["v"] = handle
|
|
714
|
+
if existing:
|
|
715
|
+
result["source"] = identity
|
|
716
|
+
if dynamic_refresh is not None:
|
|
717
|
+
result["refresh"] = dynamic_refresh
|
|
718
|
+
return result
|
|
719
|
+
|
|
493
720
|
def close(self):
|
|
494
721
|
if self._source_watcher is not None:
|
|
495
722
|
self._source_watcher.close()
|
|
496
723
|
self._source_watcher = None
|
|
497
724
|
self._source_manager = None
|
|
725
|
+
self._read_source_manager = None
|
|
498
726
|
if self._index is not None:
|
|
499
727
|
self._index.close()
|
|
500
728
|
self._index = None
|
|
729
|
+
if self._read_index is not None:
|
|
730
|
+
self._read_index.close()
|
|
731
|
+
self._read_index = None
|
|
732
|
+
if self._job_store is not None:
|
|
733
|
+
self._job_store.close()
|
|
734
|
+
self._job_store = None
|
|
735
|
+
|
|
736
|
+
def _local(self, readonly=False):
|
|
737
|
+
"""Return the shared index handle; reads use a read-only connection.
|
|
501
738
|
|
|
502
|
-
|
|
739
|
+
A second MCP process used to open the read-write handle for every
|
|
740
|
+
``read``/``walk`` and pay the schema/WAL write lock each time (4-15s
|
|
741
|
+
under concurrency). Read-only handles skip schema writes and never
|
|
742
|
+
block the writer.
|
|
743
|
+
"""
|
|
744
|
+
if readonly:
|
|
745
|
+
if self._read_index is None:
|
|
746
|
+
from bookmark_index import BookmarkIndex
|
|
747
|
+
self._read_index = BookmarkIndex(self.db_path, readonly=True)
|
|
748
|
+
return self._read_index
|
|
503
749
|
if self._index is None:
|
|
750
|
+
# SQLite cannot run the read-write schema bootstrap while a read-only
|
|
751
|
+
# handle on the same file is open (it surfaces as "disk I/O error"), so
|
|
752
|
+
# an upgrade to the writer drops the cached reader first.
|
|
753
|
+
if self._read_index is not None:
|
|
754
|
+
self._read_index.close()
|
|
755
|
+
self._read_index = None
|
|
756
|
+
self._read_source_manager = None
|
|
504
757
|
from bookmark_index import BookmarkIndex
|
|
505
758
|
self._index = BookmarkIndex(self.db_path)
|
|
506
759
|
return self._index
|
|
507
760
|
|
|
508
|
-
def _sources(self):
|
|
761
|
+
def _sources(self, readonly=False):
|
|
762
|
+
if readonly:
|
|
763
|
+
if self._read_source_manager is None:
|
|
764
|
+
from source_manager import SourceManager
|
|
765
|
+
self._read_source_manager = SourceManager(self._local(readonly=True))
|
|
766
|
+
return self._read_source_manager
|
|
509
767
|
if self._source_manager is None:
|
|
510
768
|
from source_manager import SourceManager
|
|
511
769
|
self._source_manager = SourceManager(self._local())
|
|
@@ -554,6 +812,12 @@ class StdioMcpServer:
|
|
|
554
812
|
research_sessions=self._research())
|
|
555
813
|
return self._wiki_store
|
|
556
814
|
|
|
815
|
+
def _jobs(self):
|
|
816
|
+
if self._job_store is None:
|
|
817
|
+
from jobs import JobStore
|
|
818
|
+
self._job_store = JobStore()
|
|
819
|
+
return self._job_store
|
|
820
|
+
|
|
557
821
|
def _raw(self):
|
|
558
822
|
# Rebuilt per call so archive/output setting changes apply without restart.
|
|
559
823
|
if self._raw_library is not None:
|
|
@@ -587,11 +851,36 @@ class StdioMcpServer:
|
|
|
587
851
|
small without deleting any implementation.
|
|
588
852
|
"""
|
|
589
853
|
scope = self._tool_scope()
|
|
590
|
-
|
|
854
|
+
expired = _legacy_compatibility_expired(release_version())
|
|
855
|
+
available = [name for name in TOOL_SCHEMAS
|
|
856
|
+
if not expired or name not in DEPRECATED_TOOL_TARGETS]
|
|
857
|
+
if not scope:
|
|
858
|
+
return available
|
|
859
|
+
allowed = set(available)
|
|
860
|
+
return list(dict.fromkeys(name for name in scope if name in allowed))
|
|
861
|
+
|
|
862
|
+
@staticmethod
|
|
863
|
+
def _legacy_tool_removed(name):
|
|
864
|
+
return (name in DEPRECATED_TOOL_TARGETS
|
|
865
|
+
and _legacy_compatibility_expired(release_version()))
|
|
591
866
|
|
|
592
867
|
def _tool_enabled(self, name):
|
|
593
|
-
|
|
594
|
-
|
|
868
|
+
"""Explicit scopes restrict calls; quick/deep presets only filter discovery.
|
|
869
|
+
|
|
870
|
+
The quick preset must keep deprecated handlers callable during the
|
|
871
|
+
compatibility window even though they are omitted from tools/list.
|
|
872
|
+
A host environment override or an explicit user allowlist remains a
|
|
873
|
+
hard capability boundary.
|
|
874
|
+
"""
|
|
875
|
+
override = os.environ.get("BOOKMARK_RESEARCH_TOOL_SCOPE")
|
|
876
|
+
if override is not None:
|
|
877
|
+
scope = [item.strip() for item in override.split(",") if item.strip()]
|
|
878
|
+
return not scope or name in scope
|
|
879
|
+
preferences = self.settings.load()
|
|
880
|
+
explicit = preferences.get("tool_scope") or []
|
|
881
|
+
if explicit:
|
|
882
|
+
return name in explicit
|
|
883
|
+
return preferences.get("tool_scope_preset", "quick") in ("quick", "deep", "all")
|
|
595
884
|
|
|
596
885
|
def _hydrate_sources(self, sources):
|
|
597
886
|
"""Turn capture pointers into verified evidence before bundling.
|
|
@@ -651,6 +940,42 @@ class StdioMcpServer:
|
|
|
651
940
|
hydrated.append(row)
|
|
652
941
|
return hydrated
|
|
653
942
|
|
|
943
|
+
def _locate_canvas(self, arguments):
|
|
944
|
+
"""Thin MCP adapter around the shared Canvas locate primitive."""
|
|
945
|
+
arguments = dict(arguments)
|
|
946
|
+
package_path = arguments.pop("package_path", None)
|
|
947
|
+
imported = None
|
|
948
|
+
if package_path is not None:
|
|
949
|
+
if arguments.get("source_id"):
|
|
950
|
+
raise ValueError("pass package_path or source_id, not both")
|
|
951
|
+
if arguments.get("handle"):
|
|
952
|
+
raise ValueError("package_path cannot be combined with a continuation handle")
|
|
953
|
+
if not isinstance(package_path, str) or not package_path.strip():
|
|
954
|
+
raise ValueError("package_path must be a non-empty string")
|
|
955
|
+
resolved_path = Path(package_path).expanduser()
|
|
956
|
+
if not resolved_path.is_absolute():
|
|
957
|
+
raise ValueError("package_path must be an absolute path; pass the full path to the export or package")
|
|
958
|
+
imported = self._sources().sync(str(resolved_path), mode="snapshot", completeness="partial")
|
|
959
|
+
arguments["source_id"] = imported["source_id"]
|
|
960
|
+
self._start_watcher()
|
|
961
|
+
query_keys = ("text", "url", "folder_id", "tags", "tag_colors", "has_note", "has_tags")
|
|
962
|
+
if not any(arguments.get(key) is not None for key in query_keys):
|
|
963
|
+
summary = self._local(readonly=True).source_summary(imported["source_id"])
|
|
964
|
+
summary["imported"] = {"input_path": imported.get("input_path"),
|
|
965
|
+
"counts": imported.get("counts"),
|
|
966
|
+
"changed_files": imported.get("changed_files")}
|
|
967
|
+
summary["next"] = ("call locate(source_id=%r, text=...) to search, or read(ref=...) "
|
|
968
|
+
"on a hit" % imported["source_id"])
|
|
969
|
+
return summary
|
|
970
|
+
requested = arguments.get("in", ["canvas"])
|
|
971
|
+
if arguments.get("handle"):
|
|
972
|
+
requested = continuation_scope(arguments["handle"], self.db_path)
|
|
973
|
+
wiki = self._wiki() if requested == ["wiki"] else None
|
|
974
|
+
raw = self._raw() if requested == ["archive"] else None
|
|
975
|
+
refresh = bool(arguments.get("refresh", False))
|
|
976
|
+
return locate_core(self._local(readonly=not refresh), self._sources(readonly=not refresh),
|
|
977
|
+
arguments, wiki=wiki, raw=raw)
|
|
978
|
+
|
|
654
979
|
def _call(self, name, arguments):
|
|
655
980
|
# Explicit dispatch ensures input can never select a Python method,
|
|
656
981
|
# executable, SQL statement, or unadvertised capability.
|
|
@@ -697,12 +1022,222 @@ class StdioMcpServer:
|
|
|
697
1022
|
return self._sources().remove(**arguments)
|
|
698
1023
|
if name == "source_merge":
|
|
699
1024
|
return self._sources().merge(**arguments)
|
|
1025
|
+
if name == "locate":
|
|
1026
|
+
return self._locate_canvas(arguments)
|
|
1027
|
+
if name == "search":
|
|
1028
|
+
return self._web().search(
|
|
1029
|
+
targets=[{"target": arguments["query"], "query": arguments["query"]}],
|
|
1030
|
+
providers=arguments.get("providers"),
|
|
1031
|
+
limit_per_target=arguments.get("limit", 8),
|
|
1032
|
+
)
|
|
1033
|
+
if name == "fetch":
|
|
1034
|
+
return self._call("fetch_web", dict(arguments))
|
|
1035
|
+
if name == "ingest":
|
|
1036
|
+
batches = arguments.get("batches")
|
|
1037
|
+
if isinstance(batches, list) and len(batches) > 16:
|
|
1038
|
+
raise ValueError("ingest accepts at most 16 batches; use the CLI file path for larger input")
|
|
1039
|
+
encoded = json.dumps(arguments, ensure_ascii=False, allow_nan=False, separators=(",", ":"))
|
|
1040
|
+
if len(encoded) > 8000:
|
|
1041
|
+
raise ValueError("ingest payload exceeds 8,000 characters; use the CLI file path for larger input")
|
|
1042
|
+
return self._call("merge_search_results", dict(arguments))
|
|
1043
|
+
if name == "spawn":
|
|
1044
|
+
task = arguments.get("task") or "deep_research"
|
|
1045
|
+
if task in ("fetch", "ingest", "research"):
|
|
1046
|
+
payload = arguments.get("args") or {}
|
|
1047
|
+
if not isinstance(payload, dict):
|
|
1048
|
+
raise ValueError("spawn args must be an object")
|
|
1049
|
+
job = self._jobs().submit(task, payload,
|
|
1050
|
+
idempotency_key=arguments.get("operation_id"),
|
|
1051
|
+
priority=int(arguments.get("priority", 0)))
|
|
1052
|
+
return {"job_id": job["job_id"], "state": job["state"], "kind": job["kind"],
|
|
1053
|
+
"note": "queued; run `br job worker` (or it already runs) and observe with job(job_id)"}
|
|
1054
|
+
if not arguments.get("input") or not arguments.get("operation_id"):
|
|
1055
|
+
raise ValueError("spawn(task='deep_research') requires input and operation_id")
|
|
1056
|
+
options = {key: value for key, value in arguments.items()
|
|
1057
|
+
if key not in ("task", "args", "priority")}
|
|
1058
|
+
return self._call("research_service_start", options)
|
|
1059
|
+
if name == "job":
|
|
1060
|
+
options = dict(arguments)
|
|
1061
|
+
action = options.pop("action", "status")
|
|
1062
|
+
identifier = options.pop("job_id", None) or options.pop("external_id", None)
|
|
1063
|
+
if not identifier:
|
|
1064
|
+
raise ValueError("job requires job_id")
|
|
1065
|
+
if identifier.startswith("j-"):
|
|
1066
|
+
store = self._jobs()
|
|
1067
|
+
job = store.get(identifier)
|
|
1068
|
+
if job is None:
|
|
1069
|
+
raise ValueError("Unknown job: " + identifier)
|
|
1070
|
+
if action == "status":
|
|
1071
|
+
return {"job": job}
|
|
1072
|
+
if action == "result":
|
|
1073
|
+
return {"job_id": job["job_id"], "state": job["state"],
|
|
1074
|
+
"result": job["result"], "artifact_path": job["artifact_path"],
|
|
1075
|
+
"error": job["error"]}
|
|
1076
|
+
return {"job": store.cancel(identifier)}
|
|
1077
|
+
if action == "status":
|
|
1078
|
+
return self._services().status(identifier, options.get("refresh", False))
|
|
1079
|
+
if action == "result":
|
|
1080
|
+
return self._services().result(identifier, options.get("refresh", False),
|
|
1081
|
+
options.get("offset", 0), options.get("limit", 12000))
|
|
1082
|
+
return self._services().cancel(identifier)
|
|
1083
|
+
if name == "health":
|
|
1084
|
+
source_report = self._sources(readonly=True).status()
|
|
1085
|
+
sources = []
|
|
1086
|
+
for source in source_report.get("sources", []):
|
|
1087
|
+
sources.append({key: source.get(key) for key in ("source_id", "mode", "state", "version_id")
|
|
1088
|
+
if source.get(key) is not None})
|
|
1089
|
+
watcher = self._source_watcher.status() if self._source_watcher else {"running": False}
|
|
1090
|
+
return {"status": "ok" if not source_report.get("errors") else "degraded",
|
|
1091
|
+
"sources": sources, "watcher": {"running": bool(watcher.get("running"))}}
|
|
1092
|
+
if name == "read":
|
|
1093
|
+
options = dict(arguments)
|
|
1094
|
+
version_handle = options.pop("v", None)
|
|
1095
|
+
if version_handle:
|
|
1096
|
+
cached = self._envelope_versions.get(version_handle)
|
|
1097
|
+
if cached is None:
|
|
1098
|
+
raise ValueError("unknown or expired source metadata ref")
|
|
1099
|
+
self._envelope_versions.move_to_end(version_handle)
|
|
1100
|
+
metadata = {"v": version_handle, "source": {**cached["source"], **cached["identity"]}}
|
|
1101
|
+
if cached.get("sync") is not None:
|
|
1102
|
+
metadata["refresh"] = {"sync": cached["sync"]}
|
|
1103
|
+
return metadata
|
|
1104
|
+
ref = options.pop("ref", None)
|
|
1105
|
+
canvas_ref = False
|
|
1106
|
+
default_canvas_read = False
|
|
1107
|
+
if ref:
|
|
1108
|
+
resolved = resolve_ref(ref)
|
|
1109
|
+
if resolved is None:
|
|
1110
|
+
raise ValueError("Unknown or expired ref; call locate again")
|
|
1111
|
+
if resolved["kind"] == "w":
|
|
1112
|
+
page_id, revision = resolved["identity"]
|
|
1113
|
+
options.setdefault("page_id", page_id)
|
|
1114
|
+
options.setdefault("revision", int(revision))
|
|
1115
|
+
elif resolved["kind"] == "b":
|
|
1116
|
+
canvas_ref = True
|
|
1117
|
+
default_canvas_read = "collection" not in options
|
|
1118
|
+
source_id, version_id, section_id, item_id = resolved["identity"]
|
|
1119
|
+
current = self._sources(readonly=True).source_status(source_id)
|
|
1120
|
+
if version_id and current.get("version_id") != version_id:
|
|
1121
|
+
raise ValueError("stale_ref: source version changed; call locate again")
|
|
1122
|
+
options.setdefault("source_id", source_id)
|
|
1123
|
+
options.setdefault("section", section_id)
|
|
1124
|
+
options.setdefault("item_id", item_id)
|
|
1125
|
+
elif resolved["kind"] == "a":
|
|
1126
|
+
capture_id, expected_sha, url = resolved["identity"]
|
|
1127
|
+
if not capture_id or not url:
|
|
1128
|
+
raise ValueError("Archive ref has no readable capture; use fetch on the hit URL")
|
|
1129
|
+
from archive import SourceArchive
|
|
1130
|
+
prefs = self.settings.load()["fetch"]
|
|
1131
|
+
archived = SourceArchive(self.settings.load()["archive"]["directory"]).read_capture(
|
|
1132
|
+
capture_id, [url], max_characters=prefs["max_characters"],
|
|
1133
|
+
max_total_characters=prefs["max_total_characters"],
|
|
1134
|
+
fair_share_min=prefs["fair_share_min"])
|
|
1135
|
+
page = archived.get("pages", [{}])[0]
|
|
1136
|
+
if expected_sha and page.get("sha256") != expected_sha:
|
|
1137
|
+
raise ValueError("stale_ref: archived capture hash changed")
|
|
1138
|
+
return archived
|
|
1139
|
+
else:
|
|
1140
|
+
raise ValueError("Unsupported ref kind")
|
|
1141
|
+
if options.get("page_id"):
|
|
1142
|
+
page_id = options.pop("page_id")
|
|
1143
|
+
revision = options.pop("revision", None)
|
|
1144
|
+
return self._wiki().get(page_id, revision)
|
|
1145
|
+
if options.get("research_id"):
|
|
1146
|
+
research_id = options.pop("research_id")
|
|
1147
|
+
evidence_source_id = options.pop("evidence_source_id", None)
|
|
1148
|
+
evidence_source_ids = options.pop("evidence_source_ids", None)
|
|
1149
|
+
if options.keys() - {"offset", "limit"}:
|
|
1150
|
+
raise ValueError("research evidence read accepts research_id, source IDs, offset and limit")
|
|
1151
|
+
return self._research().source(research_id, source_id=evidence_source_id,
|
|
1152
|
+
offset=options.get("offset", 0),
|
|
1153
|
+
limit=options.get("limit", 12000),
|
|
1154
|
+
source_ids=evidence_source_ids)
|
|
1155
|
+
if "source_id" not in options:
|
|
1156
|
+
raise ValueError("read requires a Canvas ref/source_id, Wiki ref/page_id, or research_id")
|
|
1157
|
+
if canvas_ref and options.get("collection") == "all":
|
|
1158
|
+
options.pop("collection")
|
|
1159
|
+
elif canvas_ref and "collection" not in options:
|
|
1160
|
+
options.setdefault("collection", "items")
|
|
1161
|
+
options.setdefault("limit", 1)
|
|
1162
|
+
options.setdefault("compact", True)
|
|
1163
|
+
options.setdefault("refresh", False)
|
|
1164
|
+
result = self._call("get_context", options)
|
|
1165
|
+
if default_canvas_read and isinstance(result, dict):
|
|
1166
|
+
pagination = result.get("pagination", {})
|
|
1167
|
+
items = result.get("items", [])
|
|
1168
|
+
result = {
|
|
1169
|
+
"source_id": result.get("source_id"),
|
|
1170
|
+
"items": items,
|
|
1171
|
+
"source": result.get("source", {}),
|
|
1172
|
+
"refresh": result.get("refresh", {}),
|
|
1173
|
+
"read_scope": "target_item_and_folder_ancestors",
|
|
1174
|
+
"pagination": {"items": pagination.get("items", {})},
|
|
1175
|
+
}
|
|
1176
|
+
# A folder ref is a request for its children, not for one row.
|
|
1177
|
+
folder_item = (items[0] if len(items) == 1
|
|
1178
|
+
and items[0].get("item_type") == "folder" else None)
|
|
1179
|
+
if folder_item is not None:
|
|
1180
|
+
index = self._local(readonly=True)
|
|
1181
|
+
listing = index.folder_children(
|
|
1182
|
+
options["source_id"], folder_item["item_id"],
|
|
1183
|
+
section=folder_item.get("section_id"), limit=5)
|
|
1184
|
+
version = self._sources(readonly=True).source_status(
|
|
1185
|
+
options["source_id"]).get("version_id")
|
|
1186
|
+
children = []
|
|
1187
|
+
for child in listing["results"]:
|
|
1188
|
+
entry = {"ref": locate_ref(options["source_id"], child["section_id"],
|
|
1189
|
+
child["item_id"], version),
|
|
1190
|
+
"kind": child["item_type"], "item_id": child["item_id"],
|
|
1191
|
+
"title": child.get("title") or "",
|
|
1192
|
+
"position": child.get("position")}
|
|
1193
|
+
for field in ("url", "note", "note_color", "tags"):
|
|
1194
|
+
if child.get(field):
|
|
1195
|
+
entry[field] = child[field]
|
|
1196
|
+
children.append(entry)
|
|
1197
|
+
result["folder"] = {"folder_id": folder_item["item_id"],
|
|
1198
|
+
"title": folder_item.get("title") or "",
|
|
1199
|
+
"path": folder_item.get("path") or []}
|
|
1200
|
+
result["children"] = children
|
|
1201
|
+
result["child_count"] = listing["total"]
|
|
1202
|
+
result["read_scope"] = "folder_children"
|
|
1203
|
+
result["next"] = ("%d direct children; read a child ref for its card"
|
|
1204
|
+
% listing["total"]) if listing["total"] else "Folder is empty"
|
|
1205
|
+
if isinstance(result, dict) and isinstance(result.get("source"), dict):
|
|
1206
|
+
result["source"].setdefault("source_id", options["source_id"])
|
|
1207
|
+
return self._compact_source_envelope(result)
|
|
1208
|
+
if name == "walk":
|
|
1209
|
+
options = dict(arguments)
|
|
1210
|
+
ref = options.pop("ref", None)
|
|
1211
|
+
if ref:
|
|
1212
|
+
resolved = resolve_ref(ref)
|
|
1213
|
+
if resolved is None:
|
|
1214
|
+
raise ValueError("Unknown or expired ref; call locate again")
|
|
1215
|
+
if resolved["kind"] != "b":
|
|
1216
|
+
raise ValueError("walk supports Canvas bookmark refs only")
|
|
1217
|
+
source_id, version_id, section_id, item_id = resolved["identity"]
|
|
1218
|
+
current = self._sources(readonly=True).source_status(source_id)
|
|
1219
|
+
if version_id and current.get("version_id") != version_id:
|
|
1220
|
+
raise ValueError("stale_ref: source version changed; call locate again")
|
|
1221
|
+
options.setdefault("source_id", source_id)
|
|
1222
|
+
options.setdefault("section", section_id)
|
|
1223
|
+
options.setdefault("item_id", item_id)
|
|
1224
|
+
if "source_id" not in options:
|
|
1225
|
+
raise ValueError("walk requires ref or source_id")
|
|
1226
|
+
options.setdefault("refresh", False)
|
|
1227
|
+
return self._call("traverse_graph", options)
|
|
1228
|
+
if name == "save":
|
|
1229
|
+
return self._call("wiki_write", dict(arguments))
|
|
700
1230
|
if name == "plan_research" and not arguments.get("source_id"):
|
|
701
1231
|
from research_plan import ResearchPlan
|
|
702
1232
|
options = dict(arguments)
|
|
703
1233
|
options.pop("refresh", None)
|
|
704
1234
|
return ResearchPlan(settings=self.settings, wiki=self._wiki(), raw=self._raw()).build(**options)
|
|
705
|
-
if name
|
|
1235
|
+
if name == "lookup_url":
|
|
1236
|
+
index = self._local()
|
|
1237
|
+
with index.read_snapshot():
|
|
1238
|
+
return index.lookup_url(arguments["url"], source_id=arguments.get("source_id"),
|
|
1239
|
+
limit=arguments.get("limit", 100), offset=arguments.get("offset", 0))
|
|
1240
|
+
if name in ("search_bookmarks", "get_context", "traverse_graph", "plan_research"):
|
|
706
1241
|
options = dict(arguments)
|
|
707
1242
|
refresh = options.pop("refresh", True)
|
|
708
1243
|
pagination = {key: options.pop(key, default) for key, default in (
|
|
@@ -711,7 +1246,7 @@ class StdioMcpServer:
|
|
|
711
1246
|
scope_notes = None if name != "get_context" or "item_id" in options else {
|
|
712
1247
|
"items": "get_context fills items only for one item_id; read a card's bookmarks with "
|
|
713
1248
|
"search_bookmarks(section=...) or plan_research."}
|
|
714
|
-
index = self._local()
|
|
1249
|
+
index = self._local(readonly=not refresh)
|
|
715
1250
|
update = self._sources().refresh(options["source_id"]) if refresh else None
|
|
716
1251
|
if update and update["state"] in ("error", "unavailable"):
|
|
717
1252
|
raise ValueError(update["error"] + "; use refresh=false to read the saved index")
|
|
@@ -720,6 +1255,8 @@ class StdioMcpServer:
|
|
|
720
1255
|
from research_plan import ResearchPlan
|
|
721
1256
|
result = ResearchPlan(index=index, settings=self.settings,
|
|
722
1257
|
wiki=self._wiki(), raw=self._raw()).build(**options)
|
|
1258
|
+
elif name == "traverse_graph":
|
|
1259
|
+
result = index.traverse(**options)
|
|
723
1260
|
else:
|
|
724
1261
|
method = index.search if name == "search_bookmarks" else index.context
|
|
725
1262
|
result = method(**options)
|
|
@@ -729,17 +1266,26 @@ class StdioMcpServer:
|
|
|
729
1266
|
result = page_context(result, scope_notes=scope_notes, **pagination)
|
|
730
1267
|
if compact:
|
|
731
1268
|
result = compact_context(result)
|
|
732
|
-
result["source"] = self._sources().source_status(options["source_id"])
|
|
1269
|
+
result["source"] = self._sources(readonly=not refresh).source_status(options["source_id"])
|
|
733
1270
|
result["refresh"] = {"performed": bool(refresh and result["source"]["mode"] == "live"),
|
|
734
1271
|
"index_updated": bool(update and update.get("performed")),
|
|
735
1272
|
"requested": refresh, "mode": "checked_package" if refresh and result["source"]["mode"] == "live" else "saved_snapshot", "sync": update}
|
|
736
1273
|
return result
|
|
737
1274
|
if name == "index_status":
|
|
738
|
-
result = self._sources().status(**arguments)
|
|
1275
|
+
result = self._sources(readonly=True).status(**arguments)
|
|
739
1276
|
result["monitor"] = self._source_watcher.status() if self._source_watcher else {"running": False, "mechanism": "poll"}
|
|
740
1277
|
return result
|
|
741
1278
|
if name == "search_web":
|
|
742
1279
|
return self._web().search(**arguments)
|
|
1280
|
+
if name == "agentic_search":
|
|
1281
|
+
from grok_search import GrokSearch, OpenAICompatibleAgenticSearch
|
|
1282
|
+
provider = arguments.pop("provider", "grok")
|
|
1283
|
+
if provider == "grok":
|
|
1284
|
+
return GrokSearch(Credentials(self.settings), self.settings.load()["timeout_seconds"]).search(**arguments)
|
|
1285
|
+
if arguments.get("include_x"):
|
|
1286
|
+
raise ValueError("include_x is available only for the Grok provider")
|
|
1287
|
+
return OpenAICompatibleAgenticSearch(
|
|
1288
|
+
self.settings, Credentials(self.settings), self.settings.load()["timeout_seconds"]).search(**arguments)
|
|
743
1289
|
if name == "fetch_web":
|
|
744
1290
|
from archive import SourceArchive
|
|
745
1291
|
options = dict(arguments)
|
|
@@ -758,7 +1304,8 @@ class StdioMcpServer:
|
|
|
758
1304
|
max_total_characters=total, offset=offset, fair_share_min=fair)
|
|
759
1305
|
if offset:
|
|
760
1306
|
raise ValueError("offset requires capture_id so continued reads use the same saved evidence")
|
|
761
|
-
|
|
1307
|
+
engine = self._web()
|
|
1308
|
+
result = engine._fetch(**options) if raw else engine.fetch(**options)
|
|
762
1309
|
return result if raw else SourceArchive.compact_fetch(
|
|
763
1310
|
result, max_total_characters=total, fair_share_min=fair)
|
|
764
1311
|
if name == "search_archive":
|
|
@@ -767,6 +1314,9 @@ class StdioMcpServer:
|
|
|
767
1314
|
options = dict(arguments)
|
|
768
1315
|
probe = options.pop("probe", False)
|
|
769
1316
|
return self._web().probe(**options) if probe else self._web().describe(**options)
|
|
1317
|
+
if name == "merge_search_results":
|
|
1318
|
+
from search_results import fuse_results
|
|
1319
|
+
return fuse_results(arguments, archive=arguments.get("archive", False), settings=self.settings)
|
|
770
1320
|
raise RpcError(-32602, "Unknown tool: " + name)
|
|
771
1321
|
|
|
772
1322
|
def _tool_result(self, value, is_error=False):
|
|
@@ -788,7 +1338,7 @@ class StdioMcpServer:
|
|
|
788
1338
|
params = _params(message.get("params", {}), ("name", "arguments"), ("name",))
|
|
789
1339
|
name, arguments = params["name"], params.get("arguments", {})
|
|
790
1340
|
_validate(_text_schema(128), name, "name")
|
|
791
|
-
if name not in TOOL_SCHEMAS:
|
|
1341
|
+
if name not in TOOL_SCHEMAS or self._legacy_tool_removed(name):
|
|
792
1342
|
return False
|
|
793
1343
|
if not self._tool_enabled(name):
|
|
794
1344
|
return False
|
|
@@ -801,7 +1351,7 @@ class StdioMcpServer:
|
|
|
801
1351
|
return arguments.get("probe", False)
|
|
802
1352
|
if name == "research_readiness":
|
|
803
1353
|
return not arguments.get("offline", False)
|
|
804
|
-
return name in {"search_web", "research_search", "research_fetch", "wiki_refresh",
|
|
1354
|
+
return name in {"search_web", "agentic_search", "research_search", "research_fetch", "wiki_refresh",
|
|
805
1355
|
"research_service_start", "research_service_status", "research_service_result",
|
|
806
1356
|
"research_service_cancel", "research_service_attach", "research_service_import"}
|
|
807
1357
|
|
|
@@ -873,7 +1423,7 @@ class StdioMcpServer:
|
|
|
873
1423
|
_params(params, ("name", "arguments"), ("name",))
|
|
874
1424
|
name = params["name"]
|
|
875
1425
|
_validate(_text_schema(128), name, "name")
|
|
876
|
-
if name not in TOOL_SCHEMAS:
|
|
1426
|
+
if name not in TOOL_SCHEMAS or self._legacy_tool_removed(name):
|
|
877
1427
|
raise RpcError(-32602, "Unknown tool: " + name)
|
|
878
1428
|
if not self._tool_enabled(name):
|
|
879
1429
|
raise RpcError(-32602, "Tool is outside this profile's tool_scope: " + name)
|