@yahyahammad/venastine 0.2.0 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/.env.example +9 -4
  2. package/CONFIG_ARCHITECTURE.md +2030 -0
  3. package/LICENSE +201 -201
  4. package/NOTICE +1 -1
  5. package/README.md +1034 -895
  6. package/agents/builtin/build.md +73 -0
  7. package/agents/builtin/compactor.md +75 -75
  8. package/agents/builtin/explore.md +77 -77
  9. package/agents/builtin/general.md +80 -0
  10. package/agents/builtin/pipeline-reviewer.md +100 -100
  11. package/agents/builtin/plan.md +106 -88
  12. package/agents/builtin/review.md +74 -74
  13. package/agents/builtin/test.md +65 -0
  14. package/agents/builtin/writer.md +67 -0
  15. package/agents/manager.py +68 -2
  16. package/agents/subagent_tool.py +472 -238
  17. package/agents/tui_commands.py +238 -151
  18. package/bin/venastine.mjs +150 -4
  19. package/bin/venastine.py +71 -0
  20. package/cli_commands.py +838 -0
  21. package/config.py +95 -1198
  22. package/config.yaml +1103 -0
  23. package/config_edit.py +1301 -0
  24. package/config_schema.py +971 -0
  25. package/config_update.py +749 -0
  26. package/core/agent_activity.py +256 -118
  27. package/core/ansi.py +452 -0
  28. package/core/approval.py +137 -115
  29. package/core/call_view.py +364 -0
  30. package/core/cancel.py +121 -0
  31. package/core/chatgpt_auth.py +747 -0
  32. package/core/claims_view.py +107 -0
  33. package/core/client.py +2715 -1559
  34. package/core/commands.py +460 -0
  35. package/core/compaction.py +1735 -971
  36. package/core/config_loader.py +1982 -1563
  37. package/core/connect.py +686 -0
  38. package/core/events.py +123 -61
  39. package/core/file_mentions.py +280 -0
  40. package/core/interaction.py +411 -328
  41. package/core/line_pattern.py +128 -0
  42. package/core/loop.py +2566 -1490
  43. package/core/memory.py +482 -370
  44. package/core/pipeline_models.py +190 -176
  45. package/core/provider_errors.py +234 -0
  46. package/core/reasoning/authorization.py +130 -130
  47. package/core/reasoning/base.py +219 -219
  48. package/core/reasoning/confidence_scoring.py +484 -468
  49. package/core/reasoning/events.py +149 -149
  50. package/core/reasoning/json_retry.py +19 -1
  51. package/core/reasoning/orchestrator.py +1561 -1382
  52. package/core/reasoning/output_writer.py +345 -286
  53. package/core/reasoning/payload_validation.py +524 -398
  54. package/core/reasoning/pipeline_storage.py +88 -22
  55. package/core/reasoning/review.py +659 -615
  56. package/core/reasoning/scholar.py +458 -434
  57. package/core/reasoning/source_corpus.py +433 -327
  58. package/core/reasoning/source_scoring.py +855 -855
  59. package/core/replay.py +315 -183
  60. package/core/session.py +36 -12
  61. package/core/session_view.py +259 -0
  62. package/core/session_wake.py +259 -0
  63. package/core/shell_sessions.py +1553 -0
  64. package/core/workspace_trust.py +1 -1
  65. package/credentials.py +148 -20
  66. package/database.py +333 -252
  67. package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
  68. package/docs/SECURITY.md +108 -0
  69. package/docs/THIRD_PARTY_NOTICES.md +148 -0
  70. package/env_secrets.py +50 -9
  71. package/json_store.py +343 -123
  72. package/logging_setup.py +295 -185
  73. package/main.py +3433 -1961
  74. package/mcp_client/client.py +733 -499
  75. package/mcp_client/config.py +126 -21
  76. package/mcp_client/registration.py +343 -182
  77. package/memories/manager.py +147 -132
  78. package/memories/tui_commands.py +319 -244
  79. package/package.json +303 -73
  80. package/project_init/config_files.py +282 -278
  81. package/project_init/generator.py +728 -564
  82. package/project_init/manifest.py +18 -6
  83. package/project_init/tui_commands.py +147 -144
  84. package/prompts/critic_pass.md +24 -24
  85. package/prompts/revalidate.md +25 -25
  86. package/prompts/revise.md +2 -0
  87. package/prompts/source_grounding.md +66 -66
  88. package/prompts/system_prompts.py +316 -262
  89. package/providers.json.example +4 -2
  90. package/pyproject.toml +149 -100
  91. package/requirements.txt +295 -72
  92. package/safety/policy_enforcement.py +1082 -746
  93. package/security/askpass.py +419 -0
  94. package/security/capability.py +207 -177
  95. package/security/confine.py +484 -0
  96. package/security/hosted.py +182 -0
  97. package/security/permissions.py +237 -190
  98. package/security/posture.py +124 -12
  99. package/security/protected_paths.py +501 -309
  100. package/security/sandbox.py +4108 -985
  101. package/security/secrets.py +855 -0
  102. package/skills/builtin/math/formal-specification.md +67 -67
  103. package/skills/builtin/math/numerical-methods.md +66 -66
  104. package/skills/builtin/research/experiment-design.md +62 -62
  105. package/skills/builtin/research/reproducibility.md +55 -55
  106. package/skills/builtin/research/source-evaluation.md +70 -70
  107. package/skills/builtin/research/statistical-inference.md +63 -63
  108. package/skills/builtin/research/technical-writing.md +59 -59
  109. package/skills/builtin/software/code-review.md +61 -61
  110. package/skills/builtin/software/debugging.md +61 -61
  111. package/skills/builtin/software/test-design.md +67 -67
  112. package/skills/tui_commands.py +110 -111
  113. package/storage.py +1338 -1005
  114. package/tools/base.py +161 -1
  115. package/tools/builtin/_math_common.py +667 -377
  116. package/tools/builtin/_net_common.py +225 -19
  117. package/tools/builtin/arxiv.py +497 -39
  118. package/tools/builtin/ask_user.py +231 -213
  119. package/tools/builtin/discrete_math.py +35 -4
  120. package/tools/builtin/fetch_url.py +371 -153
  121. package/tools/builtin/file_ops.py +720 -412
  122. package/tools/builtin/geometry.py +48 -12
  123. package/tools/builtin/get_time.py +2 -2
  124. package/tools/builtin/linear_algebra.py +66 -23
  125. package/tools/builtin/load_skill.py +76 -66
  126. package/tools/builtin/logic.py +171 -12
  127. package/tools/builtin/pin.py +227 -193
  128. package/tools/builtin/probability_stats.py +12 -16
  129. package/tools/builtin/project_docs.py +464 -405
  130. package/tools/builtin/remember.py +175 -158
  131. package/tools/builtin/shell.py +980 -461
  132. package/tools/builtin/shell_sessions.py +613 -0
  133. package/tools/builtin/symbolic_math.py +80 -10
  134. package/tools/builtin/todo.py +149 -147
  135. package/tools/builtin/web_search.py +191 -13
  136. package/tools/context.py +159 -136
  137. package/tools/isolation.py +107 -34
  138. package/tools/registry.py +1242 -925
  139. package/tui/app.py +8318 -4100
  140. package/tui/app.tcss +747 -556
  141. package/tui/commands.py +26 -170
  142. package/tui/diffs.py +34 -13
  143. package/tui/history.py +128 -128
  144. package/tui/hosted_driver.py +102 -0
  145. package/tui/markdown.py +1329 -1279
  146. package/tui/meters.py +364 -268
  147. package/tui/ravens.py +146 -146
  148. package/tui/screens.py +1255 -943
  149. package/tui/themes.py +599 -559
  150. package/tui/widgets.py +3813 -2332
  151. package/SECURITY.md +0 -88
  152. package/THIRD_PARTY_NOTICES.md +0 -100
@@ -1,434 +1,458 @@
1
- """
2
- core/reasoning/scholar.py
3
-
4
- What a research paper is worth, from OpenAlex.
5
-
6
- ROADMAP_v2 §45 (SQ5/SQ9). A cited paper's authority is more than its
7
- host: arxiv.org serves a landmark and a paper nobody ever read from the
8
- same domain, and the domain class cannot tell them apart. This asks the
9
- one question a domain cannot -- how does this paper stand among its own
10
- peers -- and answers it from counts rather than from a model.
11
-
12
- WHY COHORT COUNTING RATHER THAN OPENALEX'S OWN NORMALISATION. `fwci` and
13
- `citation_normalized_percentile` are exactly the field-and-age-normalised
14
- metrics this needs, and BOTH ARE NULL ON PREPRINTS -- verified against
15
- the API on a paper with 3019 citations. arXiv is the only scholarly
16
- source this harness currently reaches, so the ready-made answer does not
17
- cover the case that matters. Two counting queries do: how many works
18
- share this paper's field, type and publication window, and how many of
19
- them have fewer citations than it. That ratio IS the field-and-age
20
- normalisation, measured rather than modelled, and it self-calibrates
21
- across fields because the comparison set is the same field.
22
-
23
- WHY NO LIFECYCLE CURVE. Matching the cohort on a date window makes the
24
- age normalisation exact rather than approximate, so a fitted accrual
25
- curve would add nothing for a paper old enough to carry signal. Below
26
- that, it would be fitting noise: measured while specifying this, 72% of
27
- computer-science preprints published in 2024 have ZERO citations. A
28
- three-week-old paper's zero says nothing at all, so the term is dropped
29
- and the remaining weights renormalise -- absence of evidence about a new
30
- paper is not evidence of low impact.
31
-
32
- THE EGRESS IS DELIBERATE, NAMED, AND OFF BY DEFAULT (SQ9). This is the
33
- first network call in this harness made by something that is not a
34
- registered tool, so it does not pass through the approval registry and
35
- does not appear in granted_calls.json. That is stated rather than worked
36
- around: routing deterministic scoring code through the tool layer would
37
- make a pure function call a dispatcher. What it does instead is reuse
38
- `is_url_permitted`, cache like the network tools do, trace every lookup,
39
- and stay off until someone turns it on.
40
-
41
- EVERY SCORING FUNCTION HERE IS PURE and takes a plain dict, so the whole
42
- of it is testable against fixtures with no network -- which is not a
43
- convenience, it is the suite's contract.
44
- """
45
-
46
- from __future__ import annotations
47
-
48
- import logging
49
- import math
50
- import re
51
- from datetime import date, timedelta
52
-
53
- import config
54
- from tools.builtin._net_common import TTLCache
55
-
56
- logger = logging.getLogger(__name__)
57
-
58
- _cache = TTLCache(config.SCHOLAR_CACHE_TTL_S)
59
-
60
- # A DOI as it appears inside a URL. Deliberately not anchored: the point
61
- # is to find one in `https://doi.org/10.1234/x` and in a publisher's own
62
- # `.../article/10.1234/x/full`.
63
- _DOI_RE = re.compile(r"10\.\d{4,9}/[^\s?#]+", re.IGNORECASE)
64
-
65
- # Trailing punctuation a URL path collects that a DOI does not own.
66
- _DOI_TRIM = ".,;:)]}>'\"/"
67
-
68
-
69
- def doi_for(url: str) -> str | None:
70
- """The DOI this URL identifies, or None.
71
-
72
- arXiv is handled through `source_corpus.arxiv_id_from_url` rather than
73
- a second id parser here: two copies of "what counts as the same paper"
74
- is how the corpus and this lookup come to disagree about one source in
75
- one artifact.
76
- """
77
- from core.reasoning.source_corpus import arxiv_id_from_url
78
-
79
- arxiv_id = arxiv_id_from_url(url or "")
80
- if arxiv_id:
81
- # The DOI arXiv mints for every submission since 2022, and
82
- # backfilled. OpenAlex indexes preprints under it.
83
- return f"10.48550/arxiv.{arxiv_id}".lower()
84
-
85
- match = _DOI_RE.search(url or "")
86
- if not match:
87
- return None
88
- return match.group(0).rstrip(_DOI_TRIM).lower()
89
-
90
-
91
- # ---------------------------------------------------------------------------
92
- # ---- Pure scoring ---------------------------------------------------------
93
- # ---------------------------------------------------------------------------
94
-
95
- def _location_type(work: dict) -> str:
96
- location = work.get("primary_location") or {}
97
- source = location.get("source") or {}
98
- return (source.get("type") or "").lower()
99
-
100
-
101
- def is_peer_reviewed(work: dict) -> bool:
102
- """Whether this record is a published, peer-reviewed version.
103
-
104
- THREE FIELDS, because one is not enough. `type == "preprint"` catches
105
- the ordinary case; a repository-hosted record with
106
- `version == "submittedVersion"` catches a work typed as an article
107
- that is really an author manuscript; and `is_published` is the
108
- publisher's own answer where there is one. A record has to survive all
109
- three to count.
110
- """
111
- location = work.get("primary_location") or {}
112
- if (work.get("type") or "").lower() == "preprint":
113
- return False
114
- if _location_type(work) not in ("journal", "conference", "book series"):
115
- return False
116
- if location.get("version") == "submittedVersion":
117
- return False
118
- return location.get("is_published", True) is not False
119
-
120
-
121
- def venue_credit(work: dict) -> float:
122
- """What the venue alone says, before any citation is counted."""
123
- if is_peer_reviewed(work):
124
- return config.SCHOLAR_VENUE_CREDIT.get(
125
- _location_type(work), config.SCHOLAR_VENUE_CREDIT["unknown"])
126
- return config.SCHOLAR_VENUE_CREDIT["repository"]
127
-
128
-
129
- def author_standing(h_indexes, saturation: int | None = None):
130
- """max h-index across the authors, log-saturated onto 0-1, or None.
131
-
132
- MAX rather than mean: a first-author graduate student on a paper with
133
- a senior collaborator does not make the work less credible, and
134
- averaging would say it does.
135
-
136
- SATURATING rather than linear, because the difference between an
137
- h-index of 60 and one of 90 says more about field size and career
138
- length than about this paper. None when no author has a usable index
139
- -- the term is then dropped and the rest renormalise, rather than
140
- scoring an unindexed author as a bad one.
141
- """
142
- saturation = saturation or config.SCHOLAR_H_SATURATION
143
- usable = [h for h in (h_indexes or [])
144
- if isinstance(h, int) and not isinstance(h, bool) and h >= 0]
145
- if not usable:
146
- return None
147
- return min(1.0, math.log1p(max(usable)) / math.log1p(saturation))
148
-
149
-
150
- def paper_score(venue, citation, author, weights: dict | None = None) -> float:
151
- """Blend whichever signals exist, renormalising over them.
152
-
153
- THE RENORMALISATION IS THE POINT. A three-week-old preprint has no
154
- citation percentile and an unindexed author has no h-index; folding a
155
- missing term in as zero would score "we could not measure this" and
156
- "this measured badly" identically, which is the distinction §45 keeps
157
- making. So a missing term takes its weight out of the denominator too.
158
- """
159
- weights = weights or {}
160
- terms = [
161
- (venue, weights.get("venue_weight", config.SCHOLAR_VENUE_WEIGHT)),
162
- (citation, weights.get("citation_weight", config.SCHOLAR_CITATION_WEIGHT)),
163
- (author, weights.get("author_weight", config.SCHOLAR_AUTHOR_WEIGHT)),
164
- ]
165
- present = [(value, weight) for value, weight in terms
166
- if value is not None and weight > 0]
167
- if not present:
168
- return config.DEFAULT_DOMAIN_AUTHORITY
169
- total = sum(weight for _, weight in present)
170
- return min(1.0, max(0.0, sum(value * weight for value, weight in present) / total))
171
-
172
-
173
- def _age_days(work: dict, today: date | None = None) -> int | None:
174
- published = work.get("publication_date")
175
- if not isinstance(published, str):
176
- return None
177
- try:
178
- when = date.fromisoformat(published[:10])
179
- except ValueError:
180
- return None
181
- return ((today or date.today()) - when).days
182
-
183
-
184
- # ---------------------------------------------------------------------------
185
- # ---- The client -----------------------------------------------------------
186
- # ---------------------------------------------------------------------------
187
-
188
- class ScholarLookup:
189
- """OpenAlex, cached and bounded, or a no-op when disabled.
190
-
191
- A CLASS holding one run's cache and failure state, for
192
- EmbeddingScorer's reason: one outage should stop the lookups rather
193
- than be retried once per paper, and the run should say so once.
194
- """
195
-
196
- def __init__(self, enabled: bool | None = None, fetch=None,
197
- knobs: dict | None = None) -> None:
198
- self.enabled = (config.SCHOLAR_LOOKUP if enabled is None else enabled)
199
- self.knobs = knobs or {}
200
- self.failed_reason = None
201
- self.calls = 0
202
- self._fetch = fetch or self._http_get
203
- self._cohorts: dict = {}
204
-
205
- @property
206
- def usable(self) -> bool:
207
- return self.enabled and self.failed_reason is None
208
-
209
- # -- transport -------------------------------------------------------
210
-
211
- def _http_get(self, path: str, params: dict):
212
- """The only place this module touches the network.
213
-
214
- INJECTABLE, and every test injects: the suite runs offline, and a
215
- scoring rule tested through a real socket is a scoring rule tested
216
- on somebody's connection.
217
- """
218
- import httpx
219
-
220
- from safety.policy_enforcement import is_url_permitted
221
-
222
- url = f"{config.SCHOLAR_API_URL.rstrip('/')}/{path.lstrip('/')}"
223
- refusal = is_url_permitted(url, resolve=False)
224
- if refusal:
225
- raise RuntimeError(refusal)
226
- if config.SCHOLAR_MAILTO:
227
- # Only ever what the user configured. An address taken from a
228
- # git config or an environment variable is not a rate-limit
229
- # optimisation, it is an unasked-for disclosure.
230
- params = {**params, "mailto": config.SCHOLAR_MAILTO}
231
- response = httpx.get(
232
- url, params=params, timeout=config.SCHOLAR_TIMEOUT_S,
233
- headers={"User-Agent": "venastine-research-harness/1.0"},
234
- follow_redirects=True)
235
- response.raise_for_status()
236
- return response.json()
237
-
238
- def _get(self, path: str, params: dict):
239
- """Cached, retried, and terminal on exhaustion. None on failure."""
240
- if not self.usable:
241
- return None
242
- key = f"{path}|{sorted(params.items())}"
243
- hit = _cache.get(key)
244
- if hit is not None:
245
- return hit
246
-
247
- last = None
248
- for _ in range(config.SCHOLAR_MAX_RETRIES + 1):
249
- try:
250
- payload = self._fetch(path, params)
251
- except Exception as exc: # noqa: BLE001 - degraded, never fatal
252
- last = exc
253
- continue
254
- self.calls += 1
255
- _cache.set(key, payload)
256
- return payload
257
- # A 404 is not an outage -- plenty of cited URLs are not indexed
258
- # works -- so only a repeated failure disables the lookup, and the
259
- # caller distinguishes "no record" from "could not ask".
260
- self.failed_reason = f"{type(last).__name__}: {last}"
261
- return None
262
-
263
- # -- lookups ---------------------------------------------------------
264
-
265
- def work(self, url: str):
266
- """The OpenAlex record for `url`, or None if it is not a paper."""
267
- doi = doi_for(url)
268
- if not doi or not self.usable:
269
- return None
270
- payload = self._get(f"works/doi:{doi}", {"select": _WORK_SELECT})
271
- if not isinstance(payload, dict) or not payload.get("id"):
272
- return None
273
- return payload
274
-
275
- def _count(self, filters: str) -> int | None:
276
- payload = self._get("works", {"filter": filters, "per_page": 1,
277
- "select": "id"})
278
- if not isinstance(payload, dict):
279
- return None
280
- count = (payload.get("meta") or {}).get("count")
281
- return count if isinstance(count, int) else None
282
-
283
- def citation_percentile(self, work: dict, today: date | None = None):
284
- """Where this paper sits among works like it, or None with a reason.
285
-
286
- Returns (percentile, evidence) where evidence is one of
287
- "cohort", "insufficient_age" or "insufficient_cohort" -- the
288
- second and third are answers, not failures, and the caller drops
289
- the term rather than scoring it low.
290
- """
291
- age = _age_days(work, today)
292
- min_age = self.knobs.get("min_citation_age_days",
293
- config.SCHOLAR_MIN_CITATION_AGE_DAYS)
294
- if age is None or age < min_age:
295
- return None, "insufficient_age"
296
-
297
- cites = work.get("cited_by_count")
298
- if not isinstance(cites, int) or isinstance(cites, bool):
299
- return None, "insufficient_cohort"
300
-
301
- field = ((work.get("primary_topic") or {}).get("field") or {}).get("id")
302
- domain = ((work.get("primary_topic") or {}).get("domain") or {}).get("id")
303
- work_type = (work.get("type") or "").lower() or "article"
304
- published = date.fromisoformat(work["publication_date"][:10])
305
-
306
- minimum = self.knobs.get("min_cohort_size", config.SCHOLAR_MIN_COHORT_SIZE)
307
- for subject in _cohort_subjects(field, domain):
308
- for window in _cohort_windows(published, age):
309
- base = _cohort_filter(subject, work_type, window)
310
- total = self._count(base)
311
- if total is None:
312
- return None, "insufficient_cohort"
313
- if total < minimum:
314
- continue
315
- below = self._count(f"{base},cited_by_count:<{cites}")
316
- if below is None:
317
- return None, "insufficient_cohort"
318
- return min(1.0, max(0.0, below / total)), "cohort"
319
- return None, "insufficient_cohort"
320
-
321
- def author_standing(self, work: dict):
322
- """The max h-index across this paper's authors, 0-1, or None.
323
-
324
- ONE BATCHED REQUEST per paper rather than one per author: a paper
325
- with nine authors is nine lookups the naive way, and the whole
326
- reason cohort counts are cached is that this stage runs inside a
327
- pass someone is waiting on.
328
- """
329
- ids = []
330
- for authorship in (work.get("authorships") or [])[:_MAX_AUTHORS]:
331
- author = (authorship or {}).get("author") or {}
332
- identifier = (author.get("id") or "").rsplit("/", 1)[-1]
333
- if identifier.startswith("A"):
334
- ids.append(identifier)
335
- if not ids:
336
- return None
337
- payload = self._get("authors", {
338
- "filter": "openalex_id:" + "|".join(ids),
339
- "select": "id,summary_stats", "per-page": len(ids)})
340
- if not isinstance(payload, dict):
341
- return None
342
- indexes = [((entry or {}).get("summary_stats") or {}).get("h_index")
343
- for entry in payload.get("results") or []]
344
- return author_standing(
345
- indexes, self.knobs.get("h_saturation", config.SCHOLAR_H_SATURATION))
346
-
347
- # -- the whole answer -------------------------------------------------
348
-
349
- def authority_for(self, url: str, today: date | None = None):
350
- """(authority, signals) for a cited paper, or (None, None).
351
-
352
- None means "this is not a paper, or nothing could be looked up" --
353
- the caller keeps the domain class, which is the right answer for
354
- both.
355
- """
356
- work = self.work(url)
357
- if work is None:
358
- return None, None
359
-
360
- if work.get("is_retracted"):
361
- # Overrides everything. A retracted paper is not evidence
362
- # whatever its venue or citation count, and a heavily cited
363
- # retraction is the most dangerous shape there is.
364
- return config.SCHOLAR_RETRACTED_AUTHORITY, {
365
- "openalex_id": work.get("id"), "is_retracted": True,
366
- "citation_evidence": "retracted"}
367
-
368
- venue = venue_credit(work)
369
- percentile, evidence = self.citation_percentile(work, today)
370
- author = self.author_standing(work)
371
- score = paper_score(venue, percentile, author, self.knobs)
372
-
373
- return score, {
374
- "openalex_id": work.get("id"),
375
- "peer_reviewed": is_peer_reviewed(work),
376
- "venue_type": _location_type(work) or None,
377
- "venue_credit": round(venue, 4),
378
- "cited_by_count": work.get("cited_by_count"),
379
- "citation_percentile": (None if percentile is None
380
- else round(percentile, 4)),
381
- "citation_evidence": evidence,
382
- "author_standing": None if author is None else round(author, 4),
383
- "is_retracted": False,
384
- }
385
-
386
-
387
- _WORK_SELECT = ("id,doi,type,publication_date,cited_by_count,is_retracted,"
388
- "primary_location,primary_topic,authorships")
389
-
390
- # Enough to find the strongest name on a paper without turning a
391
- # hyperauthorship record into a 3000-id query string.
392
- _MAX_AUTHORS = 25
393
-
394
-
395
- def _cohort_subjects(field, domain):
396
- """Field first, then domain, then everything of this type and age.
397
-
398
- WIDENING RATHER THAN GIVING UP, because a narrow field in a narrow
399
- window can genuinely hold fewer works than a percentile needs -- and a
400
- percentile against a broader cohort is a weaker statement, not a wrong
401
- one. The order is most-specific-first so the strongest available
402
- comparison is the one used.
403
- """
404
- seen = []
405
- for subject in (field, domain, None):
406
- if subject not in seen:
407
- seen.append(subject)
408
- return seen
409
-
410
-
411
- def _cohort_windows(published: date, age_days: int):
412
- """The date windows to try, narrowest first.
413
-
414
- A +/-30 day window for a paper under six months old, because within a
415
- single calendar year a January paper has eleven months more to
416
- accumulate citations than a December one -- and at that age eleven
417
- months is the whole signal. Older than that, the calendar year is both
418
- sufficient and larger.
419
- """
420
- if age_days < 183:
421
- yield (published - timedelta(days=30), published + timedelta(days=30))
422
- yield (date(published.year, 1, 1), date(published.year, 12, 31))
423
-
424
-
425
- def _cohort_filter(subject, work_type: str, window) -> str:
426
- start, end = window
427
- parts = [f"type:{work_type}",
428
- f"from_publication_date:{start.isoformat()}",
429
- f"to_publication_date:{end.isoformat()}"]
430
- if subject:
431
- key = ("primary_topic.domain.id" if "/domains/" in subject
432
- else "primary_topic.field.id")
433
- parts.insert(0, f"{key}:{subject}")
434
- return ",".join(parts)
1
+ """
2
+ core/reasoning/scholar.py
3
+
4
+ What a research paper is worth, from OpenAlex.
5
+
6
+ ROADMAP_v2 §45 (SQ5/SQ9). A cited paper's authority is more than its
7
+ host: arxiv.org serves a landmark and a paper nobody ever read from the
8
+ same domain, and the domain class cannot tell them apart. This asks the
9
+ one question a domain cannot -- how does this paper stand among its own
10
+ peers -- and answers it from counts rather than from a model.
11
+
12
+ WHY COHORT COUNTING RATHER THAN OPENALEX'S OWN NORMALISATION. `fwci` and
13
+ `citation_normalized_percentile` are exactly the field-and-age-normalised
14
+ metrics this needs, and BOTH ARE NULL ON PREPRINTS -- verified against
15
+ the API on a paper with 3019 citations. arXiv is the only scholarly
16
+ source this harness currently reaches, so the ready-made answer does not
17
+ cover the case that matters. Two counting queries do: how many works
18
+ share this paper's field, type and publication window, and how many of
19
+ them have fewer citations than it. That ratio IS the field-and-age
20
+ normalisation, measured rather than modelled, and it self-calibrates
21
+ across fields because the comparison set is the same field.
22
+
23
+ WHY NO LIFECYCLE CURVE. Matching the cohort on a date window makes the
24
+ age normalisation exact rather than approximate, so a fitted accrual
25
+ curve would add nothing for a paper old enough to carry signal. Below
26
+ that, it would be fitting noise: measured while specifying this, 72% of
27
+ computer-science preprints published in 2024 have ZERO citations. A
28
+ three-week-old paper's zero says nothing at all, so the term is dropped
29
+ and the remaining weights renormalise -- absence of evidence about a new
30
+ paper is not evidence of low impact.
31
+
32
+ THE EGRESS IS DELIBERATE, NAMED, AND OFF BY DEFAULT (SQ9). This is the
33
+ first network call in this harness made by something that is not a
34
+ registered tool, so it does not pass through the approval registry and
35
+ does not appear in granted_calls.json. That is stated rather than worked
36
+ around: routing deterministic scoring code through the tool layer would
37
+ make a pure function call a dispatcher. What it does instead is reuse
38
+ `is_url_permitted`, cache like the network tools do, trace every lookup,
39
+ and stay off until someone turns it on.
40
+
41
+ EVERY SCORING FUNCTION HERE IS PURE and takes a plain dict, so the whole
42
+ of it is testable against fixtures with no network -- which is not a
43
+ convenience, it is the suite's contract.
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ import logging
49
+ import math
50
+ import re
51
+ from datetime import date, timedelta
52
+
53
+ import config
54
+ from tools.builtin._net_common import TTLCache
55
+
56
+ logger = logging.getLogger(__name__)
57
+
58
+ _cache = TTLCache(config.SCHOLAR_CACHE_TTL_S)
59
+
60
+ # A DOI as it appears inside a URL. Deliberately not anchored: the point
61
+ # is to find one in `https://doi.org/10.1234/x` and in a publisher's own
62
+ # `.../article/10.1234/x/full`.
63
+ _DOI_RE = re.compile(r"10\.\d{4,9}/[^\s?#]+", re.IGNORECASE)
64
+
65
+ # Trailing punctuation a URL path collects that a DOI does not own.
66
+ _DOI_TRIM = ".,;:)]}>'\"/"
67
+
68
+
69
+ def doi_for(url: str) -> str | None:
70
+ """The DOI this URL identifies, or None.
71
+
72
+ arXiv is handled through `source_corpus.arxiv_id_from_url` rather than
73
+ a second id parser here: two copies of "what counts as the same paper"
74
+ is how the corpus and this lookup come to disagree about one source in
75
+ one artifact.
76
+ """
77
+ from core.reasoning.source_corpus import arxiv_id_from_url
78
+
79
+ arxiv_id = arxiv_id_from_url(url or "")
80
+ if arxiv_id:
81
+ # The DOI arXiv mints for every submission since 2022, and
82
+ # backfilled. OpenAlex indexes preprints under it.
83
+ return f"10.48550/arxiv.{arxiv_id}".lower()
84
+
85
+ match = _DOI_RE.search(url or "")
86
+ if not match:
87
+ return None
88
+ return match.group(0).rstrip(_DOI_TRIM).lower()
89
+
90
+
91
+ # ---------------------------------------------------------------------------
92
+ # ---- Pure scoring ---------------------------------------------------------
93
+ # ---------------------------------------------------------------------------
94
+
95
+ def _obj(value) -> dict:
96
+ """`value` if OpenAlex sent an object there, else an empty one.
97
+
98
+ Batch 219 (#218). Every read of a record used to trust its shape --
99
+ `(x or {}).get(...)` survives a null and nothing else -- so one
100
+ non-object where an object was expected raised AttributeError out of
101
+ an optional stage and failed a run that had already grounded and
102
+ scored. Driven: 104 malformed shapes across ten keys. A shape this
103
+ module cannot read is a signal that is absent, and every term here
104
+ already knows how to drop an absent one.
105
+ """
106
+ return value if isinstance(value, dict) else {}
107
+
108
+
109
+ def _text(value) -> str:
110
+ """`value` if it is a string, else "" -- `_obj`'s rule for a leaf."""
111
+ return value if isinstance(value, str) else ""
112
+
113
+
114
+ def _location_type(work: dict) -> str:
115
+ location = _obj(work.get("primary_location"))
116
+ source = _obj(location.get("source"))
117
+ return _text(source.get("type")).lower()
118
+
119
+
120
+ def is_peer_reviewed(work: dict) -> bool:
121
+ """Whether this record is a published, peer-reviewed version.
122
+
123
+ THREE FIELDS, because one is not enough. `type == "preprint"` catches
124
+ the ordinary case; a repository-hosted record with
125
+ `version == "submittedVersion"` catches a work typed as an article
126
+ that is really an author manuscript; and `is_published` is the
127
+ publisher's own answer where there is one. A record has to survive all
128
+ three to count.
129
+ """
130
+ location = _obj(work.get("primary_location"))
131
+ if _text(work.get("type")).lower() == "preprint":
132
+ return False
133
+ if _location_type(work) not in ("journal", "conference", "book series"):
134
+ return False
135
+ if location.get("version") == "submittedVersion":
136
+ return False
137
+ return location.get("is_published", True) is not False
138
+
139
+
140
+ def venue_credit(work: dict) -> float:
141
+ """What the venue alone says, before any citation is counted."""
142
+ if is_peer_reviewed(work):
143
+ return config.SCHOLAR_VENUE_CREDIT.get(
144
+ _location_type(work), config.SCHOLAR_VENUE_CREDIT["unknown"])
145
+ return config.SCHOLAR_VENUE_CREDIT["repository"]
146
+
147
+
148
+ def author_standing(h_indexes, saturation: int | None = None):
149
+ """max h-index across the authors, log-saturated onto 0-1, or None.
150
+
151
+ MAX rather than mean: a first-author graduate student on a paper with
152
+ a senior collaborator does not make the work less credible, and
153
+ averaging would say it does.
154
+
155
+ SATURATING rather than linear, because the difference between an
156
+ h-index of 60 and one of 90 says more about field size and career
157
+ length than about this paper. None when no author has a usable index
158
+ -- the term is then dropped and the rest renormalise, rather than
159
+ scoring an unindexed author as a bad one.
160
+ """
161
+ saturation = saturation or config.SCHOLAR_H_SATURATION
162
+ usable = [h for h in (h_indexes or [])
163
+ if isinstance(h, int) and not isinstance(h, bool) and h >= 0]
164
+ if not usable:
165
+ return None
166
+ return min(1.0, math.log1p(max(usable)) / math.log1p(saturation))
167
+
168
+
169
+ def paper_score(venue, citation, author, weights: dict | None = None) -> float:
170
+ """Blend whichever signals exist, renormalising over them.
171
+
172
+ THE RENORMALISATION IS THE POINT. A three-week-old preprint has no
173
+ citation percentile and an unindexed author has no h-index; folding a
174
+ missing term in as zero would score "we could not measure this" and
175
+ "this measured badly" identically, which is the distinction §45 keeps
176
+ making. So a missing term takes its weight out of the denominator too.
177
+ """
178
+ weights = weights or {}
179
+ terms = [
180
+ (venue, weights.get("venue_weight", config.SCHOLAR_VENUE_WEIGHT)),
181
+ (citation, weights.get("citation_weight", config.SCHOLAR_CITATION_WEIGHT)),
182
+ (author, weights.get("author_weight", config.SCHOLAR_AUTHOR_WEIGHT)),
183
+ ]
184
+ present = [(value, weight) for value, weight in terms
185
+ if value is not None and weight > 0]
186
+ if not present:
187
+ return config.DEFAULT_DOMAIN_AUTHORITY
188
+ total = sum(weight for _, weight in present)
189
+ return min(1.0, max(0.0, sum(value * weight for value, weight in present) / total))
190
+
191
+
192
+ def _age_days(work: dict, today: date | None = None) -> int | None:
193
+ published = work.get("publication_date")
194
+ if not isinstance(published, str):
195
+ return None
196
+ try:
197
+ when = date.fromisoformat(published[:10])
198
+ except ValueError:
199
+ return None
200
+ return ((today or date.today()) - when).days
201
+
202
+
203
+ # ---------------------------------------------------------------------------
204
+ # ---- The client -----------------------------------------------------------
205
+ # ---------------------------------------------------------------------------
206
+
207
+ class ScholarLookup:
208
+ """OpenAlex, cached and bounded, or a no-op when disabled.
209
+
210
+ A CLASS holding one run's cache and failure state, for
211
+ EmbeddingScorer's reason: one outage should stop the lookups rather
212
+ than be retried once per paper, and the run should say so once.
213
+ """
214
+
215
+ def __init__(self, enabled: bool | None = None, fetch=None,
216
+ knobs: dict | None = None) -> None:
217
+ self.enabled = (config.SCHOLAR_LOOKUP if enabled is None else enabled)
218
+ self.knobs = knobs or {}
219
+ self.failed_reason = None
220
+ self.calls = 0
221
+ self._fetch = fetch or self._http_get
222
+ self._cohorts: dict = {}
223
+
224
+ @property
225
+ def usable(self) -> bool:
226
+ return self.enabled and self.failed_reason is None
227
+
228
+ # -- transport -------------------------------------------------------
229
+
230
+ def _http_get(self, path: str, params: dict):
231
+ """The only place this module touches the network.
232
+
233
+ INJECTABLE, and every test injects: the suite runs offline, and a
234
+ scoring rule tested through a real socket is a scoring rule tested
235
+ on somebody's connection.
236
+ """
237
+ import httpx
238
+
239
+ from safety.policy_enforcement import is_url_permitted
240
+
241
+ url = f"{config.SCHOLAR_API_URL.rstrip('/')}/{path.lstrip('/')}"
242
+ refusal = is_url_permitted(url, resolve=False)
243
+ if refusal:
244
+ raise RuntimeError(refusal)
245
+ if config.SCHOLAR_MAILTO:
246
+ # Only ever what the user configured. An address taken from a
247
+ # git config or an environment variable is not a rate-limit
248
+ # optimisation, it is an unasked-for disclosure.
249
+ params = {**params, "mailto": config.SCHOLAR_MAILTO}
250
+ response = httpx.get(
251
+ url, params=params, timeout=config.SCHOLAR_TIMEOUT_S,
252
+ headers={"User-Agent": "venastine-research-harness/1.0"},
253
+ follow_redirects=True)
254
+ response.raise_for_status()
255
+ return response.json()
256
+
257
+ def _get(self, path: str, params: dict):
258
+ """Cached, retried, and terminal on exhaustion. None on failure."""
259
+ if not self.usable:
260
+ return None
261
+ key = f"{path}|{sorted(params.items())}"
262
+ hit = _cache.get(key)
263
+ if hit is not None:
264
+ return hit
265
+
266
+ last = None
267
+ for _ in range(config.SCHOLAR_MAX_RETRIES + 1):
268
+ try:
269
+ payload = self._fetch(path, params)
270
+ except Exception as exc: # noqa: BLE001 - degraded, never fatal
271
+ last = exc
272
+ continue
273
+ self.calls += 1
274
+ _cache.set(key, payload)
275
+ return payload
276
+ # A 404 is not an outage -- plenty of cited URLs are not indexed
277
+ # works -- so only a repeated failure disables the lookup, and the
278
+ # caller distinguishes "no record" from "could not ask".
279
+ self.failed_reason = f"{type(last).__name__}: {last}"
280
+ return None
281
+
282
+ # -- lookups ---------------------------------------------------------
283
+
284
+ def work(self, url: str):
285
+ """The OpenAlex record for `url`, or None if it is not a paper."""
286
+ doi = doi_for(url)
287
+ if not doi or not self.usable:
288
+ return None
289
+ payload = self._get(f"works/doi:{doi}", {"select": _WORK_SELECT})
290
+ if not isinstance(payload, dict) or not payload.get("id"):
291
+ return None
292
+ return payload
293
+
294
+ def _count(self, filters: str) -> int | None:
295
+ payload = self._get("works", {"filter": filters, "per_page": 1,
296
+ "select": "id"})
297
+ if not isinstance(payload, dict):
298
+ return None
299
+ count = _obj(payload.get("meta")).get("count")
300
+ return count if isinstance(count, int) else None
301
+
302
+ def citation_percentile(self, work: dict, today: date | None = None):
303
+ """Where this paper sits among works like it, or None with a reason.
304
+
305
+ Returns (percentile, evidence) where evidence is one of
306
+ "cohort", "insufficient_age" or "insufficient_cohort" -- the
307
+ second and third are answers, not failures, and the caller drops
308
+ the term rather than scoring it low.
309
+ """
310
+ age = _age_days(work, today)
311
+ min_age = self.knobs.get("min_citation_age_days",
312
+ config.SCHOLAR_MIN_CITATION_AGE_DAYS)
313
+ if age is None or age < min_age:
314
+ return None, "insufficient_age"
315
+
316
+ cites = work.get("cited_by_count")
317
+ if not isinstance(cites, int) or isinstance(cites, bool):
318
+ return None, "insufficient_cohort"
319
+
320
+ topic = _obj(work.get("primary_topic"))
321
+ field = _text(_obj(topic.get("field")).get("id")) or None
322
+ domain = _text(_obj(topic.get("domain")).get("id")) or None
323
+ work_type = _text(work.get("type")).lower() or "article"
324
+ published = date.fromisoformat(work["publication_date"][:10])
325
+
326
+ minimum = self.knobs.get("min_cohort_size", config.SCHOLAR_MIN_COHORT_SIZE)
327
+ for subject in _cohort_subjects(field, domain):
328
+ for window in _cohort_windows(published, age):
329
+ base = _cohort_filter(subject, work_type, window)
330
+ total = self._count(base)
331
+ if total is None:
332
+ return None, "insufficient_cohort"
333
+ if total < minimum:
334
+ continue
335
+ below = self._count(f"{base},cited_by_count:<{cites}")
336
+ if below is None:
337
+ return None, "insufficient_cohort"
338
+ return min(1.0, max(0.0, below / total)), "cohort"
339
+ return None, "insufficient_cohort"
340
+
341
+ def author_standing(self, work: dict):
342
+ """The max h-index across this paper's authors, 0-1, or None.
343
+
344
+ ONE BATCHED REQUEST per paper rather than one per author: a paper
345
+ with nine authors is nine lookups the naive way, and the whole
346
+ reason cohort counts are cached is that this stage runs inside a
347
+ pass someone is waiting on.
348
+ """
349
+ ids = []
350
+ authorships = work.get("authorships")
351
+ if not isinstance(authorships, list):
352
+ authorships = []
353
+ for authorship in authorships[:_MAX_AUTHORS]:
354
+ author = _obj(_obj(authorship).get("author"))
355
+ identifier = _text(author.get("id")).rsplit("/", 1)[-1]
356
+ if identifier.startswith("A"):
357
+ ids.append(identifier)
358
+ if not ids:
359
+ return None
360
+ payload = self._get("authors", {
361
+ "filter": "openalex_id:" + "|".join(ids),
362
+ "select": "id,summary_stats", "per-page": len(ids)})
363
+ if not isinstance(payload, dict):
364
+ return None
365
+ results = payload.get("results")
366
+ indexes = [_obj(_obj(entry).get("summary_stats")).get("h_index")
367
+ for entry in (results if isinstance(results, list) else [])]
368
+ return author_standing(
369
+ indexes, self.knobs.get("h_saturation", config.SCHOLAR_H_SATURATION))
370
+
371
+ # -- the whole answer -------------------------------------------------
372
+
373
+ def authority_for(self, url: str, today: date | None = None):
374
+ """(authority, signals) for a cited paper, or (None, None).
375
+
376
+ None means "this is not a paper, or nothing could be looked up" --
377
+ the caller keeps the domain class, which is the right answer for
378
+ both.
379
+ """
380
+ work = self.work(url)
381
+ if work is None:
382
+ return None, None
383
+
384
+ if work.get("is_retracted"):
385
+ # Overrides everything. A retracted paper is not evidence
386
+ # whatever its venue or citation count, and a heavily cited
387
+ # retraction is the most dangerous shape there is.
388
+ return config.SCHOLAR_RETRACTED_AUTHORITY, {
389
+ "openalex_id": work.get("id"), "is_retracted": True,
390
+ "citation_evidence": "retracted"}
391
+
392
+ venue = venue_credit(work)
393
+ percentile, evidence = self.citation_percentile(work, today)
394
+ author = self.author_standing(work)
395
+ score = paper_score(venue, percentile, author, self.knobs)
396
+
397
+ return score, {
398
+ "openalex_id": work.get("id"),
399
+ "peer_reviewed": is_peer_reviewed(work),
400
+ "venue_type": _location_type(work) or None,
401
+ "venue_credit": round(venue, 4),
402
+ "cited_by_count": work.get("cited_by_count"),
403
+ "citation_percentile": (None if percentile is None
404
+ else round(percentile, 4)),
405
+ "citation_evidence": evidence,
406
+ "author_standing": None if author is None else round(author, 4),
407
+ "is_retracted": False,
408
+ }
409
+
410
+
411
+ _WORK_SELECT = ("id,doi,type,publication_date,cited_by_count,is_retracted,"
412
+ "primary_location,primary_topic,authorships")
413
+
414
+ # Enough to find the strongest name on a paper without turning a
415
+ # hyperauthorship record into a 3000-id query string.
416
+ _MAX_AUTHORS = 25
417
+
418
+
419
+ def _cohort_subjects(field, domain):
420
+ """Field first, then domain, then everything of this type and age.
421
+
422
+ WIDENING RATHER THAN GIVING UP, because a narrow field in a narrow
423
+ window can genuinely hold fewer works than a percentile needs -- and a
424
+ percentile against a broader cohort is a weaker statement, not a wrong
425
+ one. The order is most-specific-first so the strongest available
426
+ comparison is the one used.
427
+ """
428
+ seen = []
429
+ for subject in (field, domain, None):
430
+ if subject not in seen:
431
+ seen.append(subject)
432
+ return seen
433
+
434
+
435
+ def _cohort_windows(published: date, age_days: int):
436
+ """The date windows to try, narrowest first.
437
+
438
+ A +/-30 day window for a paper under six months old, because within a
439
+ single calendar year a January paper has eleven months more to
440
+ accumulate citations than a December one -- and at that age eleven
441
+ months is the whole signal. Older than that, the calendar year is both
442
+ sufficient and larger.
443
+ """
444
+ if age_days < 183:
445
+ yield (published - timedelta(days=30), published + timedelta(days=30))
446
+ yield (date(published.year, 1, 1), date(published.year, 12, 31))
447
+
448
+
449
+ def _cohort_filter(subject, work_type: str, window) -> str:
450
+ start, end = window
451
+ parts = [f"type:{work_type}",
452
+ f"from_publication_date:{start.isoformat()}",
453
+ f"to_publication_date:{end.isoformat()}"]
454
+ if subject:
455
+ key = ("primary_topic.domain.id" if "/domains/" in subject
456
+ else "primary_topic.field.id")
457
+ parts.insert(0, f"{key}:{subject}")
458
+ return ",".join(parts)