@yahyahammad/venastine 0.3.8 → 0.3.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/.env.example +19 -19
  2. package/CONFIG_ARCHITECTURE.md +2030 -2002
  3. package/LICENSE +201 -201
  4. package/README.md +1034 -1034
  5. package/agents/builtin/compactor.md +75 -75
  6. package/agents/builtin/explore.md +77 -77
  7. package/agents/builtin/pipeline-reviewer.md +100 -100
  8. package/agents/builtin/plan.md +106 -106
  9. package/agents/builtin/review.md +74 -74
  10. package/agents/manager.py +261 -261
  11. package/agents/subagent_tool.py +472 -472
  12. package/agents/tui_commands.py +238 -146
  13. package/bin/venastine.mjs +528 -528
  14. package/bin/venastine.py +71 -71
  15. package/cli_commands.py +838 -835
  16. package/config.yaml +13 -0
  17. package/config_edit.py +1301 -1301
  18. package/config_schema.py +971 -966
  19. package/config_update.py +749 -749
  20. package/core/agent_activity.py +256 -256
  21. package/core/ansi.py +452 -452
  22. package/core/approval.py +137 -137
  23. package/core/call_view.py +364 -364
  24. package/core/cancel.py +121 -121
  25. package/core/chatgpt_auth.py +747 -747
  26. package/core/claims_view.py +107 -107
  27. package/core/client.py +2715 -2715
  28. package/core/commands.py +460 -454
  29. package/core/compaction.py +1735 -1735
  30. package/core/config_loader.py +1982 -1982
  31. package/core/connect.py +686 -686
  32. package/core/events.py +123 -123
  33. package/core/file_mentions.py +280 -280
  34. package/core/interaction.py +411 -411
  35. package/core/loop.py +2566 -2566
  36. package/core/memory.py +482 -482
  37. package/core/pipeline_models.py +190 -190
  38. package/core/provider_errors.py +234 -234
  39. package/core/reasoning/authorization.py +130 -130
  40. package/core/reasoning/base.py +219 -219
  41. package/core/reasoning/confidence_scoring.py +484 -484
  42. package/core/reasoning/events.py +149 -149
  43. package/core/reasoning/orchestrator.py +1561 -1561
  44. package/core/reasoning/output_writer.py +345 -345
  45. package/core/reasoning/payload_validation.py +524 -524
  46. package/core/reasoning/pipeline_storage.py +427 -427
  47. package/core/reasoning/scholar.py +458 -458
  48. package/core/reasoning/source_corpus.py +433 -433
  49. package/core/reasoning/source_scoring.py +855 -855
  50. package/core/replay.py +315 -315
  51. package/core/shell_sessions.py +1553 -1553
  52. package/credentials.py +307 -307
  53. package/database.py +333 -333
  54. package/docs/PRIVACY.md +78 -78
  55. package/docs/SECURITY.md +108 -108
  56. package/docs/THIRD_PARTY_NOTICES.md +148 -148
  57. package/env_secrets.py +118 -118
  58. package/json_store.py +343 -343
  59. package/logging_setup.py +295 -295
  60. package/main.py +3433 -3433
  61. package/mcp_client/client.py +733 -733
  62. package/mcp_client/config.py +484 -484
  63. package/mcp_client/registration.py +343 -343
  64. package/memories/manager.py +147 -147
  65. package/memories/tui_commands.py +319 -239
  66. package/package.json +303 -303
  67. package/project_init/config_files.py +282 -282
  68. package/project_init/generator.py +728 -728
  69. package/project_init/manifest.py +418 -418
  70. package/project_init/tui_commands.py +147 -147
  71. package/prompts/critic_pass.md +24 -24
  72. package/prompts/revalidate.md +25 -25
  73. package/prompts/revise.md +17 -17
  74. package/prompts/source_grounding.md +66 -66
  75. package/prompts/system_prompts.py +316 -316
  76. package/providers.json.example +94 -94
  77. package/pyproject.toml +149 -149
  78. package/requirements.txt +295 -295
  79. package/safety/policy_enforcement.py +1082 -1082
  80. package/security/askpass.py +419 -419
  81. package/security/capability.py +207 -207
  82. package/security/confine.py +484 -484
  83. package/security/hosted.py +182 -182
  84. package/security/permissions.py +237 -237
  85. package/security/protected_paths.py +501 -501
  86. package/security/sandbox.py +4108 -4108
  87. package/skills/builtin/math/formal-specification.md +67 -67
  88. package/skills/builtin/math/numerical-methods.md +66 -66
  89. package/skills/builtin/research/experiment-design.md +62 -62
  90. package/skills/builtin/research/reproducibility.md +55 -55
  91. package/skills/builtin/research/source-evaluation.md +70 -70
  92. package/skills/builtin/research/statistical-inference.md +63 -63
  93. package/skills/builtin/research/technical-writing.md +59 -59
  94. package/skills/builtin/software/code-review.md +61 -61
  95. package/skills/builtin/software/debugging.md +61 -61
  96. package/skills/builtin/software/test-design.md +67 -67
  97. package/skills/tui_commands.py +110 -110
  98. package/storage.py +1338 -1338
  99. package/tools/base.py +465 -465
  100. package/tools/builtin/_math_common.py +667 -667
  101. package/tools/builtin/_net_common.py +267 -267
  102. package/tools/builtin/arxiv.py +681 -681
  103. package/tools/builtin/fetch_url.py +371 -371
  104. package/tools/builtin/file_ops.py +720 -720
  105. package/tools/builtin/geometry.py +184 -184
  106. package/tools/builtin/load_skill.py +76 -76
  107. package/tools/builtin/pin.py +227 -227
  108. package/tools/builtin/probability_stats.py +224 -224
  109. package/tools/builtin/project_docs.py +464 -464
  110. package/tools/builtin/remember.py +175 -175
  111. package/tools/builtin/shell.py +980 -980
  112. package/tools/builtin/todo.py +149 -149
  113. package/tools/builtin/web_search.py +328 -328
  114. package/tools/context.py +159 -159
  115. package/tools/isolation.py +387 -387
  116. package/tools/registry.py +1242 -1242
  117. package/tui/app.py +8318 -8307
  118. package/tui/app.tcss +747 -739
  119. package/tui/commands.py +26 -26
  120. package/tui/diffs.py +286 -286
  121. package/tui/history.py +128 -128
  122. package/tui/hosted_driver.py +102 -102
  123. package/tui/markdown.py +1329 -1329
  124. package/tui/meters.py +364 -364
  125. package/tui/ravens.py +146 -146
  126. package/tui/screens.py +1255 -1197
  127. package/tui/themes.py +599 -599
  128. package/tui/widgets.py +3813 -3796
@@ -1,458 +1,458 @@
1
- """
2
- core/reasoning/scholar.py
3
-
4
- What a research paper is worth, from OpenAlex.
5
-
6
- ROADMAP_v2 §45 (SQ5/SQ9). A cited paper's authority is more than its
7
- host: arxiv.org serves a landmark and a paper nobody ever read from the
8
- same domain, and the domain class cannot tell them apart. This asks the
9
- one question a domain cannot -- how does this paper stand among its own
10
- peers -- and answers it from counts rather than from a model.
11
-
12
- WHY COHORT COUNTING RATHER THAN OPENALEX'S OWN NORMALISATION. `fwci` and
13
- `citation_normalized_percentile` are exactly the field-and-age-normalised
14
- metrics this needs, and BOTH ARE NULL ON PREPRINTS -- verified against
15
- the API on a paper with 3019 citations. arXiv is the only scholarly
16
- source this harness currently reaches, so the ready-made answer does not
17
- cover the case that matters. Two counting queries do: how many works
18
- share this paper's field, type and publication window, and how many of
19
- them have fewer citations than it. That ratio IS the field-and-age
20
- normalisation, measured rather than modelled, and it self-calibrates
21
- across fields because the comparison set is the same field.
22
-
23
- WHY NO LIFECYCLE CURVE. Matching the cohort on a date window makes the
24
- age normalisation exact rather than approximate, so a fitted accrual
25
- curve would add nothing for a paper old enough to carry signal. Below
26
- that, it would be fitting noise: measured while specifying this, 72% of
27
- computer-science preprints published in 2024 have ZERO citations. A
28
- three-week-old paper's zero says nothing at all, so the term is dropped
29
- and the remaining weights renormalise -- absence of evidence about a new
30
- paper is not evidence of low impact.
31
-
32
- THE EGRESS IS DELIBERATE, NAMED, AND OFF BY DEFAULT (SQ9). This is the
33
- first network call in this harness made by something that is not a
34
- registered tool, so it does not pass through the approval registry and
35
- does not appear in granted_calls.json. That is stated rather than worked
36
- around: routing deterministic scoring code through the tool layer would
37
- make a pure function call a dispatcher. What it does instead is reuse
38
- `is_url_permitted`, cache like the network tools do, trace every lookup,
39
- and stay off until someone turns it on.
40
-
41
- EVERY SCORING FUNCTION HERE IS PURE and takes a plain dict, so the whole
42
- of it is testable against fixtures with no network -- which is not a
43
- convenience, it is the suite's contract.
44
- """
45
-
46
- from __future__ import annotations
47
-
48
- import logging
49
- import math
50
- import re
51
- from datetime import date, timedelta
52
-
53
- import config
54
- from tools.builtin._net_common import TTLCache
55
-
56
- logger = logging.getLogger(__name__)
57
-
58
- _cache = TTLCache(config.SCHOLAR_CACHE_TTL_S)
59
-
60
- # A DOI as it appears inside a URL. Deliberately not anchored: the point
61
- # is to find one in `https://doi.org/10.1234/x` and in a publisher's own
62
- # `.../article/10.1234/x/full`.
63
- _DOI_RE = re.compile(r"10\.\d{4,9}/[^\s?#]+", re.IGNORECASE)
64
-
65
- # Trailing punctuation a URL path collects that a DOI does not own.
66
- _DOI_TRIM = ".,;:)]}>'\"/"
67
-
68
-
69
- def doi_for(url: str) -> str | None:
70
- """The DOI this URL identifies, or None.
71
-
72
- arXiv is handled through `source_corpus.arxiv_id_from_url` rather than
73
- a second id parser here: two copies of "what counts as the same paper"
74
- is how the corpus and this lookup come to disagree about one source in
75
- one artifact.
76
- """
77
- from core.reasoning.source_corpus import arxiv_id_from_url
78
-
79
- arxiv_id = arxiv_id_from_url(url or "")
80
- if arxiv_id:
81
- # The DOI arXiv mints for every submission since 2022, and
82
- # backfilled. OpenAlex indexes preprints under it.
83
- return f"10.48550/arxiv.{arxiv_id}".lower()
84
-
85
- match = _DOI_RE.search(url or "")
86
- if not match:
87
- return None
88
- return match.group(0).rstrip(_DOI_TRIM).lower()
89
-
90
-
91
- # ---------------------------------------------------------------------------
92
- # ---- Pure scoring ---------------------------------------------------------
93
- # ---------------------------------------------------------------------------
94
-
95
- def _obj(value) -> dict:
96
- """`value` if OpenAlex sent an object there, else an empty one.
97
-
98
- Batch 219 (#218). Every read of a record used to trust its shape --
99
- `(x or {}).get(...)` survives a null and nothing else -- so one
100
- non-object where an object was expected raised AttributeError out of
101
- an optional stage and failed a run that had already grounded and
102
- scored. Driven: 104 malformed shapes across ten keys. A shape this
103
- module cannot read is a signal that is absent, and every term here
104
- already knows how to drop an absent one.
105
- """
106
- return value if isinstance(value, dict) else {}
107
-
108
-
109
- def _text(value) -> str:
110
- """`value` if it is a string, else "" -- `_obj`'s rule for a leaf."""
111
- return value if isinstance(value, str) else ""
112
-
113
-
114
- def _location_type(work: dict) -> str:
115
- location = _obj(work.get("primary_location"))
116
- source = _obj(location.get("source"))
117
- return _text(source.get("type")).lower()
118
-
119
-
120
- def is_peer_reviewed(work: dict) -> bool:
121
- """Whether this record is a published, peer-reviewed version.
122
-
123
- THREE FIELDS, because one is not enough. `type == "preprint"` catches
124
- the ordinary case; a repository-hosted record with
125
- `version == "submittedVersion"` catches a work typed as an article
126
- that is really an author manuscript; and `is_published` is the
127
- publisher's own answer where there is one. A record has to survive all
128
- three to count.
129
- """
130
- location = _obj(work.get("primary_location"))
131
- if _text(work.get("type")).lower() == "preprint":
132
- return False
133
- if _location_type(work) not in ("journal", "conference", "book series"):
134
- return False
135
- if location.get("version") == "submittedVersion":
136
- return False
137
- return location.get("is_published", True) is not False
138
-
139
-
140
- def venue_credit(work: dict) -> float:
141
- """What the venue alone says, before any citation is counted."""
142
- if is_peer_reviewed(work):
143
- return config.SCHOLAR_VENUE_CREDIT.get(
144
- _location_type(work), config.SCHOLAR_VENUE_CREDIT["unknown"])
145
- return config.SCHOLAR_VENUE_CREDIT["repository"]
146
-
147
-
148
- def author_standing(h_indexes, saturation: int | None = None):
149
- """max h-index across the authors, log-saturated onto 0-1, or None.
150
-
151
- MAX rather than mean: a first-author graduate student on a paper with
152
- a senior collaborator does not make the work less credible, and
153
- averaging would say it does.
154
-
155
- SATURATING rather than linear, because the difference between an
156
- h-index of 60 and one of 90 says more about field size and career
157
- length than about this paper. None when no author has a usable index
158
- -- the term is then dropped and the rest renormalise, rather than
159
- scoring an unindexed author as a bad one.
160
- """
161
- saturation = saturation or config.SCHOLAR_H_SATURATION
162
- usable = [h for h in (h_indexes or [])
163
- if isinstance(h, int) and not isinstance(h, bool) and h >= 0]
164
- if not usable:
165
- return None
166
- return min(1.0, math.log1p(max(usable)) / math.log1p(saturation))
167
-
168
-
169
- def paper_score(venue, citation, author, weights: dict | None = None) -> float:
170
- """Blend whichever signals exist, renormalising over them.
171
-
172
- THE RENORMALISATION IS THE POINT. A three-week-old preprint has no
173
- citation percentile and an unindexed author has no h-index; folding a
174
- missing term in as zero would score "we could not measure this" and
175
- "this measured badly" identically, which is the distinction §45 keeps
176
- making. So a missing term takes its weight out of the denominator too.
177
- """
178
- weights = weights or {}
179
- terms = [
180
- (venue, weights.get("venue_weight", config.SCHOLAR_VENUE_WEIGHT)),
181
- (citation, weights.get("citation_weight", config.SCHOLAR_CITATION_WEIGHT)),
182
- (author, weights.get("author_weight", config.SCHOLAR_AUTHOR_WEIGHT)),
183
- ]
184
- present = [(value, weight) for value, weight in terms
185
- if value is not None and weight > 0]
186
- if not present:
187
- return config.DEFAULT_DOMAIN_AUTHORITY
188
- total = sum(weight for _, weight in present)
189
- return min(1.0, max(0.0, sum(value * weight for value, weight in present) / total))
190
-
191
-
192
- def _age_days(work: dict, today: date | None = None) -> int | None:
193
- published = work.get("publication_date")
194
- if not isinstance(published, str):
195
- return None
196
- try:
197
- when = date.fromisoformat(published[:10])
198
- except ValueError:
199
- return None
200
- return ((today or date.today()) - when).days
201
-
202
-
203
- # ---------------------------------------------------------------------------
204
- # ---- The client -----------------------------------------------------------
205
- # ---------------------------------------------------------------------------
206
-
207
- class ScholarLookup:
208
- """OpenAlex, cached and bounded, or a no-op when disabled.
209
-
210
- A CLASS holding one run's cache and failure state, for
211
- EmbeddingScorer's reason: one outage should stop the lookups rather
212
- than be retried once per paper, and the run should say so once.
213
- """
214
-
215
- def __init__(self, enabled: bool | None = None, fetch=None,
216
- knobs: dict | None = None) -> None:
217
- self.enabled = (config.SCHOLAR_LOOKUP if enabled is None else enabled)
218
- self.knobs = knobs or {}
219
- self.failed_reason = None
220
- self.calls = 0
221
- self._fetch = fetch or self._http_get
222
- self._cohorts: dict = {}
223
-
224
- @property
225
- def usable(self) -> bool:
226
- return self.enabled and self.failed_reason is None
227
-
228
- # -- transport -------------------------------------------------------
229
-
230
- def _http_get(self, path: str, params: dict):
231
- """The only place this module touches the network.
232
-
233
- INJECTABLE, and every test injects: the suite runs offline, and a
234
- scoring rule tested through a real socket is a scoring rule tested
235
- on somebody's connection.
236
- """
237
- import httpx
238
-
239
- from safety.policy_enforcement import is_url_permitted
240
-
241
- url = f"{config.SCHOLAR_API_URL.rstrip('/')}/{path.lstrip('/')}"
242
- refusal = is_url_permitted(url, resolve=False)
243
- if refusal:
244
- raise RuntimeError(refusal)
245
- if config.SCHOLAR_MAILTO:
246
- # Only ever what the user configured. An address taken from a
247
- # git config or an environment variable is not a rate-limit
248
- # optimisation, it is an unasked-for disclosure.
249
- params = {**params, "mailto": config.SCHOLAR_MAILTO}
250
- response = httpx.get(
251
- url, params=params, timeout=config.SCHOLAR_TIMEOUT_S,
252
- headers={"User-Agent": "venastine-research-harness/1.0"},
253
- follow_redirects=True)
254
- response.raise_for_status()
255
- return response.json()
256
-
257
- def _get(self, path: str, params: dict):
258
- """Cached, retried, and terminal on exhaustion. None on failure."""
259
- if not self.usable:
260
- return None
261
- key = f"{path}|{sorted(params.items())}"
262
- hit = _cache.get(key)
263
- if hit is not None:
264
- return hit
265
-
266
- last = None
267
- for _ in range(config.SCHOLAR_MAX_RETRIES + 1):
268
- try:
269
- payload = self._fetch(path, params)
270
- except Exception as exc: # noqa: BLE001 - degraded, never fatal
271
- last = exc
272
- continue
273
- self.calls += 1
274
- _cache.set(key, payload)
275
- return payload
276
- # A 404 is not an outage -- plenty of cited URLs are not indexed
277
- # works -- so only a repeated failure disables the lookup, and the
278
- # caller distinguishes "no record" from "could not ask".
279
- self.failed_reason = f"{type(last).__name__}: {last}"
280
- return None
281
-
282
- # -- lookups ---------------------------------------------------------
283
-
284
- def work(self, url: str):
285
- """The OpenAlex record for `url`, or None if it is not a paper."""
286
- doi = doi_for(url)
287
- if not doi or not self.usable:
288
- return None
289
- payload = self._get(f"works/doi:{doi}", {"select": _WORK_SELECT})
290
- if not isinstance(payload, dict) or not payload.get("id"):
291
- return None
292
- return payload
293
-
294
- def _count(self, filters: str) -> int | None:
295
- payload = self._get("works", {"filter": filters, "per_page": 1,
296
- "select": "id"})
297
- if not isinstance(payload, dict):
298
- return None
299
- count = _obj(payload.get("meta")).get("count")
300
- return count if isinstance(count, int) else None
301
-
302
- def citation_percentile(self, work: dict, today: date | None = None):
303
- """Where this paper sits among works like it, or None with a reason.
304
-
305
- Returns (percentile, evidence) where evidence is one of
306
- "cohort", "insufficient_age" or "insufficient_cohort" -- the
307
- second and third are answers, not failures, and the caller drops
308
- the term rather than scoring it low.
309
- """
310
- age = _age_days(work, today)
311
- min_age = self.knobs.get("min_citation_age_days",
312
- config.SCHOLAR_MIN_CITATION_AGE_DAYS)
313
- if age is None or age < min_age:
314
- return None, "insufficient_age"
315
-
316
- cites = work.get("cited_by_count")
317
- if not isinstance(cites, int) or isinstance(cites, bool):
318
- return None, "insufficient_cohort"
319
-
320
- topic = _obj(work.get("primary_topic"))
321
- field = _text(_obj(topic.get("field")).get("id")) or None
322
- domain = _text(_obj(topic.get("domain")).get("id")) or None
323
- work_type = _text(work.get("type")).lower() or "article"
324
- published = date.fromisoformat(work["publication_date"][:10])
325
-
326
- minimum = self.knobs.get("min_cohort_size", config.SCHOLAR_MIN_COHORT_SIZE)
327
- for subject in _cohort_subjects(field, domain):
328
- for window in _cohort_windows(published, age):
329
- base = _cohort_filter(subject, work_type, window)
330
- total = self._count(base)
331
- if total is None:
332
- return None, "insufficient_cohort"
333
- if total < minimum:
334
- continue
335
- below = self._count(f"{base},cited_by_count:<{cites}")
336
- if below is None:
337
- return None, "insufficient_cohort"
338
- return min(1.0, max(0.0, below / total)), "cohort"
339
- return None, "insufficient_cohort"
340
-
341
- def author_standing(self, work: dict):
342
- """The max h-index across this paper's authors, 0-1, or None.
343
-
344
- ONE BATCHED REQUEST per paper rather than one per author: a paper
345
- with nine authors is nine lookups the naive way, and the whole
346
- reason cohort counts are cached is that this stage runs inside a
347
- pass someone is waiting on.
348
- """
349
- ids = []
350
- authorships = work.get("authorships")
351
- if not isinstance(authorships, list):
352
- authorships = []
353
- for authorship in authorships[:_MAX_AUTHORS]:
354
- author = _obj(_obj(authorship).get("author"))
355
- identifier = _text(author.get("id")).rsplit("/", 1)[-1]
356
- if identifier.startswith("A"):
357
- ids.append(identifier)
358
- if not ids:
359
- return None
360
- payload = self._get("authors", {
361
- "filter": "openalex_id:" + "|".join(ids),
362
- "select": "id,summary_stats", "per-page": len(ids)})
363
- if not isinstance(payload, dict):
364
- return None
365
- results = payload.get("results")
366
- indexes = [_obj(_obj(entry).get("summary_stats")).get("h_index")
367
- for entry in (results if isinstance(results, list) else [])]
368
- return author_standing(
369
- indexes, self.knobs.get("h_saturation", config.SCHOLAR_H_SATURATION))
370
-
371
- # -- the whole answer -------------------------------------------------
372
-
373
- def authority_for(self, url: str, today: date | None = None):
374
- """(authority, signals) for a cited paper, or (None, None).
375
-
376
- None means "this is not a paper, or nothing could be looked up" --
377
- the caller keeps the domain class, which is the right answer for
378
- both.
379
- """
380
- work = self.work(url)
381
- if work is None:
382
- return None, None
383
-
384
- if work.get("is_retracted"):
385
- # Overrides everything. A retracted paper is not evidence
386
- # whatever its venue or citation count, and a heavily cited
387
- # retraction is the most dangerous shape there is.
388
- return config.SCHOLAR_RETRACTED_AUTHORITY, {
389
- "openalex_id": work.get("id"), "is_retracted": True,
390
- "citation_evidence": "retracted"}
391
-
392
- venue = venue_credit(work)
393
- percentile, evidence = self.citation_percentile(work, today)
394
- author = self.author_standing(work)
395
- score = paper_score(venue, percentile, author, self.knobs)
396
-
397
- return score, {
398
- "openalex_id": work.get("id"),
399
- "peer_reviewed": is_peer_reviewed(work),
400
- "venue_type": _location_type(work) or None,
401
- "venue_credit": round(venue, 4),
402
- "cited_by_count": work.get("cited_by_count"),
403
- "citation_percentile": (None if percentile is None
404
- else round(percentile, 4)),
405
- "citation_evidence": evidence,
406
- "author_standing": None if author is None else round(author, 4),
407
- "is_retracted": False,
408
- }
409
-
410
-
411
- _WORK_SELECT = ("id,doi,type,publication_date,cited_by_count,is_retracted,"
412
- "primary_location,primary_topic,authorships")
413
-
414
- # Enough to find the strongest name on a paper without turning a
415
- # hyperauthorship record into a 3000-id query string.
416
- _MAX_AUTHORS = 25
417
-
418
-
419
- def _cohort_subjects(field, domain):
420
- """Field first, then domain, then everything of this type and age.
421
-
422
- WIDENING RATHER THAN GIVING UP, because a narrow field in a narrow
423
- window can genuinely hold fewer works than a percentile needs -- and a
424
- percentile against a broader cohort is a weaker statement, not a wrong
425
- one. The order is most-specific-first so the strongest available
426
- comparison is the one used.
427
- """
428
- seen = []
429
- for subject in (field, domain, None):
430
- if subject not in seen:
431
- seen.append(subject)
432
- return seen
433
-
434
-
435
- def _cohort_windows(published: date, age_days: int):
436
- """The date windows to try, narrowest first.
437
-
438
- A +/-30 day window for a paper under six months old, because within a
439
- single calendar year a January paper has eleven months more to
440
- accumulate citations than a December one -- and at that age eleven
441
- months is the whole signal. Older than that, the calendar year is both
442
- sufficient and larger.
443
- """
444
- if age_days < 183:
445
- yield (published - timedelta(days=30), published + timedelta(days=30))
446
- yield (date(published.year, 1, 1), date(published.year, 12, 31))
447
-
448
-
449
- def _cohort_filter(subject, work_type: str, window) -> str:
450
- start, end = window
451
- parts = [f"type:{work_type}",
452
- f"from_publication_date:{start.isoformat()}",
453
- f"to_publication_date:{end.isoformat()}"]
454
- if subject:
455
- key = ("primary_topic.domain.id" if "/domains/" in subject
456
- else "primary_topic.field.id")
457
- parts.insert(0, f"{key}:{subject}")
458
- return ",".join(parts)
1
+ """
2
+ core/reasoning/scholar.py
3
+
4
+ What a research paper is worth, from OpenAlex.
5
+
6
+ ROADMAP_v2 §45 (SQ5/SQ9). A cited paper's authority is more than its
7
+ host: arxiv.org serves a landmark and a paper nobody ever read from the
8
+ same domain, and the domain class cannot tell them apart. This asks the
9
+ one question a domain cannot -- how does this paper stand among its own
10
+ peers -- and answers it from counts rather than from a model.
11
+
12
+ WHY COHORT COUNTING RATHER THAN OPENALEX'S OWN NORMALISATION. `fwci` and
13
+ `citation_normalized_percentile` are exactly the field-and-age-normalised
14
+ metrics this needs, and BOTH ARE NULL ON PREPRINTS -- verified against
15
+ the API on a paper with 3019 citations. arXiv is the only scholarly
16
+ source this harness currently reaches, so the ready-made answer does not
17
+ cover the case that matters. Two counting queries do: how many works
18
+ share this paper's field, type and publication window, and how many of
19
+ them have fewer citations than it. That ratio IS the field-and-age
20
+ normalisation, measured rather than modelled, and it self-calibrates
21
+ across fields because the comparison set is the same field.
22
+
23
+ WHY NO LIFECYCLE CURVE. Matching the cohort on a date window makes the
24
+ age normalisation exact rather than approximate, so a fitted accrual
25
+ curve would add nothing for a paper old enough to carry signal. Below
26
+ that, it would be fitting noise: measured while specifying this, 72% of
27
+ computer-science preprints published in 2024 have ZERO citations. A
28
+ three-week-old paper's zero says nothing at all, so the term is dropped
29
+ and the remaining weights renormalise -- absence of evidence about a new
30
+ paper is not evidence of low impact.
31
+
32
+ THE EGRESS IS DELIBERATE, NAMED, AND OFF BY DEFAULT (SQ9). This is the
33
+ first network call in this harness made by something that is not a
34
+ registered tool, so it does not pass through the approval registry and
35
+ does not appear in granted_calls.json. That is stated rather than worked
36
+ around: routing deterministic scoring code through the tool layer would
37
+ make a pure function call a dispatcher. What it does instead is reuse
38
+ `is_url_permitted`, cache like the network tools do, trace every lookup,
39
+ and stay off until someone turns it on.
40
+
41
+ EVERY SCORING FUNCTION HERE IS PURE and takes a plain dict, so the whole
42
+ of it is testable against fixtures with no network -- which is not a
43
+ convenience, it is the suite's contract.
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ import logging
49
+ import math
50
+ import re
51
+ from datetime import date, timedelta
52
+
53
+ import config
54
+ from tools.builtin._net_common import TTLCache
55
+
56
+ logger = logging.getLogger(__name__)
57
+
58
+ _cache = TTLCache(config.SCHOLAR_CACHE_TTL_S)
59
+
60
+ # A DOI as it appears inside a URL. Deliberately not anchored: the point
61
+ # is to find one in `https://doi.org/10.1234/x` and in a publisher's own
62
+ # `.../article/10.1234/x/full`.
63
+ _DOI_RE = re.compile(r"10\.\d{4,9}/[^\s?#]+", re.IGNORECASE)
64
+
65
+ # Trailing punctuation a URL path collects that a DOI does not own.
66
+ _DOI_TRIM = ".,;:)]}>'\"/"
67
+
68
+
69
+ def doi_for(url: str) -> str | None:
70
+ """The DOI this URL identifies, or None.
71
+
72
+ arXiv is handled through `source_corpus.arxiv_id_from_url` rather than
73
+ a second id parser here: two copies of "what counts as the same paper"
74
+ is how the corpus and this lookup come to disagree about one source in
75
+ one artifact.
76
+ """
77
+ from core.reasoning.source_corpus import arxiv_id_from_url
78
+
79
+ arxiv_id = arxiv_id_from_url(url or "")
80
+ if arxiv_id:
81
+ # The DOI arXiv mints for every submission since 2022, and
82
+ # backfilled. OpenAlex indexes preprints under it.
83
+ return f"10.48550/arxiv.{arxiv_id}".lower()
84
+
85
+ match = _DOI_RE.search(url or "")
86
+ if not match:
87
+ return None
88
+ return match.group(0).rstrip(_DOI_TRIM).lower()
89
+
90
+
91
+ # ---------------------------------------------------------------------------
92
+ # ---- Pure scoring ---------------------------------------------------------
93
+ # ---------------------------------------------------------------------------
94
+
95
+ def _obj(value) -> dict:
96
+ """`value` if OpenAlex sent an object there, else an empty one.
97
+
98
+ Batch 219 (#218). Every read of a record used to trust its shape --
99
+ `(x or {}).get(...)` survives a null and nothing else -- so one
100
+ non-object where an object was expected raised AttributeError out of
101
+ an optional stage and failed a run that had already grounded and
102
+ scored. Driven: 104 malformed shapes across ten keys. A shape this
103
+ module cannot read is a signal that is absent, and every term here
104
+ already knows how to drop an absent one.
105
+ """
106
+ return value if isinstance(value, dict) else {}
107
+
108
+
109
+ def _text(value) -> str:
110
+ """`value` if it is a string, else "" -- `_obj`'s rule for a leaf."""
111
+ return value if isinstance(value, str) else ""
112
+
113
+
114
+ def _location_type(work: dict) -> str:
115
+ location = _obj(work.get("primary_location"))
116
+ source = _obj(location.get("source"))
117
+ return _text(source.get("type")).lower()
118
+
119
+
120
+ def is_peer_reviewed(work: dict) -> bool:
121
+ """Whether this record is a published, peer-reviewed version.
122
+
123
+ THREE FIELDS, because one is not enough. `type == "preprint"` catches
124
+ the ordinary case; a repository-hosted record with
125
+ `version == "submittedVersion"` catches a work typed as an article
126
+ that is really an author manuscript; and `is_published` is the
127
+ publisher's own answer where there is one. A record has to survive all
128
+ three to count.
129
+ """
130
+ location = _obj(work.get("primary_location"))
131
+ if _text(work.get("type")).lower() == "preprint":
132
+ return False
133
+ if _location_type(work) not in ("journal", "conference", "book series"):
134
+ return False
135
+ if location.get("version") == "submittedVersion":
136
+ return False
137
+ return location.get("is_published", True) is not False
138
+
139
+
140
+ def venue_credit(work: dict) -> float:
141
+ """What the venue alone says, before any citation is counted."""
142
+ if is_peer_reviewed(work):
143
+ return config.SCHOLAR_VENUE_CREDIT.get(
144
+ _location_type(work), config.SCHOLAR_VENUE_CREDIT["unknown"])
145
+ return config.SCHOLAR_VENUE_CREDIT["repository"]
146
+
147
+
148
+ def author_standing(h_indexes, saturation: int | None = None):
149
+ """max h-index across the authors, log-saturated onto 0-1, or None.
150
+
151
+ MAX rather than mean: a first-author graduate student on a paper with
152
+ a senior collaborator does not make the work less credible, and
153
+ averaging would say it does.
154
+
155
+ SATURATING rather than linear, because the difference between an
156
+ h-index of 60 and one of 90 says more about field size and career
157
+ length than about this paper. None when no author has a usable index
158
+ -- the term is then dropped and the rest renormalise, rather than
159
+ scoring an unindexed author as a bad one.
160
+ """
161
+ saturation = saturation or config.SCHOLAR_H_SATURATION
162
+ usable = [h for h in (h_indexes or [])
163
+ if isinstance(h, int) and not isinstance(h, bool) and h >= 0]
164
+ if not usable:
165
+ return None
166
+ return min(1.0, math.log1p(max(usable)) / math.log1p(saturation))
167
+
168
+
169
+ def paper_score(venue, citation, author, weights: dict | None = None) -> float:
170
+ """Blend whichever signals exist, renormalising over them.
171
+
172
+ THE RENORMALISATION IS THE POINT. A three-week-old preprint has no
173
+ citation percentile and an unindexed author has no h-index; folding a
174
+ missing term in as zero would score "we could not measure this" and
175
+ "this measured badly" identically, which is the distinction §45 keeps
176
+ making. So a missing term takes its weight out of the denominator too.
177
+ """
178
+ weights = weights or {}
179
+ terms = [
180
+ (venue, weights.get("venue_weight", config.SCHOLAR_VENUE_WEIGHT)),
181
+ (citation, weights.get("citation_weight", config.SCHOLAR_CITATION_WEIGHT)),
182
+ (author, weights.get("author_weight", config.SCHOLAR_AUTHOR_WEIGHT)),
183
+ ]
184
+ present = [(value, weight) for value, weight in terms
185
+ if value is not None and weight > 0]
186
+ if not present:
187
+ return config.DEFAULT_DOMAIN_AUTHORITY
188
+ total = sum(weight for _, weight in present)
189
+ return min(1.0, max(0.0, sum(value * weight for value, weight in present) / total))
190
+
191
+
192
+ def _age_days(work: dict, today: date | None = None) -> int | None:
193
+ published = work.get("publication_date")
194
+ if not isinstance(published, str):
195
+ return None
196
+ try:
197
+ when = date.fromisoformat(published[:10])
198
+ except ValueError:
199
+ return None
200
+ return ((today or date.today()) - when).days
201
+
202
+
203
+ # ---------------------------------------------------------------------------
204
+ # ---- The client -----------------------------------------------------------
205
+ # ---------------------------------------------------------------------------
206
+
207
+ class ScholarLookup:
208
+ """OpenAlex, cached and bounded, or a no-op when disabled.
209
+
210
+ A CLASS holding one run's cache and failure state, for
211
+ EmbeddingScorer's reason: one outage should stop the lookups rather
212
+ than be retried once per paper, and the run should say so once.
213
+ """
214
+
215
+ def __init__(self, enabled: bool | None = None, fetch=None,
216
+ knobs: dict | None = None) -> None:
217
+ self.enabled = (config.SCHOLAR_LOOKUP if enabled is None else enabled)
218
+ self.knobs = knobs or {}
219
+ self.failed_reason = None
220
+ self.calls = 0
221
+ self._fetch = fetch or self._http_get
222
+ self._cohorts: dict = {}
223
+
224
+ @property
225
+ def usable(self) -> bool:
226
+ return self.enabled and self.failed_reason is None
227
+
228
+ # -- transport -------------------------------------------------------
229
+
230
+ def _http_get(self, path: str, params: dict):
231
+ """The only place this module touches the network.
232
+
233
+ INJECTABLE, and every test injects: the suite runs offline, and a
234
+ scoring rule tested through a real socket is a scoring rule tested
235
+ on somebody's connection.
236
+ """
237
+ import httpx
238
+
239
+ from safety.policy_enforcement import is_url_permitted
240
+
241
+ url = f"{config.SCHOLAR_API_URL.rstrip('/')}/{path.lstrip('/')}"
242
+ refusal = is_url_permitted(url, resolve=False)
243
+ if refusal:
244
+ raise RuntimeError(refusal)
245
+ if config.SCHOLAR_MAILTO:
246
+ # Only ever what the user configured. An address taken from a
247
+ # git config or an environment variable is not a rate-limit
248
+ # optimisation, it is an unasked-for disclosure.
249
+ params = {**params, "mailto": config.SCHOLAR_MAILTO}
250
+ response = httpx.get(
251
+ url, params=params, timeout=config.SCHOLAR_TIMEOUT_S,
252
+ headers={"User-Agent": "venastine-research-harness/1.0"},
253
+ follow_redirects=True)
254
+ response.raise_for_status()
255
+ return response.json()
256
+
257
+ def _get(self, path: str, params: dict):
258
+ """Cached, retried, and terminal on exhaustion. None on failure."""
259
+ if not self.usable:
260
+ return None
261
+ key = f"{path}|{sorted(params.items())}"
262
+ hit = _cache.get(key)
263
+ if hit is not None:
264
+ return hit
265
+
266
+ last = None
267
+ for _ in range(config.SCHOLAR_MAX_RETRIES + 1):
268
+ try:
269
+ payload = self._fetch(path, params)
270
+ except Exception as exc: # noqa: BLE001 - degraded, never fatal
271
+ last = exc
272
+ continue
273
+ self.calls += 1
274
+ _cache.set(key, payload)
275
+ return payload
276
+ # A 404 is not an outage -- plenty of cited URLs are not indexed
277
+ # works -- so only a repeated failure disables the lookup, and the
278
+ # caller distinguishes "no record" from "could not ask".
279
+ self.failed_reason = f"{type(last).__name__}: {last}"
280
+ return None
281
+
282
+ # -- lookups ---------------------------------------------------------
283
+
284
+ def work(self, url: str):
285
+ """The OpenAlex record for `url`, or None if it is not a paper."""
286
+ doi = doi_for(url)
287
+ if not doi or not self.usable:
288
+ return None
289
+ payload = self._get(f"works/doi:{doi}", {"select": _WORK_SELECT})
290
+ if not isinstance(payload, dict) or not payload.get("id"):
291
+ return None
292
+ return payload
293
+
294
+ def _count(self, filters: str) -> int | None:
295
+ payload = self._get("works", {"filter": filters, "per_page": 1,
296
+ "select": "id"})
297
+ if not isinstance(payload, dict):
298
+ return None
299
+ count = _obj(payload.get("meta")).get("count")
300
+ return count if isinstance(count, int) else None
301
+
302
+ def citation_percentile(self, work: dict, today: date | None = None):
303
+ """Where this paper sits among works like it, or None with a reason.
304
+
305
+ Returns (percentile, evidence) where evidence is one of
306
+ "cohort", "insufficient_age" or "insufficient_cohort" -- the
307
+ second and third are answers, not failures, and the caller drops
308
+ the term rather than scoring it low.
309
+ """
310
+ age = _age_days(work, today)
311
+ min_age = self.knobs.get("min_citation_age_days",
312
+ config.SCHOLAR_MIN_CITATION_AGE_DAYS)
313
+ if age is None or age < min_age:
314
+ return None, "insufficient_age"
315
+
316
+ cites = work.get("cited_by_count")
317
+ if not isinstance(cites, int) or isinstance(cites, bool):
318
+ return None, "insufficient_cohort"
319
+
320
+ topic = _obj(work.get("primary_topic"))
321
+ field = _text(_obj(topic.get("field")).get("id")) or None
322
+ domain = _text(_obj(topic.get("domain")).get("id")) or None
323
+ work_type = _text(work.get("type")).lower() or "article"
324
+ published = date.fromisoformat(work["publication_date"][:10])
325
+
326
+ minimum = self.knobs.get("min_cohort_size", config.SCHOLAR_MIN_COHORT_SIZE)
327
+ for subject in _cohort_subjects(field, domain):
328
+ for window in _cohort_windows(published, age):
329
+ base = _cohort_filter(subject, work_type, window)
330
+ total = self._count(base)
331
+ if total is None:
332
+ return None, "insufficient_cohort"
333
+ if total < minimum:
334
+ continue
335
+ below = self._count(f"{base},cited_by_count:<{cites}")
336
+ if below is None:
337
+ return None, "insufficient_cohort"
338
+ return min(1.0, max(0.0, below / total)), "cohort"
339
+ return None, "insufficient_cohort"
340
+
341
+ def author_standing(self, work: dict):
342
+ """The max h-index across this paper's authors, 0-1, or None.
343
+
344
+ ONE BATCHED REQUEST per paper rather than one per author: a paper
345
+ with nine authors is nine lookups the naive way, and the whole
346
+ reason cohort counts are cached is that this stage runs inside a
347
+ pass someone is waiting on.
348
+ """
349
+ ids = []
350
+ authorships = work.get("authorships")
351
+ if not isinstance(authorships, list):
352
+ authorships = []
353
+ for authorship in authorships[:_MAX_AUTHORS]:
354
+ author = _obj(_obj(authorship).get("author"))
355
+ identifier = _text(author.get("id")).rsplit("/", 1)[-1]
356
+ if identifier.startswith("A"):
357
+ ids.append(identifier)
358
+ if not ids:
359
+ return None
360
+ payload = self._get("authors", {
361
+ "filter": "openalex_id:" + "|".join(ids),
362
+ "select": "id,summary_stats", "per-page": len(ids)})
363
+ if not isinstance(payload, dict):
364
+ return None
365
+ results = payload.get("results")
366
+ indexes = [_obj(_obj(entry).get("summary_stats")).get("h_index")
367
+ for entry in (results if isinstance(results, list) else [])]
368
+ return author_standing(
369
+ indexes, self.knobs.get("h_saturation", config.SCHOLAR_H_SATURATION))
370
+
371
+ # -- the whole answer -------------------------------------------------
372
+
373
+ def authority_for(self, url: str, today: date | None = None):
374
+ """(authority, signals) for a cited paper, or (None, None).
375
+
376
+ None means "this is not a paper, or nothing could be looked up" --
377
+ the caller keeps the domain class, which is the right answer for
378
+ both.
379
+ """
380
+ work = self.work(url)
381
+ if work is None:
382
+ return None, None
383
+
384
+ if work.get("is_retracted"):
385
+ # Overrides everything. A retracted paper is not evidence
386
+ # whatever its venue or citation count, and a heavily cited
387
+ # retraction is the most dangerous shape there is.
388
+ return config.SCHOLAR_RETRACTED_AUTHORITY, {
389
+ "openalex_id": work.get("id"), "is_retracted": True,
390
+ "citation_evidence": "retracted"}
391
+
392
+ venue = venue_credit(work)
393
+ percentile, evidence = self.citation_percentile(work, today)
394
+ author = self.author_standing(work)
395
+ score = paper_score(venue, percentile, author, self.knobs)
396
+
397
+ return score, {
398
+ "openalex_id": work.get("id"),
399
+ "peer_reviewed": is_peer_reviewed(work),
400
+ "venue_type": _location_type(work) or None,
401
+ "venue_credit": round(venue, 4),
402
+ "cited_by_count": work.get("cited_by_count"),
403
+ "citation_percentile": (None if percentile is None
404
+ else round(percentile, 4)),
405
+ "citation_evidence": evidence,
406
+ "author_standing": None if author is None else round(author, 4),
407
+ "is_retracted": False,
408
+ }
409
+
410
+
411
+ _WORK_SELECT = ("id,doi,type,publication_date,cited_by_count,is_retracted,"
412
+ "primary_location,primary_topic,authorships")
413
+
414
+ # Enough to find the strongest name on a paper without turning a
415
+ # hyperauthorship record into a 3000-id query string.
416
+ _MAX_AUTHORS = 25
417
+
418
+
419
+ def _cohort_subjects(field, domain):
420
+ """Field first, then domain, then everything of this type and age.
421
+
422
+ WIDENING RATHER THAN GIVING UP, because a narrow field in a narrow
423
+ window can genuinely hold fewer works than a percentile needs -- and a
424
+ percentile against a broader cohort is a weaker statement, not a wrong
425
+ one. The order is most-specific-first so the strongest available
426
+ comparison is the one used.
427
+ """
428
+ seen = []
429
+ for subject in (field, domain, None):
430
+ if subject not in seen:
431
+ seen.append(subject)
432
+ return seen
433
+
434
+
435
+ def _cohort_windows(published: date, age_days: int):
436
+ """The date windows to try, narrowest first.
437
+
438
+ A +/-30 day window for a paper under six months old, because within a
439
+ single calendar year a January paper has eleven months more to
440
+ accumulate citations than a December one -- and at that age eleven
441
+ months is the whole signal. Older than that, the calendar year is both
442
+ sufficient and larger.
443
+ """
444
+ if age_days < 183:
445
+ yield (published - timedelta(days=30), published + timedelta(days=30))
446
+ yield (date(published.year, 1, 1), date(published.year, 12, 31))
447
+
448
+
449
+ def _cohort_filter(subject, work_type: str, window) -> str:
450
+ start, end = window
451
+ parts = [f"type:{work_type}",
452
+ f"from_publication_date:{start.isoformat()}",
453
+ f"to_publication_date:{end.isoformat()}"]
454
+ if subject:
455
+ key = ("primary_topic.domain.id" if "/domains/" in subject
456
+ else "primary_topic.field.id")
457
+ parts.insert(0, f"{key}:{subject}")
458
+ return ",".join(parts)