@yahyahammad/venastine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +14 -0
- package/LICENSE +201 -0
- package/NOTICE +14 -0
- package/PRIVACY.md +66 -0
- package/README.md +878 -0
- package/SECURITY.md +88 -0
- package/THIRD_PARTY_NOTICES.md +100 -0
- package/agents/builtin/compactor.md +75 -0
- package/agents/builtin/explore.md +77 -0
- package/agents/builtin/grill-me.md +27 -0
- package/agents/builtin/initializer.md +94 -0
- package/agents/builtin/pipeline-reviewer.md +100 -0
- package/agents/builtin/plan.md +88 -0
- package/agents/builtin/review.md +74 -0
- package/agents/manager.py +195 -0
- package/agents/subagent_tool.py +215 -0
- package/agents/tui_commands.py +136 -0
- package/bin/venastine.mjs +382 -0
- package/config.py +1198 -0
- package/core/__init__.py +0 -0
- package/core/approval.py +115 -0
- package/core/client.py +1548 -0
- package/core/compaction.py +959 -0
- package/core/config_loader.py +1563 -0
- package/core/events.py +61 -0
- package/core/interaction.py +328 -0
- package/core/loop.py +1353 -0
- package/core/memory.py +370 -0
- package/core/model_windows.py +175 -0
- package/core/pipeline_models.py +176 -0
- package/core/reasoning/__init__.py +0 -0
- package/core/reasoning/authorization.py +130 -0
- package/core/reasoning/base.py +219 -0
- package/core/reasoning/confidence_scoring.py +468 -0
- package/core/reasoning/events.py +149 -0
- package/core/reasoning/json_retry.py +219 -0
- package/core/reasoning/orchestrator.py +1382 -0
- package/core/reasoning/output_writer.py +286 -0
- package/core/reasoning/payload_validation.py +398 -0
- package/core/reasoning/pipeline_storage.py +361 -0
- package/core/reasoning/review.py +615 -0
- package/core/reasoning/scholar.py +434 -0
- package/core/reasoning/source_corpus.py +327 -0
- package/core/reasoning/source_scoring.py +855 -0
- package/core/replay.py +167 -0
- package/core/session.py +153 -0
- package/core/workspace_trust.py +207 -0
- package/credentials.py +180 -0
- package/database.py +252 -0
- package/env_secrets.py +77 -0
- package/json_store.py +123 -0
- package/logging_setup.py +185 -0
- package/main.py +1959 -0
- package/mcp_client/__init__.py +15 -0
- package/mcp_client/client.py +499 -0
- package/mcp_client/config.py +379 -0
- package/mcp_client/registration.py +182 -0
- package/memories/manager.py +132 -0
- package/memories/tui_commands.py +232 -0
- package/package.json +73 -0
- package/project_init/config_files.py +278 -0
- package/project_init/doc_sets.py +330 -0
- package/project_init/generator.py +564 -0
- package/project_init/manifest.py +406 -0
- package/project_init/tui_commands.py +144 -0
- package/prompts/__init__.py +0 -0
- package/prompts/annotate.md +14 -0
- package/prompts/assumption_audit.md +27 -0
- package/prompts/claim_extraction.md +34 -0
- package/prompts/completeness.md +18 -0
- package/prompts/confidence_tiers.md +23 -0
- package/prompts/critic_pass.md +24 -0
- package/prompts/final_synthesis.md +13 -0
- package/prompts/initial_generation.md +14 -0
- package/prompts/preliminary_plan.md +23 -0
- package/prompts/revalidate.md +25 -0
- package/prompts/revise.md +15 -0
- package/prompts/source_grounding.md +66 -0
- package/prompts/system_prompts.py +262 -0
- package/prompts/universal_system_prompt +7 -0
- package/prompts/untrusted_content +1 -0
- package/providers.json.example +92 -0
- package/pyproject.toml +100 -0
- package/requirements.txt +72 -0
- package/safety/__init__.py +0 -0
- package/safety/policy_enforcement.py +682 -0
- package/security/__init__.py +0 -0
- package/security/capability.py +177 -0
- package/security/permissions.py +190 -0
- package/security/posture.py +302 -0
- package/security/protected_paths.py +309 -0
- package/security/sandbox.py +985 -0
- package/skills/builtin/crypto/cryptography-verification.md +66 -0
- package/skills/builtin/math/formal-specification.md +67 -0
- package/skills/builtin/math/numerical-methods.md +66 -0
- package/skills/builtin/math/proof-writing.md +61 -0
- package/skills/builtin/research/experiment-design.md +62 -0
- package/skills/builtin/research/literature-review.md +65 -0
- package/skills/builtin/research/reproducibility.md +55 -0
- package/skills/builtin/research/source-evaluation.md +70 -0
- package/skills/builtin/research/statistical-inference.md +63 -0
- package/skills/builtin/research/technical-writing.md +59 -0
- package/skills/builtin/security/cybersecurity-research.md +63 -0
- package/skills/builtin/software/code-review.md +61 -0
- package/skills/builtin/software/debugging.md +61 -0
- package/skills/builtin/software/test-design.md +67 -0
- package/skills/manager.py +138 -0
- package/skills/tui_commands.py +111 -0
- package/storage.py +1005 -0
- package/tools/base.py +305 -0
- package/tools/builtin/__init__.py +0 -0
- package/tools/builtin/_math_common.py +377 -0
- package/tools/builtin/_net_common.py +61 -0
- package/tools/builtin/arxiv.py +223 -0
- package/tools/builtin/ask_user.py +213 -0
- package/tools/builtin/discrete_math.py +134 -0
- package/tools/builtin/fetch_url.py +153 -0
- package/tools/builtin/file_ops.py +412 -0
- package/tools/builtin/geometry.py +148 -0
- package/tools/builtin/get_time.py +20 -0
- package/tools/builtin/linear_algebra.py +155 -0
- package/tools/builtin/load_skill.py +66 -0
- package/tools/builtin/logic.py +96 -0
- package/tools/builtin/pin.py +193 -0
- package/tools/builtin/probability_stats.py +228 -0
- package/tools/builtin/project_docs.py +405 -0
- package/tools/builtin/remember.py +158 -0
- package/tools/builtin/shell.py +461 -0
- package/tools/builtin/symbolic_math.py +119 -0
- package/tools/builtin/todo.py +147 -0
- package/tools/builtin/web_search.py +151 -0
- package/tools/context.py +136 -0
- package/tools/isolation.py +314 -0
- package/tools/registry.py +897 -0
- package/tui/app.py +3423 -0
- package/tui/app.tcss +458 -0
- package/tui/commands.py +61 -0
- package/tui/diffs.py +265 -0
- package/tui/preferences.py +182 -0
- package/tui/ravens.py +146 -0
- package/tui/screens.py +943 -0
- package/tui/themes.py +366 -0
- package/tui/widgets.py +1239 -0
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
"""
|
|
2
|
+
core/reasoning/scholar.py
|
|
3
|
+
|
|
4
|
+
What a research paper is worth, from OpenAlex.
|
|
5
|
+
|
|
6
|
+
ROADMAP_v2 §45 (SQ5/SQ9). A cited paper's authority is more than its
|
|
7
|
+
host: arxiv.org serves a landmark and a paper nobody ever read from the
|
|
8
|
+
same domain, and the domain class cannot tell them apart. This asks the
|
|
9
|
+
one question a domain cannot -- how does this paper stand among its own
|
|
10
|
+
peers -- and answers it from counts rather than from a model.
|
|
11
|
+
|
|
12
|
+
WHY COHORT COUNTING RATHER THAN OPENALEX'S OWN NORMALISATION. `fwci` and
|
|
13
|
+
`citation_normalized_percentile` are exactly the field-and-age-normalised
|
|
14
|
+
metrics this needs, and BOTH ARE NULL ON PREPRINTS -- verified against
|
|
15
|
+
the API on a paper with 3019 citations. arXiv is the only scholarly
|
|
16
|
+
source this harness currently reaches, so the ready-made answer does not
|
|
17
|
+
cover the case that matters. Two counting queries do: how many works
|
|
18
|
+
share this paper's field, type and publication window, and how many of
|
|
19
|
+
them have fewer citations than it. That ratio IS the field-and-age
|
|
20
|
+
normalisation, measured rather than modelled, and it self-calibrates
|
|
21
|
+
across fields because the comparison set is the same field.
|
|
22
|
+
|
|
23
|
+
WHY NO LIFECYCLE CURVE. Matching the cohort on a date window makes the
|
|
24
|
+
age normalisation exact rather than approximate, so a fitted accrual
|
|
25
|
+
curve would add nothing for a paper old enough to carry signal. Below
|
|
26
|
+
that, it would be fitting noise: measured while specifying this, 72% of
|
|
27
|
+
computer-science preprints published in 2024 have ZERO citations. A
|
|
28
|
+
three-week-old paper's zero says nothing at all, so the term is dropped
|
|
29
|
+
and the remaining weights renormalise -- absence of evidence about a new
|
|
30
|
+
paper is not evidence of low impact.
|
|
31
|
+
|
|
32
|
+
THE EGRESS IS DELIBERATE, NAMED, AND OFF BY DEFAULT (SQ9). This is the
|
|
33
|
+
first network call in this harness made by something that is not a
|
|
34
|
+
registered tool, so it does not pass through the approval registry and
|
|
35
|
+
does not appear in granted_calls.json. That is stated rather than worked
|
|
36
|
+
around: routing deterministic scoring code through the tool layer would
|
|
37
|
+
make a pure function call a dispatcher. What it does instead is reuse
|
|
38
|
+
`is_url_permitted`, cache like the network tools do, trace every lookup,
|
|
39
|
+
and stay off until someone turns it on.
|
|
40
|
+
|
|
41
|
+
EVERY SCORING FUNCTION HERE IS PURE and takes a plain dict, so the whole
|
|
42
|
+
of it is testable against fixtures with no network -- which is not a
|
|
43
|
+
convenience, it is the suite's contract.
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
from __future__ import annotations
|
|
47
|
+
|
|
48
|
+
import logging
|
|
49
|
+
import math
|
|
50
|
+
import re
|
|
51
|
+
from datetime import date, timedelta
|
|
52
|
+
|
|
53
|
+
import config
|
|
54
|
+
from tools.builtin._net_common import TTLCache
|
|
55
|
+
|
|
56
|
+
logger = logging.getLogger(__name__)
|
|
57
|
+
|
|
58
|
+
_cache = TTLCache(config.SCHOLAR_CACHE_TTL_S)
|
|
59
|
+
|
|
60
|
+
# A DOI as it appears inside a URL. Deliberately not anchored: the point
|
|
61
|
+
# is to find one in `https://doi.org/10.1234/x` and in a publisher's own
|
|
62
|
+
# `.../article/10.1234/x/full`.
|
|
63
|
+
_DOI_RE = re.compile(r"10\.\d{4,9}/[^\s?#]+", re.IGNORECASE)
|
|
64
|
+
|
|
65
|
+
# Trailing punctuation a URL path collects that a DOI does not own.
|
|
66
|
+
_DOI_TRIM = ".,;:)]}>'\"/"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def doi_for(url: str) -> str | None:
|
|
70
|
+
"""The DOI this URL identifies, or None.
|
|
71
|
+
|
|
72
|
+
arXiv is handled through `source_corpus.arxiv_id_from_url` rather than
|
|
73
|
+
a second id parser here: two copies of "what counts as the same paper"
|
|
74
|
+
is how the corpus and this lookup come to disagree about one source in
|
|
75
|
+
one artifact.
|
|
76
|
+
"""
|
|
77
|
+
from core.reasoning.source_corpus import arxiv_id_from_url
|
|
78
|
+
|
|
79
|
+
arxiv_id = arxiv_id_from_url(url or "")
|
|
80
|
+
if arxiv_id:
|
|
81
|
+
# The DOI arXiv mints for every submission since 2022, and
|
|
82
|
+
# backfilled. OpenAlex indexes preprints under it.
|
|
83
|
+
return f"10.48550/arxiv.{arxiv_id}".lower()
|
|
84
|
+
|
|
85
|
+
match = _DOI_RE.search(url or "")
|
|
86
|
+
if not match:
|
|
87
|
+
return None
|
|
88
|
+
return match.group(0).rstrip(_DOI_TRIM).lower()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
# ---------------------------------------------------------------------------
|
|
92
|
+
# ---- Pure scoring ---------------------------------------------------------
|
|
93
|
+
# ---------------------------------------------------------------------------
|
|
94
|
+
|
|
95
|
+
def _location_type(work: dict) -> str:
|
|
96
|
+
location = work.get("primary_location") or {}
|
|
97
|
+
source = location.get("source") or {}
|
|
98
|
+
return (source.get("type") or "").lower()
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def is_peer_reviewed(work: dict) -> bool:
|
|
102
|
+
"""Whether this record is a published, peer-reviewed version.
|
|
103
|
+
|
|
104
|
+
THREE FIELDS, because one is not enough. `type == "preprint"` catches
|
|
105
|
+
the ordinary case; a repository-hosted record with
|
|
106
|
+
`version == "submittedVersion"` catches a work typed as an article
|
|
107
|
+
that is really an author manuscript; and `is_published` is the
|
|
108
|
+
publisher's own answer where there is one. A record has to survive all
|
|
109
|
+
three to count.
|
|
110
|
+
"""
|
|
111
|
+
location = work.get("primary_location") or {}
|
|
112
|
+
if (work.get("type") or "").lower() == "preprint":
|
|
113
|
+
return False
|
|
114
|
+
if _location_type(work) not in ("journal", "conference", "book series"):
|
|
115
|
+
return False
|
|
116
|
+
if location.get("version") == "submittedVersion":
|
|
117
|
+
return False
|
|
118
|
+
return location.get("is_published", True) is not False
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def venue_credit(work: dict) -> float:
|
|
122
|
+
"""What the venue alone says, before any citation is counted."""
|
|
123
|
+
if is_peer_reviewed(work):
|
|
124
|
+
return config.SCHOLAR_VENUE_CREDIT.get(
|
|
125
|
+
_location_type(work), config.SCHOLAR_VENUE_CREDIT["unknown"])
|
|
126
|
+
return config.SCHOLAR_VENUE_CREDIT["repository"]
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def author_standing(h_indexes, saturation: int | None = None):
|
|
130
|
+
"""max h-index across the authors, log-saturated onto 0-1, or None.
|
|
131
|
+
|
|
132
|
+
MAX rather than mean: a first-author graduate student on a paper with
|
|
133
|
+
a senior collaborator does not make the work less credible, and
|
|
134
|
+
averaging would say it does.
|
|
135
|
+
|
|
136
|
+
SATURATING rather than linear, because the difference between an
|
|
137
|
+
h-index of 60 and one of 90 says more about field size and career
|
|
138
|
+
length than about this paper. None when no author has a usable index
|
|
139
|
+
-- the term is then dropped and the rest renormalise, rather than
|
|
140
|
+
scoring an unindexed author as a bad one.
|
|
141
|
+
"""
|
|
142
|
+
saturation = saturation or config.SCHOLAR_H_SATURATION
|
|
143
|
+
usable = [h for h in (h_indexes or [])
|
|
144
|
+
if isinstance(h, int) and not isinstance(h, bool) and h >= 0]
|
|
145
|
+
if not usable:
|
|
146
|
+
return None
|
|
147
|
+
return min(1.0, math.log1p(max(usable)) / math.log1p(saturation))
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def paper_score(venue, citation, author, weights: dict | None = None) -> float:
|
|
151
|
+
"""Blend whichever signals exist, renormalising over them.
|
|
152
|
+
|
|
153
|
+
THE RENORMALISATION IS THE POINT. A three-week-old preprint has no
|
|
154
|
+
citation percentile and an unindexed author has no h-index; folding a
|
|
155
|
+
missing term in as zero would score "we could not measure this" and
|
|
156
|
+
"this measured badly" identically, which is the distinction §45 keeps
|
|
157
|
+
making. So a missing term takes its weight out of the denominator too.
|
|
158
|
+
"""
|
|
159
|
+
weights = weights or {}
|
|
160
|
+
terms = [
|
|
161
|
+
(venue, weights.get("venue_weight", config.SCHOLAR_VENUE_WEIGHT)),
|
|
162
|
+
(citation, weights.get("citation_weight", config.SCHOLAR_CITATION_WEIGHT)),
|
|
163
|
+
(author, weights.get("author_weight", config.SCHOLAR_AUTHOR_WEIGHT)),
|
|
164
|
+
]
|
|
165
|
+
present = [(value, weight) for value, weight in terms
|
|
166
|
+
if value is not None and weight > 0]
|
|
167
|
+
if not present:
|
|
168
|
+
return config.DEFAULT_DOMAIN_AUTHORITY
|
|
169
|
+
total = sum(weight for _, weight in present)
|
|
170
|
+
return min(1.0, max(0.0, sum(value * weight for value, weight in present) / total))
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _age_days(work: dict, today: date | None = None) -> int | None:
|
|
174
|
+
published = work.get("publication_date")
|
|
175
|
+
if not isinstance(published, str):
|
|
176
|
+
return None
|
|
177
|
+
try:
|
|
178
|
+
when = date.fromisoformat(published[:10])
|
|
179
|
+
except ValueError:
|
|
180
|
+
return None
|
|
181
|
+
return ((today or date.today()) - when).days
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
# ---------------------------------------------------------------------------
|
|
185
|
+
# ---- The client -----------------------------------------------------------
|
|
186
|
+
# ---------------------------------------------------------------------------
|
|
187
|
+
|
|
188
|
+
class ScholarLookup:
|
|
189
|
+
"""OpenAlex, cached and bounded, or a no-op when disabled.
|
|
190
|
+
|
|
191
|
+
A CLASS holding one run's cache and failure state, for
|
|
192
|
+
EmbeddingScorer's reason: one outage should stop the lookups rather
|
|
193
|
+
than be retried once per paper, and the run should say so once.
|
|
194
|
+
"""
|
|
195
|
+
|
|
196
|
+
def __init__(self, enabled: bool | None = None, fetch=None,
|
|
197
|
+
knobs: dict | None = None) -> None:
|
|
198
|
+
self.enabled = (config.SCHOLAR_LOOKUP if enabled is None else enabled)
|
|
199
|
+
self.knobs = knobs or {}
|
|
200
|
+
self.failed_reason = None
|
|
201
|
+
self.calls = 0
|
|
202
|
+
self._fetch = fetch or self._http_get
|
|
203
|
+
self._cohorts: dict = {}
|
|
204
|
+
|
|
205
|
+
@property
|
|
206
|
+
def usable(self) -> bool:
|
|
207
|
+
return self.enabled and self.failed_reason is None
|
|
208
|
+
|
|
209
|
+
# -- transport -------------------------------------------------------
|
|
210
|
+
|
|
211
|
+
def _http_get(self, path: str, params: dict):
|
|
212
|
+
"""The only place this module touches the network.
|
|
213
|
+
|
|
214
|
+
INJECTABLE, and every test injects: the suite runs offline, and a
|
|
215
|
+
scoring rule tested through a real socket is a scoring rule tested
|
|
216
|
+
on somebody's connection.
|
|
217
|
+
"""
|
|
218
|
+
import httpx
|
|
219
|
+
|
|
220
|
+
from safety.policy_enforcement import is_url_permitted
|
|
221
|
+
|
|
222
|
+
url = f"{config.SCHOLAR_API_URL.rstrip('/')}/{path.lstrip('/')}"
|
|
223
|
+
refusal = is_url_permitted(url, resolve=False)
|
|
224
|
+
if refusal:
|
|
225
|
+
raise RuntimeError(refusal)
|
|
226
|
+
if config.SCHOLAR_MAILTO:
|
|
227
|
+
# Only ever what the user configured. An address taken from a
|
|
228
|
+
# git config or an environment variable is not a rate-limit
|
|
229
|
+
# optimisation, it is an unasked-for disclosure.
|
|
230
|
+
params = {**params, "mailto": config.SCHOLAR_MAILTO}
|
|
231
|
+
response = httpx.get(
|
|
232
|
+
url, params=params, timeout=config.SCHOLAR_TIMEOUT_S,
|
|
233
|
+
headers={"User-Agent": "venastine-research-harness/1.0"},
|
|
234
|
+
follow_redirects=True)
|
|
235
|
+
response.raise_for_status()
|
|
236
|
+
return response.json()
|
|
237
|
+
|
|
238
|
+
def _get(self, path: str, params: dict):
|
|
239
|
+
"""Cached, retried, and terminal on exhaustion. None on failure."""
|
|
240
|
+
if not self.usable:
|
|
241
|
+
return None
|
|
242
|
+
key = f"{path}|{sorted(params.items())}"
|
|
243
|
+
hit = _cache.get(key)
|
|
244
|
+
if hit is not None:
|
|
245
|
+
return hit
|
|
246
|
+
|
|
247
|
+
last = None
|
|
248
|
+
for _ in range(config.SCHOLAR_MAX_RETRIES + 1):
|
|
249
|
+
try:
|
|
250
|
+
payload = self._fetch(path, params)
|
|
251
|
+
except Exception as exc: # noqa: BLE001 - degraded, never fatal
|
|
252
|
+
last = exc
|
|
253
|
+
continue
|
|
254
|
+
self.calls += 1
|
|
255
|
+
_cache.set(key, payload)
|
|
256
|
+
return payload
|
|
257
|
+
# A 404 is not an outage -- plenty of cited URLs are not indexed
|
|
258
|
+
# works -- so only a repeated failure disables the lookup, and the
|
|
259
|
+
# caller distinguishes "no record" from "could not ask".
|
|
260
|
+
self.failed_reason = f"{type(last).__name__}: {last}"
|
|
261
|
+
return None
|
|
262
|
+
|
|
263
|
+
# -- lookups ---------------------------------------------------------
|
|
264
|
+
|
|
265
|
+
def work(self, url: str):
|
|
266
|
+
"""The OpenAlex record for `url`, or None if it is not a paper."""
|
|
267
|
+
doi = doi_for(url)
|
|
268
|
+
if not doi or not self.usable:
|
|
269
|
+
return None
|
|
270
|
+
payload = self._get(f"works/doi:{doi}", {"select": _WORK_SELECT})
|
|
271
|
+
if not isinstance(payload, dict) or not payload.get("id"):
|
|
272
|
+
return None
|
|
273
|
+
return payload
|
|
274
|
+
|
|
275
|
+
def _count(self, filters: str) -> int | None:
|
|
276
|
+
payload = self._get("works", {"filter": filters, "per_page": 1,
|
|
277
|
+
"select": "id"})
|
|
278
|
+
if not isinstance(payload, dict):
|
|
279
|
+
return None
|
|
280
|
+
count = (payload.get("meta") or {}).get("count")
|
|
281
|
+
return count if isinstance(count, int) else None
|
|
282
|
+
|
|
283
|
+
def citation_percentile(self, work: dict, today: date | None = None):
|
|
284
|
+
"""Where this paper sits among works like it, or None with a reason.
|
|
285
|
+
|
|
286
|
+
Returns (percentile, evidence) where evidence is one of
|
|
287
|
+
"cohort", "insufficient_age" or "insufficient_cohort" -- the
|
|
288
|
+
second and third are answers, not failures, and the caller drops
|
|
289
|
+
the term rather than scoring it low.
|
|
290
|
+
"""
|
|
291
|
+
age = _age_days(work, today)
|
|
292
|
+
min_age = self.knobs.get("min_citation_age_days",
|
|
293
|
+
config.SCHOLAR_MIN_CITATION_AGE_DAYS)
|
|
294
|
+
if age is None or age < min_age:
|
|
295
|
+
return None, "insufficient_age"
|
|
296
|
+
|
|
297
|
+
cites = work.get("cited_by_count")
|
|
298
|
+
if not isinstance(cites, int) or isinstance(cites, bool):
|
|
299
|
+
return None, "insufficient_cohort"
|
|
300
|
+
|
|
301
|
+
field = ((work.get("primary_topic") or {}).get("field") or {}).get("id")
|
|
302
|
+
domain = ((work.get("primary_topic") or {}).get("domain") or {}).get("id")
|
|
303
|
+
work_type = (work.get("type") or "").lower() or "article"
|
|
304
|
+
published = date.fromisoformat(work["publication_date"][:10])
|
|
305
|
+
|
|
306
|
+
minimum = self.knobs.get("min_cohort_size", config.SCHOLAR_MIN_COHORT_SIZE)
|
|
307
|
+
for subject in _cohort_subjects(field, domain):
|
|
308
|
+
for window in _cohort_windows(published, age):
|
|
309
|
+
base = _cohort_filter(subject, work_type, window)
|
|
310
|
+
total = self._count(base)
|
|
311
|
+
if total is None:
|
|
312
|
+
return None, "insufficient_cohort"
|
|
313
|
+
if total < minimum:
|
|
314
|
+
continue
|
|
315
|
+
below = self._count(f"{base},cited_by_count:<{cites}")
|
|
316
|
+
if below is None:
|
|
317
|
+
return None, "insufficient_cohort"
|
|
318
|
+
return min(1.0, max(0.0, below / total)), "cohort"
|
|
319
|
+
return None, "insufficient_cohort"
|
|
320
|
+
|
|
321
|
+
def author_standing(self, work: dict):
|
|
322
|
+
"""The max h-index across this paper's authors, 0-1, or None.
|
|
323
|
+
|
|
324
|
+
ONE BATCHED REQUEST per paper rather than one per author: a paper
|
|
325
|
+
with nine authors is nine lookups the naive way, and the whole
|
|
326
|
+
reason cohort counts are cached is that this stage runs inside a
|
|
327
|
+
pass someone is waiting on.
|
|
328
|
+
"""
|
|
329
|
+
ids = []
|
|
330
|
+
for authorship in (work.get("authorships") or [])[:_MAX_AUTHORS]:
|
|
331
|
+
author = (authorship or {}).get("author") or {}
|
|
332
|
+
identifier = (author.get("id") or "").rsplit("/", 1)[-1]
|
|
333
|
+
if identifier.startswith("A"):
|
|
334
|
+
ids.append(identifier)
|
|
335
|
+
if not ids:
|
|
336
|
+
return None
|
|
337
|
+
payload = self._get("authors", {
|
|
338
|
+
"filter": "openalex_id:" + "|".join(ids),
|
|
339
|
+
"select": "id,summary_stats", "per-page": len(ids)})
|
|
340
|
+
if not isinstance(payload, dict):
|
|
341
|
+
return None
|
|
342
|
+
indexes = [((entry or {}).get("summary_stats") or {}).get("h_index")
|
|
343
|
+
for entry in payload.get("results") or []]
|
|
344
|
+
return author_standing(
|
|
345
|
+
indexes, self.knobs.get("h_saturation", config.SCHOLAR_H_SATURATION))
|
|
346
|
+
|
|
347
|
+
# -- the whole answer -------------------------------------------------
|
|
348
|
+
|
|
349
|
+
def authority_for(self, url: str, today: date | None = None):
|
|
350
|
+
"""(authority, signals) for a cited paper, or (None, None).
|
|
351
|
+
|
|
352
|
+
None means "this is not a paper, or nothing could be looked up" --
|
|
353
|
+
the caller keeps the domain class, which is the right answer for
|
|
354
|
+
both.
|
|
355
|
+
"""
|
|
356
|
+
work = self.work(url)
|
|
357
|
+
if work is None:
|
|
358
|
+
return None, None
|
|
359
|
+
|
|
360
|
+
if work.get("is_retracted"):
|
|
361
|
+
# Overrides everything. A retracted paper is not evidence
|
|
362
|
+
# whatever its venue or citation count, and a heavily cited
|
|
363
|
+
# retraction is the most dangerous shape there is.
|
|
364
|
+
return config.SCHOLAR_RETRACTED_AUTHORITY, {
|
|
365
|
+
"openalex_id": work.get("id"), "is_retracted": True,
|
|
366
|
+
"citation_evidence": "retracted"}
|
|
367
|
+
|
|
368
|
+
venue = venue_credit(work)
|
|
369
|
+
percentile, evidence = self.citation_percentile(work, today)
|
|
370
|
+
author = self.author_standing(work)
|
|
371
|
+
score = paper_score(venue, percentile, author, self.knobs)
|
|
372
|
+
|
|
373
|
+
return score, {
|
|
374
|
+
"openalex_id": work.get("id"),
|
|
375
|
+
"peer_reviewed": is_peer_reviewed(work),
|
|
376
|
+
"venue_type": _location_type(work) or None,
|
|
377
|
+
"venue_credit": round(venue, 4),
|
|
378
|
+
"cited_by_count": work.get("cited_by_count"),
|
|
379
|
+
"citation_percentile": (None if percentile is None
|
|
380
|
+
else round(percentile, 4)),
|
|
381
|
+
"citation_evidence": evidence,
|
|
382
|
+
"author_standing": None if author is None else round(author, 4),
|
|
383
|
+
"is_retracted": False,
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
_WORK_SELECT = ("id,doi,type,publication_date,cited_by_count,is_retracted,"
|
|
388
|
+
"primary_location,primary_topic,authorships")
|
|
389
|
+
|
|
390
|
+
# Enough to find the strongest name on a paper without turning a
|
|
391
|
+
# hyperauthorship record into a 3000-id query string.
|
|
392
|
+
_MAX_AUTHORS = 25
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _cohort_subjects(field, domain):
|
|
396
|
+
"""Field first, then domain, then everything of this type and age.
|
|
397
|
+
|
|
398
|
+
WIDENING RATHER THAN GIVING UP, because a narrow field in a narrow
|
|
399
|
+
window can genuinely hold fewer works than a percentile needs -- and a
|
|
400
|
+
percentile against a broader cohort is a weaker statement, not a wrong
|
|
401
|
+
one. The order is most-specific-first so the strongest available
|
|
402
|
+
comparison is the one used.
|
|
403
|
+
"""
|
|
404
|
+
seen = []
|
|
405
|
+
for subject in (field, domain, None):
|
|
406
|
+
if subject not in seen:
|
|
407
|
+
seen.append(subject)
|
|
408
|
+
return seen
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
def _cohort_windows(published: date, age_days: int):
|
|
412
|
+
"""The date windows to try, narrowest first.
|
|
413
|
+
|
|
414
|
+
A +/-30 day window for a paper under six months old, because within a
|
|
415
|
+
single calendar year a January paper has eleven months more to
|
|
416
|
+
accumulate citations than a December one -- and at that age eleven
|
|
417
|
+
months is the whole signal. Older than that, the calendar year is both
|
|
418
|
+
sufficient and larger.
|
|
419
|
+
"""
|
|
420
|
+
if age_days < 183:
|
|
421
|
+
yield (published - timedelta(days=30), published + timedelta(days=30))
|
|
422
|
+
yield (date(published.year, 1, 1), date(published.year, 12, 31))
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def _cohort_filter(subject, work_type: str, window) -> str:
|
|
426
|
+
start, end = window
|
|
427
|
+
parts = [f"type:{work_type}",
|
|
428
|
+
f"from_publication_date:{start.isoformat()}",
|
|
429
|
+
f"to_publication_date:{end.isoformat()}"]
|
|
430
|
+
if subject:
|
|
431
|
+
key = ("primary_topic.domain.id" if "/domains/" in subject
|
|
432
|
+
else "primary_topic.field.id")
|
|
433
|
+
parts.insert(0, f"{key}:{subject}")
|
|
434
|
+
return ",".join(parts)
|