@yahyahammad/venastine 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/.env.example +14 -0
  2. package/LICENSE +201 -0
  3. package/NOTICE +14 -0
  4. package/PRIVACY.md +66 -0
  5. package/README.md +878 -0
  6. package/SECURITY.md +88 -0
  7. package/THIRD_PARTY_NOTICES.md +100 -0
  8. package/agents/builtin/compactor.md +75 -0
  9. package/agents/builtin/explore.md +77 -0
  10. package/agents/builtin/grill-me.md +27 -0
  11. package/agents/builtin/initializer.md +94 -0
  12. package/agents/builtin/pipeline-reviewer.md +100 -0
  13. package/agents/builtin/plan.md +88 -0
  14. package/agents/builtin/review.md +74 -0
  15. package/agents/manager.py +195 -0
  16. package/agents/subagent_tool.py +215 -0
  17. package/agents/tui_commands.py +136 -0
  18. package/bin/venastine.mjs +382 -0
  19. package/config.py +1198 -0
  20. package/core/__init__.py +0 -0
  21. package/core/approval.py +115 -0
  22. package/core/client.py +1548 -0
  23. package/core/compaction.py +959 -0
  24. package/core/config_loader.py +1563 -0
  25. package/core/events.py +61 -0
  26. package/core/interaction.py +328 -0
  27. package/core/loop.py +1353 -0
  28. package/core/memory.py +370 -0
  29. package/core/model_windows.py +175 -0
  30. package/core/pipeline_models.py +176 -0
  31. package/core/reasoning/__init__.py +0 -0
  32. package/core/reasoning/authorization.py +130 -0
  33. package/core/reasoning/base.py +219 -0
  34. package/core/reasoning/confidence_scoring.py +468 -0
  35. package/core/reasoning/events.py +149 -0
  36. package/core/reasoning/json_retry.py +219 -0
  37. package/core/reasoning/orchestrator.py +1382 -0
  38. package/core/reasoning/output_writer.py +286 -0
  39. package/core/reasoning/payload_validation.py +398 -0
  40. package/core/reasoning/pipeline_storage.py +361 -0
  41. package/core/reasoning/review.py +615 -0
  42. package/core/reasoning/scholar.py +434 -0
  43. package/core/reasoning/source_corpus.py +327 -0
  44. package/core/reasoning/source_scoring.py +855 -0
  45. package/core/replay.py +167 -0
  46. package/core/session.py +153 -0
  47. package/core/workspace_trust.py +207 -0
  48. package/credentials.py +180 -0
  49. package/database.py +252 -0
  50. package/env_secrets.py +77 -0
  51. package/json_store.py +123 -0
  52. package/logging_setup.py +185 -0
  53. package/main.py +1959 -0
  54. package/mcp_client/__init__.py +15 -0
  55. package/mcp_client/client.py +499 -0
  56. package/mcp_client/config.py +379 -0
  57. package/mcp_client/registration.py +182 -0
  58. package/memories/manager.py +132 -0
  59. package/memories/tui_commands.py +232 -0
  60. package/package.json +73 -0
  61. package/project_init/config_files.py +278 -0
  62. package/project_init/doc_sets.py +330 -0
  63. package/project_init/generator.py +564 -0
  64. package/project_init/manifest.py +406 -0
  65. package/project_init/tui_commands.py +144 -0
  66. package/prompts/__init__.py +0 -0
  67. package/prompts/annotate.md +14 -0
  68. package/prompts/assumption_audit.md +27 -0
  69. package/prompts/claim_extraction.md +34 -0
  70. package/prompts/completeness.md +18 -0
  71. package/prompts/confidence_tiers.md +23 -0
  72. package/prompts/critic_pass.md +24 -0
  73. package/prompts/final_synthesis.md +13 -0
  74. package/prompts/initial_generation.md +14 -0
  75. package/prompts/preliminary_plan.md +23 -0
  76. package/prompts/revalidate.md +25 -0
  77. package/prompts/revise.md +15 -0
  78. package/prompts/source_grounding.md +66 -0
  79. package/prompts/system_prompts.py +262 -0
  80. package/prompts/universal_system_prompt +7 -0
  81. package/prompts/untrusted_content +1 -0
  82. package/providers.json.example +92 -0
  83. package/pyproject.toml +100 -0
  84. package/requirements.txt +72 -0
  85. package/safety/__init__.py +0 -0
  86. package/safety/policy_enforcement.py +682 -0
  87. package/security/__init__.py +0 -0
  88. package/security/capability.py +177 -0
  89. package/security/permissions.py +190 -0
  90. package/security/posture.py +302 -0
  91. package/security/protected_paths.py +309 -0
  92. package/security/sandbox.py +985 -0
  93. package/skills/builtin/crypto/cryptography-verification.md +66 -0
  94. package/skills/builtin/math/formal-specification.md +67 -0
  95. package/skills/builtin/math/numerical-methods.md +66 -0
  96. package/skills/builtin/math/proof-writing.md +61 -0
  97. package/skills/builtin/research/experiment-design.md +62 -0
  98. package/skills/builtin/research/literature-review.md +65 -0
  99. package/skills/builtin/research/reproducibility.md +55 -0
  100. package/skills/builtin/research/source-evaluation.md +70 -0
  101. package/skills/builtin/research/statistical-inference.md +63 -0
  102. package/skills/builtin/research/technical-writing.md +59 -0
  103. package/skills/builtin/security/cybersecurity-research.md +63 -0
  104. package/skills/builtin/software/code-review.md +61 -0
  105. package/skills/builtin/software/debugging.md +61 -0
  106. package/skills/builtin/software/test-design.md +67 -0
  107. package/skills/manager.py +138 -0
  108. package/skills/tui_commands.py +111 -0
  109. package/storage.py +1005 -0
  110. package/tools/base.py +305 -0
  111. package/tools/builtin/__init__.py +0 -0
  112. package/tools/builtin/_math_common.py +377 -0
  113. package/tools/builtin/_net_common.py +61 -0
  114. package/tools/builtin/arxiv.py +223 -0
  115. package/tools/builtin/ask_user.py +213 -0
  116. package/tools/builtin/discrete_math.py +134 -0
  117. package/tools/builtin/fetch_url.py +153 -0
  118. package/tools/builtin/file_ops.py +412 -0
  119. package/tools/builtin/geometry.py +148 -0
  120. package/tools/builtin/get_time.py +20 -0
  121. package/tools/builtin/linear_algebra.py +155 -0
  122. package/tools/builtin/load_skill.py +66 -0
  123. package/tools/builtin/logic.py +96 -0
  124. package/tools/builtin/pin.py +193 -0
  125. package/tools/builtin/probability_stats.py +228 -0
  126. package/tools/builtin/project_docs.py +405 -0
  127. package/tools/builtin/remember.py +158 -0
  128. package/tools/builtin/shell.py +461 -0
  129. package/tools/builtin/symbolic_math.py +119 -0
  130. package/tools/builtin/todo.py +147 -0
  131. package/tools/builtin/web_search.py +151 -0
  132. package/tools/context.py +136 -0
  133. package/tools/isolation.py +314 -0
  134. package/tools/registry.py +897 -0
  135. package/tui/app.py +3423 -0
  136. package/tui/app.tcss +458 -0
  137. package/tui/commands.py +61 -0
  138. package/tui/diffs.py +265 -0
  139. package/tui/preferences.py +182 -0
  140. package/tui/ravens.py +146 -0
  141. package/tui/screens.py +943 -0
  142. package/tui/themes.py +366 -0
  143. package/tui/widgets.py +1239 -0
@@ -0,0 +1,434 @@
1
+ """
2
+ core/reasoning/scholar.py
3
+
4
+ What a research paper is worth, from OpenAlex.
5
+
6
+ ROADMAP_v2 §45 (SQ5/SQ9). A cited paper's authority is more than its
7
+ host: arxiv.org serves a landmark and a paper nobody ever read from the
8
+ same domain, and the domain class cannot tell them apart. This asks the
9
+ one question a domain cannot -- how does this paper stand among its own
10
+ peers -- and answers it from counts rather than from a model.
11
+
12
+ WHY COHORT COUNTING RATHER THAN OPENALEX'S OWN NORMALISATION. `fwci` and
13
+ `citation_normalized_percentile` are exactly the field-and-age-normalised
14
+ metrics this needs, and BOTH ARE NULL ON PREPRINTS -- verified against
15
+ the API on a paper with 3019 citations. arXiv is the only scholarly
16
+ source this harness currently reaches, so the ready-made answer does not
17
+ cover the case that matters. Two counting queries do: how many works
18
+ share this paper's field, type and publication window, and how many of
19
+ them have fewer citations than it. That ratio IS the field-and-age
20
+ normalisation, measured rather than modelled, and it self-calibrates
21
+ across fields because the comparison set is the same field.
22
+
23
+ WHY NO LIFECYCLE CURVE. Matching the cohort on a date window makes the
24
+ age normalisation exact rather than approximate, so a fitted accrual
25
+ curve would add nothing for a paper old enough to carry signal. Below
26
+ that, it would be fitting noise: measured while specifying this, 72% of
27
+ computer-science preprints published in 2024 have ZERO citations. A
28
+ three-week-old paper's zero says nothing at all, so the term is dropped
29
+ and the remaining weights renormalise -- absence of evidence about a new
30
+ paper is not evidence of low impact.
31
+
32
+ THE EGRESS IS DELIBERATE, NAMED, AND OFF BY DEFAULT (SQ9). This is the
33
+ first network call in this harness made by something that is not a
34
+ registered tool, so it does not pass through the approval registry and
35
+ does not appear in granted_calls.json. That is stated rather than worked
36
+ around: routing deterministic scoring code through the tool layer would
37
+ make a pure function call a dispatcher. What it does instead is reuse
38
+ `is_url_permitted`, cache like the network tools do, trace every lookup,
39
+ and stay off until someone turns it on.
40
+
41
+ EVERY SCORING FUNCTION HERE IS PURE and takes a plain dict, so the whole
42
+ of it is testable against fixtures with no network -- which is not a
43
+ convenience, it is the suite's contract.
44
+ """
45
+
46
+ from __future__ import annotations
47
+
48
+ import logging
49
+ import math
50
+ import re
51
+ from datetime import date, timedelta
52
+
53
+ import config
54
+ from tools.builtin._net_common import TTLCache
55
+
56
+ logger = logging.getLogger(__name__)
57
+
58
+ _cache = TTLCache(config.SCHOLAR_CACHE_TTL_S)
59
+
60
+ # A DOI as it appears inside a URL. Deliberately not anchored: the point
61
+ # is to find one in `https://doi.org/10.1234/x` and in a publisher's own
62
+ # `.../article/10.1234/x/full`.
63
+ _DOI_RE = re.compile(r"10\.\d{4,9}/[^\s?#]+", re.IGNORECASE)
64
+
65
+ # Trailing punctuation a URL path collects that a DOI does not own.
66
+ _DOI_TRIM = ".,;:)]}>'\"/"
67
+
68
+
69
+ def doi_for(url: str) -> str | None:
70
+ """The DOI this URL identifies, or None.
71
+
72
+ arXiv is handled through `source_corpus.arxiv_id_from_url` rather than
73
+ a second id parser here: two copies of "what counts as the same paper"
74
+ is how the corpus and this lookup come to disagree about one source in
75
+ one artifact.
76
+ """
77
+ from core.reasoning.source_corpus import arxiv_id_from_url
78
+
79
+ arxiv_id = arxiv_id_from_url(url or "")
80
+ if arxiv_id:
81
+ # The DOI arXiv mints for every submission since 2022, and
82
+ # backfilled. OpenAlex indexes preprints under it.
83
+ return f"10.48550/arxiv.{arxiv_id}".lower()
84
+
85
+ match = _DOI_RE.search(url or "")
86
+ if not match:
87
+ return None
88
+ return match.group(0).rstrip(_DOI_TRIM).lower()
89
+
90
+
91
+ # ---------------------------------------------------------------------------
92
+ # ---- Pure scoring ---------------------------------------------------------
93
+ # ---------------------------------------------------------------------------
94
+
95
+ def _location_type(work: dict) -> str:
96
+ location = work.get("primary_location") or {}
97
+ source = location.get("source") or {}
98
+ return (source.get("type") or "").lower()
99
+
100
+
101
+ def is_peer_reviewed(work: dict) -> bool:
102
+ """Whether this record is a published, peer-reviewed version.
103
+
104
+ THREE FIELDS, because one is not enough. `type == "preprint"` catches
105
+ the ordinary case; a repository-hosted record with
106
+ `version == "submittedVersion"` catches a work typed as an article
107
+ that is really an author manuscript; and `is_published` is the
108
+ publisher's own answer where there is one. A record has to survive all
109
+ three to count.
110
+ """
111
+ location = work.get("primary_location") or {}
112
+ if (work.get("type") or "").lower() == "preprint":
113
+ return False
114
+ if _location_type(work) not in ("journal", "conference", "book series"):
115
+ return False
116
+ if location.get("version") == "submittedVersion":
117
+ return False
118
+ return location.get("is_published", True) is not False
119
+
120
+
121
+ def venue_credit(work: dict) -> float:
122
+ """What the venue alone says, before any citation is counted."""
123
+ if is_peer_reviewed(work):
124
+ return config.SCHOLAR_VENUE_CREDIT.get(
125
+ _location_type(work), config.SCHOLAR_VENUE_CREDIT["unknown"])
126
+ return config.SCHOLAR_VENUE_CREDIT["repository"]
127
+
128
+
129
+ def author_standing(h_indexes, saturation: int | None = None):
130
+ """max h-index across the authors, log-saturated onto 0-1, or None.
131
+
132
+ MAX rather than mean: a first-author graduate student on a paper with
133
+ a senior collaborator does not make the work less credible, and
134
+ averaging would say it does.
135
+
136
+ SATURATING rather than linear, because the difference between an
137
+ h-index of 60 and one of 90 says more about field size and career
138
+ length than about this paper. None when no author has a usable index
139
+ -- the term is then dropped and the rest renormalise, rather than
140
+ scoring an unindexed author as a bad one.
141
+ """
142
+ saturation = saturation or config.SCHOLAR_H_SATURATION
143
+ usable = [h for h in (h_indexes or [])
144
+ if isinstance(h, int) and not isinstance(h, bool) and h >= 0]
145
+ if not usable:
146
+ return None
147
+ return min(1.0, math.log1p(max(usable)) / math.log1p(saturation))
148
+
149
+
150
+ def paper_score(venue, citation, author, weights: dict | None = None) -> float:
151
+ """Blend whichever signals exist, renormalising over them.
152
+
153
+ THE RENORMALISATION IS THE POINT. A three-week-old preprint has no
154
+ citation percentile and an unindexed author has no h-index; folding a
155
+ missing term in as zero would score "we could not measure this" and
156
+ "this measured badly" identically, which is the distinction §45 keeps
157
+ making. So a missing term takes its weight out of the denominator too.
158
+ """
159
+ weights = weights or {}
160
+ terms = [
161
+ (venue, weights.get("venue_weight", config.SCHOLAR_VENUE_WEIGHT)),
162
+ (citation, weights.get("citation_weight", config.SCHOLAR_CITATION_WEIGHT)),
163
+ (author, weights.get("author_weight", config.SCHOLAR_AUTHOR_WEIGHT)),
164
+ ]
165
+ present = [(value, weight) for value, weight in terms
166
+ if value is not None and weight > 0]
167
+ if not present:
168
+ return config.DEFAULT_DOMAIN_AUTHORITY
169
+ total = sum(weight for _, weight in present)
170
+ return min(1.0, max(0.0, sum(value * weight for value, weight in present) / total))
171
+
172
+
173
+ def _age_days(work: dict, today: date | None = None) -> int | None:
174
+ published = work.get("publication_date")
175
+ if not isinstance(published, str):
176
+ return None
177
+ try:
178
+ when = date.fromisoformat(published[:10])
179
+ except ValueError:
180
+ return None
181
+ return ((today or date.today()) - when).days
182
+
183
+
184
+ # ---------------------------------------------------------------------------
185
+ # ---- The client -----------------------------------------------------------
186
+ # ---------------------------------------------------------------------------
187
+
188
+ class ScholarLookup:
189
+ """OpenAlex, cached and bounded, or a no-op when disabled.
190
+
191
+ A CLASS holding one run's cache and failure state, for
192
+ EmbeddingScorer's reason: one outage should stop the lookups rather
193
+ than be retried once per paper, and the run should say so once.
194
+ """
195
+
196
+ def __init__(self, enabled: bool | None = None, fetch=None,
197
+ knobs: dict | None = None) -> None:
198
+ self.enabled = (config.SCHOLAR_LOOKUP if enabled is None else enabled)
199
+ self.knobs = knobs or {}
200
+ self.failed_reason = None
201
+ self.calls = 0
202
+ self._fetch = fetch or self._http_get
203
+ self._cohorts: dict = {}
204
+
205
+ @property
206
+ def usable(self) -> bool:
207
+ return self.enabled and self.failed_reason is None
208
+
209
+ # -- transport -------------------------------------------------------
210
+
211
+ def _http_get(self, path: str, params: dict):
212
+ """The only place this module touches the network.
213
+
214
+ INJECTABLE, and every test injects: the suite runs offline, and a
215
+ scoring rule tested through a real socket is a scoring rule tested
216
+ on somebody's connection.
217
+ """
218
+ import httpx
219
+
220
+ from safety.policy_enforcement import is_url_permitted
221
+
222
+ url = f"{config.SCHOLAR_API_URL.rstrip('/')}/{path.lstrip('/')}"
223
+ refusal = is_url_permitted(url, resolve=False)
224
+ if refusal:
225
+ raise RuntimeError(refusal)
226
+ if config.SCHOLAR_MAILTO:
227
+ # Only ever what the user configured. An address taken from a
228
+ # git config or an environment variable is not a rate-limit
229
+ # optimisation, it is an unasked-for disclosure.
230
+ params = {**params, "mailto": config.SCHOLAR_MAILTO}
231
+ response = httpx.get(
232
+ url, params=params, timeout=config.SCHOLAR_TIMEOUT_S,
233
+ headers={"User-Agent": "venastine-research-harness/1.0"},
234
+ follow_redirects=True)
235
+ response.raise_for_status()
236
+ return response.json()
237
+
238
+ def _get(self, path: str, params: dict):
239
+ """Cached, retried, and terminal on exhaustion. None on failure."""
240
+ if not self.usable:
241
+ return None
242
+ key = f"{path}|{sorted(params.items())}"
243
+ hit = _cache.get(key)
244
+ if hit is not None:
245
+ return hit
246
+
247
+ last = None
248
+ for _ in range(config.SCHOLAR_MAX_RETRIES + 1):
249
+ try:
250
+ payload = self._fetch(path, params)
251
+ except Exception as exc: # noqa: BLE001 - degraded, never fatal
252
+ last = exc
253
+ continue
254
+ self.calls += 1
255
+ _cache.set(key, payload)
256
+ return payload
257
+ # A 404 is not an outage -- plenty of cited URLs are not indexed
258
+ # works -- so only a repeated failure disables the lookup, and the
259
+ # caller distinguishes "no record" from "could not ask".
260
+ self.failed_reason = f"{type(last).__name__}: {last}"
261
+ return None
262
+
263
+ # -- lookups ---------------------------------------------------------
264
+
265
+ def work(self, url: str):
266
+ """The OpenAlex record for `url`, or None if it is not a paper."""
267
+ doi = doi_for(url)
268
+ if not doi or not self.usable:
269
+ return None
270
+ payload = self._get(f"works/doi:{doi}", {"select": _WORK_SELECT})
271
+ if not isinstance(payload, dict) or not payload.get("id"):
272
+ return None
273
+ return payload
274
+
275
+ def _count(self, filters: str) -> int | None:
276
+ payload = self._get("works", {"filter": filters, "per_page": 1,
277
+ "select": "id"})
278
+ if not isinstance(payload, dict):
279
+ return None
280
+ count = (payload.get("meta") or {}).get("count")
281
+ return count if isinstance(count, int) else None
282
+
283
+ def citation_percentile(self, work: dict, today: date | None = None):
284
+ """Where this paper sits among works like it, or None with a reason.
285
+
286
+ Returns (percentile, evidence) where evidence is one of
287
+ "cohort", "insufficient_age" or "insufficient_cohort" -- the
288
+ second and third are answers, not failures, and the caller drops
289
+ the term rather than scoring it low.
290
+ """
291
+ age = _age_days(work, today)
292
+ min_age = self.knobs.get("min_citation_age_days",
293
+ config.SCHOLAR_MIN_CITATION_AGE_DAYS)
294
+ if age is None or age < min_age:
295
+ return None, "insufficient_age"
296
+
297
+ cites = work.get("cited_by_count")
298
+ if not isinstance(cites, int) or isinstance(cites, bool):
299
+ return None, "insufficient_cohort"
300
+
301
+ field = ((work.get("primary_topic") or {}).get("field") or {}).get("id")
302
+ domain = ((work.get("primary_topic") or {}).get("domain") or {}).get("id")
303
+ work_type = (work.get("type") or "").lower() or "article"
304
+ published = date.fromisoformat(work["publication_date"][:10])
305
+
306
+ minimum = self.knobs.get("min_cohort_size", config.SCHOLAR_MIN_COHORT_SIZE)
307
+ for subject in _cohort_subjects(field, domain):
308
+ for window in _cohort_windows(published, age):
309
+ base = _cohort_filter(subject, work_type, window)
310
+ total = self._count(base)
311
+ if total is None:
312
+ return None, "insufficient_cohort"
313
+ if total < minimum:
314
+ continue
315
+ below = self._count(f"{base},cited_by_count:<{cites}")
316
+ if below is None:
317
+ return None, "insufficient_cohort"
318
+ return min(1.0, max(0.0, below / total)), "cohort"
319
+ return None, "insufficient_cohort"
320
+
321
+ def author_standing(self, work: dict):
322
+ """The max h-index across this paper's authors, 0-1, or None.
323
+
324
+ ONE BATCHED REQUEST per paper rather than one per author: a paper
325
+ with nine authors is nine lookups the naive way, and the whole
326
+ reason cohort counts are cached is that this stage runs inside a
327
+ pass someone is waiting on.
328
+ """
329
+ ids = []
330
+ for authorship in (work.get("authorships") or [])[:_MAX_AUTHORS]:
331
+ author = (authorship or {}).get("author") or {}
332
+ identifier = (author.get("id") or "").rsplit("/", 1)[-1]
333
+ if identifier.startswith("A"):
334
+ ids.append(identifier)
335
+ if not ids:
336
+ return None
337
+ payload = self._get("authors", {
338
+ "filter": "openalex_id:" + "|".join(ids),
339
+ "select": "id,summary_stats", "per-page": len(ids)})
340
+ if not isinstance(payload, dict):
341
+ return None
342
+ indexes = [((entry or {}).get("summary_stats") or {}).get("h_index")
343
+ for entry in payload.get("results") or []]
344
+ return author_standing(
345
+ indexes, self.knobs.get("h_saturation", config.SCHOLAR_H_SATURATION))
346
+
347
+ # -- the whole answer -------------------------------------------------
348
+
349
+ def authority_for(self, url: str, today: date | None = None):
350
+ """(authority, signals) for a cited paper, or (None, None).
351
+
352
+ None means "this is not a paper, or nothing could be looked up" --
353
+ the caller keeps the domain class, which is the right answer for
354
+ both.
355
+ """
356
+ work = self.work(url)
357
+ if work is None:
358
+ return None, None
359
+
360
+ if work.get("is_retracted"):
361
+ # Overrides everything. A retracted paper is not evidence
362
+ # whatever its venue or citation count, and a heavily cited
363
+ # retraction is the most dangerous shape there is.
364
+ return config.SCHOLAR_RETRACTED_AUTHORITY, {
365
+ "openalex_id": work.get("id"), "is_retracted": True,
366
+ "citation_evidence": "retracted"}
367
+
368
+ venue = venue_credit(work)
369
+ percentile, evidence = self.citation_percentile(work, today)
370
+ author = self.author_standing(work)
371
+ score = paper_score(venue, percentile, author, self.knobs)
372
+
373
+ return score, {
374
+ "openalex_id": work.get("id"),
375
+ "peer_reviewed": is_peer_reviewed(work),
376
+ "venue_type": _location_type(work) or None,
377
+ "venue_credit": round(venue, 4),
378
+ "cited_by_count": work.get("cited_by_count"),
379
+ "citation_percentile": (None if percentile is None
380
+ else round(percentile, 4)),
381
+ "citation_evidence": evidence,
382
+ "author_standing": None if author is None else round(author, 4),
383
+ "is_retracted": False,
384
+ }
385
+
386
+
387
+ _WORK_SELECT = ("id,doi,type,publication_date,cited_by_count,is_retracted,"
388
+ "primary_location,primary_topic,authorships")
389
+
390
+ # Enough to find the strongest name on a paper without turning a
391
+ # hyperauthorship record into a 3000-id query string.
392
+ _MAX_AUTHORS = 25
393
+
394
+
395
+ def _cohort_subjects(field, domain):
396
+ """Field first, then domain, then everything of this type and age.
397
+
398
+ WIDENING RATHER THAN GIVING UP, because a narrow field in a narrow
399
+ window can genuinely hold fewer works than a percentile needs -- and a
400
+ percentile against a broader cohort is a weaker statement, not a wrong
401
+ one. The order is most-specific-first so the strongest available
402
+ comparison is the one used.
403
+ """
404
+ seen = []
405
+ for subject in (field, domain, None):
406
+ if subject not in seen:
407
+ seen.append(subject)
408
+ return seen
409
+
410
+
411
+ def _cohort_windows(published: date, age_days: int):
412
+ """The date windows to try, narrowest first.
413
+
414
+ A +/-30 day window for a paper under six months old, because within a
415
+ single calendar year a January paper has eleven months more to
416
+ accumulate citations than a December one -- and at that age eleven
417
+ months is the whole signal. Older than that, the calendar year is both
418
+ sufficient and larger.
419
+ """
420
+ if age_days < 183:
421
+ yield (published - timedelta(days=30), published + timedelta(days=30))
422
+ yield (date(published.year, 1, 1), date(published.year, 12, 31))
423
+
424
+
425
+ def _cohort_filter(subject, work_type: str, window) -> str:
426
+ start, end = window
427
+ parts = [f"type:{work_type}",
428
+ f"from_publication_date:{start.isoformat()}",
429
+ f"to_publication_date:{end.isoformat()}"]
430
+ if subject:
431
+ key = ("primary_topic.domain.id" if "/domains/" in subject
432
+ else "primary_topic.field.id")
433
+ parts.insert(0, f"{key}:{subject}")
434
+ return ",".join(parts)