opencode-bioresearcher 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +103 -0
- package/agents/bioresearcher-dr-worker.md +54 -0
- package/connector-meta.json +23 -0
- package/index.js +77 -0
- package/loader.js +3 -0
- package/package.json +42 -0
- package/skill-bundle.json +12 -0
- package/skills/bioresearcher-deep-research/SKILL.md +330 -0
- package/skills/bioresearcher-deep-research/references/analysis-methods.md +90 -0
- package/skills/bioresearcher-deep-research/references/article-literature.md +89 -0
- package/skills/bioresearcher-deep-research/references/best-practices.md +102 -0
- package/skills/bioresearcher-deep-research/references/citations.md +146 -0
- package/skills/bioresearcher-deep-research/references/clinical-trials.md +87 -0
- package/skills/bioresearcher-deep-research/references/diseases.md +94 -0
- package/skills/bioresearcher-deep-research/references/drugs.md +88 -0
- package/skills/bioresearcher-deep-research/references/ensembl-pdb.md +134 -0
- package/skills/bioresearcher-deep-research/references/functional-genomics.md +118 -0
- package/skills/bioresearcher-deep-research/references/genes.md +93 -0
- package/skills/bioresearcher-deep-research/references/optional-analysis.md +108 -0
- package/skills/bioresearcher-deep-research/references/patents.md +92 -0
- package/skills/bioresearcher-deep-research/references/rate-limiting-auth.md +95 -0
- package/skills/bioresearcher-deep-research/references/report-template.md +117 -0
- package/skills/bioresearcher-deep-research/references/tool-selection.md +142 -0
- package/skills/bioresearcher-deep-research/references/utility-config.md +116 -0
- package/skills/bioresearcher-deep-research/references/variants.md +109 -0
- package/skills/bioresearcher-deep-research/references/worker-protocol.md +110 -0
- package/skills/bioresearcher-deep-research/scripts/markdown-to-html.py +86 -0
- package/skills/bioresearcher-plot-making/SKILL.md +97 -0
- package/skills/bioresearcher-plot-making/references/literature-search-method-summary.md +163 -0
- package/skills/bioresearcher-plot-making/references/qa-gates-and-gotchas.md +156 -0
- package/skills/bioresearcher-plot-making/references/structural-biology-binder-visualization.md +206 -0
- package/skills/bioresearcher-plot-making/scripts/audit_figure_collisions.py +742 -0
- package/skills/bioresearcher-plot-making/scripts/audit_panel_alignment.py +935 -0
- package/skills/bioresearcher-plot-making/scripts/audit_pdf_text.py +152 -0
- package/skills/bioresearcher-plot-making/scripts/plot_helpers.py +177 -0
- package/skills/bioresearcher-pubmed-weekly/SKILL.md +223 -0
- package/skills/bioresearcher-pubmed-weekly/scripts/parse_updatefiles.py +272 -0
- package/skills/bioresearcher-pubmed-weekly/scripts/pubmed_weekly.py +493 -0
- package/skills/bioresearcher-python-setup-uv/SKILL.md +184 -0
|
@@ -0,0 +1,493 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""PubMed weekly daily updates downloader and combiner.
|
|
3
|
+
|
|
4
|
+
Commands (stdlib only except openpyxl, pulled in via parse_updatefiles):
|
|
5
|
+
|
|
6
|
+
- ``calculate_week``: print the previous week's (Monday-Sunday) range as
|
|
7
|
+
``YYYYMMDD-YYYYMMDD``.
|
|
8
|
+
- ``fetch_files``: list all ``pubmedNNnNNNN.xml.gz`` updatefiles on the NCBI
|
|
9
|
+
FTP server (``ftp://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/``).
|
|
10
|
+
- ``filter_files <week> <file_list>``: keep only files whose FTP modification
|
|
11
|
+
time falls inside the week.
|
|
12
|
+
- ``download_file <week> <filename>`: download one file into
|
|
13
|
+
``.download/pubmed-daily/<week>/`` with retry (3 attempts, 2s delay) and
|
|
14
|
+
resume (``.part`` files; completed files are skipped).
|
|
15
|
+
- ``combine <week>``: parse every ``.xml.gz`` in the week directory via the
|
|
16
|
+
bundled ``parse_updatefiles.py`` streaming parser and write
|
|
17
|
+
``combined.xlsx`` plus ``summary.json`` in that directory.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import glob
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import sys
|
|
26
|
+
import time
|
|
27
|
+
import urllib.request
|
|
28
|
+
from datetime import datetime, timedelta
|
|
29
|
+
from typing import Any, Dict, List
|
|
30
|
+
|
|
31
|
+
FTP_BASE = "ftp://ftp.ncbi.nlm.nih.gov/pubmed/updatefiles/"
|
|
32
|
+
|
|
33
|
+
MONTH_MAP = {
|
|
34
|
+
"Jan": 1,
|
|
35
|
+
"Feb": 2,
|
|
36
|
+
"Mar": 3,
|
|
37
|
+
"Apr": 4,
|
|
38
|
+
"May": 5,
|
|
39
|
+
"Jun": 6,
|
|
40
|
+
"Jul": 7,
|
|
41
|
+
"Aug": 8,
|
|
42
|
+
"Sep": 9,
|
|
43
|
+
"Oct": 10,
|
|
44
|
+
"Nov": 11,
|
|
45
|
+
"Dec": 12,
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def calculate_week() -> str:
|
|
52
|
+
"""Calculate the past week's date range (Monday-Sunday).
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
Week folder name in format 'YYYYMMDD-YYYYMMDD' for the PREVIOUS week
|
|
56
|
+
"""
|
|
57
|
+
today = datetime.now()
|
|
58
|
+
|
|
59
|
+
# Find the most recent Monday of the current week
|
|
60
|
+
days_since_monday = today.weekday() # Monday = 0, Sunday = 6
|
|
61
|
+
current_monday = today - timedelta(days=days_since_monday)
|
|
62
|
+
|
|
63
|
+
# Go back one week to get the previous week's Monday
|
|
64
|
+
previous_week_monday = current_monday - timedelta(days=7)
|
|
65
|
+
|
|
66
|
+
# Calculate the previous week's Sunday (6 days after Monday)
|
|
67
|
+
previous_week_sunday = previous_week_monday + timedelta(days=6)
|
|
68
|
+
|
|
69
|
+
week_start = previous_week_monday.strftime("%Y%m%d")
|
|
70
|
+
week_end = previous_week_sunday.strftime("%Y%m%d")
|
|
71
|
+
|
|
72
|
+
return f"{week_start}-{week_end}"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def infer_year(month: int, day: int, hour: int, minute: int) -> int:
|
|
76
|
+
"""Infer year for MMM DD HH:MM format.
|
|
77
|
+
|
|
78
|
+
Uses current year if date is not in the future.
|
|
79
|
+
Uses previous year if inferred date is in the future.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
month: Month number (1-12)
|
|
83
|
+
day: Day of month
|
|
84
|
+
hour: Hour (0-23)
|
|
85
|
+
minute: Minute (0-59)
|
|
86
|
+
|
|
87
|
+
Returns:
|
|
88
|
+
Inferred year as integer
|
|
89
|
+
"""
|
|
90
|
+
now = datetime.now()
|
|
91
|
+
date_this_year = datetime(now.year, month, day, hour, minute)
|
|
92
|
+
|
|
93
|
+
if date_this_year > now:
|
|
94
|
+
return now.year - 1
|
|
95
|
+
return now.year
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def parse_ftp_listing_to_dict(content: str) -> Dict[str, datetime]:
|
|
99
|
+
"""Parse FTP directory listing into {filename: datetime} dict.
|
|
100
|
+
|
|
101
|
+
Supports multiple date formats with regex fallback chain:
|
|
102
|
+
1. Unix ls format - MMM DD HH:MM (current year)
|
|
103
|
+
2. Unix ls format - MMM DD YYYY (older files)
|
|
104
|
+
3. ISO 8601 format - YYYY-MM-DD HH:MM
|
|
105
|
+
4. European format - DD-MMM-YYYY HH:MM
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
content: Raw FTP directory listing content
|
|
109
|
+
|
|
110
|
+
Returns:
|
|
111
|
+
Dictionary mapping filename to datetime object
|
|
112
|
+
"""
|
|
113
|
+
file_dates = {}
|
|
114
|
+
|
|
115
|
+
for line in content.split("\n"):
|
|
116
|
+
line = line.strip()
|
|
117
|
+
if not line or line.startswith("total"):
|
|
118
|
+
continue
|
|
119
|
+
|
|
120
|
+
filename = None
|
|
121
|
+
file_date = None
|
|
122
|
+
|
|
123
|
+
match = re.match(
|
|
124
|
+
r"^\S+\s+\d+\s+\S+\s+\S+\s+\d+\s+(\w{3})\s+(\d{1,2})\s+(\d{2}:\d{2})\s+(.+)$",
|
|
125
|
+
line,
|
|
126
|
+
)
|
|
127
|
+
if match:
|
|
128
|
+
month_str, day_str, time_str, fn = match.groups()
|
|
129
|
+
month = MONTH_MAP.get(month_str)
|
|
130
|
+
if month:
|
|
131
|
+
day = int(day_str)
|
|
132
|
+
hour, minute = map(int, time_str.split(":"))
|
|
133
|
+
year = infer_year(month, day, hour, minute)
|
|
134
|
+
filename = fn
|
|
135
|
+
file_date = datetime(year, month, day, hour, minute)
|
|
136
|
+
|
|
137
|
+
if not file_date:
|
|
138
|
+
match = re.match(
|
|
139
|
+
r"^\S+\s+\d+\s+\S+\s+\S+\s+\d+\s+(\w{3})\s+(\d{1,2})\s+(\d{4})\s+(.+)$",
|
|
140
|
+
line,
|
|
141
|
+
)
|
|
142
|
+
if match:
|
|
143
|
+
month_str, day_str, year_str, fn = match.groups()
|
|
144
|
+
month = MONTH_MAP.get(month_str)
|
|
145
|
+
if month:
|
|
146
|
+
year = int(year_str)
|
|
147
|
+
day = int(day_str)
|
|
148
|
+
filename = fn
|
|
149
|
+
file_date = datetime(year, month, day)
|
|
150
|
+
|
|
151
|
+
if not file_date:
|
|
152
|
+
match = re.match(r"^(\d{4}-\d{2}-\d{2})\s+(\d{2}:\d{2})\s+(.+)$", line)
|
|
153
|
+
if match:
|
|
154
|
+
date_str, time_str, fn = match.groups()
|
|
155
|
+
datetime_str = f"{date_str} {time_str}"
|
|
156
|
+
try:
|
|
157
|
+
file_date = datetime.strptime(datetime_str, "%Y-%m-%d %H:%M")
|
|
158
|
+
filename = fn
|
|
159
|
+
except ValueError:
|
|
160
|
+
pass
|
|
161
|
+
|
|
162
|
+
if not file_date:
|
|
163
|
+
match = re.match(
|
|
164
|
+
r"^(\d{1,2})-(\w{3})-(\d{4})\s+(\d{2}:\d{2})\s+(.+)$", line
|
|
165
|
+
)
|
|
166
|
+
if match:
|
|
167
|
+
day_str, month_str, year_str, time_str, fn = match.groups()
|
|
168
|
+
month = MONTH_MAP.get(month_str)
|
|
169
|
+
if month:
|
|
170
|
+
day = int(day_str)
|
|
171
|
+
year = int(year_str)
|
|
172
|
+
hour, minute = map(int, time_str.split(":"))
|
|
173
|
+
try:
|
|
174
|
+
file_date = datetime(year, month, day, hour, minute)
|
|
175
|
+
filename = fn
|
|
176
|
+
except ValueError:
|
|
177
|
+
pass
|
|
178
|
+
|
|
179
|
+
if filename and file_date and filename not in file_dates:
|
|
180
|
+
file_dates[filename] = file_date
|
|
181
|
+
|
|
182
|
+
return file_dates
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def fetch_ftp_file_list() -> List[str]:
|
|
186
|
+
"""Fetch list of xml.gz files from NCBI FTP server.
|
|
187
|
+
|
|
188
|
+
Returns:
|
|
189
|
+
List of xml.gz filenames from the FTP server
|
|
190
|
+
"""
|
|
191
|
+
url = FTP_BASE
|
|
192
|
+
|
|
193
|
+
try:
|
|
194
|
+
with urllib.request.urlopen(url) as response:
|
|
195
|
+
html_content = response.read().decode("utf-8")
|
|
196
|
+
|
|
197
|
+
# Parse HTML to extract filenames
|
|
198
|
+
# FTP directory listing returns HTML with links
|
|
199
|
+
filenames = []
|
|
200
|
+
for line in html_content.split("\n"):
|
|
201
|
+
match = re.search(r"pubmed\d+n\d+\.xml\.gz", line)
|
|
202
|
+
if match:
|
|
203
|
+
filename = match.group(0)
|
|
204
|
+
if filename not in filenames:
|
|
205
|
+
filenames.append(filename)
|
|
206
|
+
|
|
207
|
+
return sorted(filenames)
|
|
208
|
+
|
|
209
|
+
except Exception as e:
|
|
210
|
+
print(f"Error fetching FTP file list: {e}", file=sys.stderr)
|
|
211
|
+
sys.exit(1)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def filter_files_by_date(week_name: str, file_list: List[str]) -> List[str]:
|
|
215
|
+
"""Filter files to include only those from the past week.
|
|
216
|
+
|
|
217
|
+
PubMed updatefile names do not encode a date, so the FTP directory
|
|
218
|
+
listing (with modification timestamps) is fetched and filtered by mtime.
|
|
219
|
+
|
|
220
|
+
Args:
|
|
221
|
+
week_name: Week folder name (YYYYMMDD-YYYYMMDD)
|
|
222
|
+
file_list: List of all xml.gz filenames
|
|
223
|
+
|
|
224
|
+
Returns:
|
|
225
|
+
List of filenames that fall within the date range
|
|
226
|
+
"""
|
|
227
|
+
# Parse week dates
|
|
228
|
+
start_date_str, end_date_str = week_name.split("-")
|
|
229
|
+
start_date = datetime.strptime(start_date_str, "%Y%m%d")
|
|
230
|
+
end_date = datetime.strptime(end_date_str, "%Y%m%d").replace(
|
|
231
|
+
hour=23, minute=59, second=59
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
# Fetch directory listing with timestamps
|
|
235
|
+
url = FTP_BASE
|
|
236
|
+
|
|
237
|
+
try:
|
|
238
|
+
with urllib.request.urlopen(url) as response:
|
|
239
|
+
content = response.read().decode("utf-8", errors="ignore")
|
|
240
|
+
|
|
241
|
+
file_dates = parse_ftp_listing_to_dict(content)
|
|
242
|
+
|
|
243
|
+
# Filter files within date range AND in provided file_list
|
|
244
|
+
filtered_files = [
|
|
245
|
+
f
|
|
246
|
+
for f in file_list
|
|
247
|
+
if f in file_dates and start_date <= file_dates[f] <= end_date
|
|
248
|
+
]
|
|
249
|
+
|
|
250
|
+
return sorted(filtered_files)
|
|
251
|
+
|
|
252
|
+
except Exception as e:
|
|
253
|
+
print(f"Error filtering files by date: {e}", file=sys.stderr)
|
|
254
|
+
sys.exit(1)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _download_with_resume(url: str, part_path: str) -> None:
|
|
258
|
+
"""Stream ``url`` into ``part_path``, resuming an existing partial file.
|
|
259
|
+
|
|
260
|
+
Skips as many bytes over the remote stream as already present locally,
|
|
261
|
+
then appends the remainder. If the remote stream is not longer than the
|
|
262
|
+
partial file, restarts from scratch.
|
|
263
|
+
"""
|
|
264
|
+
existing = os.path.getsize(part_path) if os.path.exists(part_path) else 0
|
|
265
|
+
|
|
266
|
+
with urllib.request.urlopen(url) as response:
|
|
267
|
+
to_skip = existing
|
|
268
|
+
exhausted = False
|
|
269
|
+
while to_skip > 0:
|
|
270
|
+
chunk = response.read(min(65536, to_skip))
|
|
271
|
+
if not chunk:
|
|
272
|
+
exhausted = True
|
|
273
|
+
break
|
|
274
|
+
to_skip -= len(chunk)
|
|
275
|
+
|
|
276
|
+
mode = "wb" if exhausted else "ab"
|
|
277
|
+
with open(part_path, mode) as out:
|
|
278
|
+
while True:
|
|
279
|
+
chunk = response.read(65536)
|
|
280
|
+
if not chunk:
|
|
281
|
+
break
|
|
282
|
+
out.write(chunk)
|
|
283
|
+
|
|
284
|
+
if os.path.getsize(part_path) == 0:
|
|
285
|
+
raise Exception("Downloaded file is empty")
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def download_file(week_name: str, filename: str, max_retries: int = 3) -> int:
|
|
289
|
+
"""Download a single file from NCBI FTP server with retry and resume.
|
|
290
|
+
|
|
291
|
+
Completed files (existing, non-empty) are skipped so re-running the
|
|
292
|
+
workflow resumes where it left off. In-progress downloads use a
|
|
293
|
+
``.part`` file renamed into place on success.
|
|
294
|
+
|
|
295
|
+
Args:
|
|
296
|
+
week_name: Week folder name
|
|
297
|
+
filename: XML.gz filename to download
|
|
298
|
+
max_retries: Maximum number of retry attempts
|
|
299
|
+
|
|
300
|
+
Returns:
|
|
301
|
+
0 on success, 1 on failure (after all retries)
|
|
302
|
+
"""
|
|
303
|
+
url = f"{FTP_BASE}{filename}"
|
|
304
|
+
|
|
305
|
+
# Create download directory in current working directory
|
|
306
|
+
base_dir = os.getcwd()
|
|
307
|
+
download_dir = os.path.join(base_dir, ".download", "pubmed-daily", week_name)
|
|
308
|
+
os.makedirs(download_dir, exist_ok=True)
|
|
309
|
+
|
|
310
|
+
filepath = os.path.join(download_dir, filename)
|
|
311
|
+
part_path = filepath + ".part"
|
|
312
|
+
|
|
313
|
+
if os.path.exists(filepath) and os.path.getsize(filepath) > 0:
|
|
314
|
+
print(f"Already downloaded: {filename} (skipped)")
|
|
315
|
+
return 0
|
|
316
|
+
|
|
317
|
+
for attempt in range(max_retries):
|
|
318
|
+
try:
|
|
319
|
+
print(f"Downloading {filename} (attempt {attempt + 1}/{max_retries})...")
|
|
320
|
+
|
|
321
|
+
_download_with_resume(url, part_path)
|
|
322
|
+
|
|
323
|
+
os.replace(part_path, filepath)
|
|
324
|
+
print(f"Successfully downloaded {filename}")
|
|
325
|
+
return 0
|
|
326
|
+
|
|
327
|
+
except Exception as e:
|
|
328
|
+
print(f"Error downloading {filename}: {e}", file=sys.stderr)
|
|
329
|
+
|
|
330
|
+
if attempt < max_retries - 1:
|
|
331
|
+
print("Retrying in 2 seconds...")
|
|
332
|
+
time.sleep(2)
|
|
333
|
+
else:
|
|
334
|
+
print(f"Failed to download {filename} after {max_retries} attempts")
|
|
335
|
+
return 1
|
|
336
|
+
|
|
337
|
+
return 1
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _run_parser(
|
|
341
|
+
file_paths: List[str], output: str, summary_json: str
|
|
342
|
+
) -> Dict[str, Any]:
|
|
343
|
+
"""Parse xml.gz files via the bundled parse_updatefiles module.
|
|
344
|
+
|
|
345
|
+
Imports ``parse_files`` from the sibling script when possible and falls
|
|
346
|
+
back to running it as a subprocess with the current interpreter.
|
|
347
|
+
"""
|
|
348
|
+
if SCRIPT_DIR not in sys.path:
|
|
349
|
+
sys.path.insert(0, SCRIPT_DIR)
|
|
350
|
+
try:
|
|
351
|
+
from parse_updatefiles import parse_files
|
|
352
|
+
except ImportError:
|
|
353
|
+
import subprocess
|
|
354
|
+
|
|
355
|
+
command = [
|
|
356
|
+
sys.executable,
|
|
357
|
+
os.path.join(SCRIPT_DIR, "parse_updatefiles.py"),
|
|
358
|
+
*file_paths,
|
|
359
|
+
"-o",
|
|
360
|
+
output,
|
|
361
|
+
"--summary-json",
|
|
362
|
+
summary_json,
|
|
363
|
+
]
|
|
364
|
+
subprocess.run(command, check=True)
|
|
365
|
+
with open(summary_json, "r", encoding="utf-8") as handle:
|
|
366
|
+
return json.load(handle)
|
|
367
|
+
|
|
368
|
+
return parse_files(file_paths, output, summary_json)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def combine(week_name: str) -> Dict[str, Any]:
|
|
372
|
+
"""Parse and combine all xml.gz files in the week folder.
|
|
373
|
+
|
|
374
|
+
Delegates parsing/combination to ``parse_updatefiles.py`` (streaming,
|
|
375
|
+
memory-bounded) and writes ``combined.xlsx`` plus ``summary.json`` into
|
|
376
|
+
the week directory.
|
|
377
|
+
|
|
378
|
+
Args:
|
|
379
|
+
week_name: Week folder name (e.g., '20250217-20250223')
|
|
380
|
+
|
|
381
|
+
Returns:
|
|
382
|
+
Dict with success, article_count, deleted_pmid_count, source_files,
|
|
383
|
+
output_file, summary_json
|
|
384
|
+
"""
|
|
385
|
+
week_dir = os.path.join(os.getcwd(), ".download", "pubmed-daily", week_name)
|
|
386
|
+
|
|
387
|
+
if not os.path.isdir(week_dir):
|
|
388
|
+
return {
|
|
389
|
+
"success": False,
|
|
390
|
+
"error": f"Directory not found: {week_dir}",
|
|
391
|
+
"article_count": 0,
|
|
392
|
+
"source_files": [],
|
|
393
|
+
"output_file": None,
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
xml_files = sorted(glob.glob(os.path.join(week_dir, "*.xml.gz")))
|
|
397
|
+
|
|
398
|
+
if not xml_files:
|
|
399
|
+
return {
|
|
400
|
+
"success": False,
|
|
401
|
+
"error": "No .xml.gz updatefiles found to combine",
|
|
402
|
+
"article_count": 0,
|
|
403
|
+
"source_files": [],
|
|
404
|
+
"output_file": None,
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
output_path = os.path.join(week_dir, "combined.xlsx")
|
|
408
|
+
summary_path = os.path.join(week_dir, "summary.json")
|
|
409
|
+
|
|
410
|
+
try:
|
|
411
|
+
parser_summary = _run_parser(xml_files, output_path, summary_path)
|
|
412
|
+
except Exception as e: # noqa: BLE001 - report failure as JSON
|
|
413
|
+
return {
|
|
414
|
+
"success": False,
|
|
415
|
+
"error": f"Parsing failed: {e}",
|
|
416
|
+
"article_count": 0,
|
|
417
|
+
"source_files": [os.path.basename(f) for f in xml_files],
|
|
418
|
+
"output_file": None,
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
return {
|
|
422
|
+
"success": True,
|
|
423
|
+
"article_count": parser_summary.get("article_count", 0),
|
|
424
|
+
"deleted_pmid_count": len(parser_summary.get("deleted_pmids", [])),
|
|
425
|
+
"source_files": [os.path.basename(f) for f in xml_files],
|
|
426
|
+
"output_file": os.path.basename(output_path),
|
|
427
|
+
"summary_json": os.path.basename(summary_path),
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def main():
|
|
432
|
+
"""Main entry point for command-line usage."""
|
|
433
|
+
parser = argparse.ArgumentParser(
|
|
434
|
+
description="PubMed Weekly Daily Updates Downloader and Combiner"
|
|
435
|
+
)
|
|
436
|
+
parser.add_argument("command", type=str, help="Command to execute")
|
|
437
|
+
parser.add_argument("args", nargs="*", help="Command arguments")
|
|
438
|
+
|
|
439
|
+
parsed = parser.parse_args()
|
|
440
|
+
|
|
441
|
+
command = parsed.command
|
|
442
|
+
args = parsed.args
|
|
443
|
+
|
|
444
|
+
if command == "calculate_week":
|
|
445
|
+
week = calculate_week()
|
|
446
|
+
print(week)
|
|
447
|
+
|
|
448
|
+
elif command == "fetch_files":
|
|
449
|
+
files = fetch_ftp_file_list()
|
|
450
|
+
print(" ".join(files))
|
|
451
|
+
|
|
452
|
+
elif command == "filter_files":
|
|
453
|
+
if len(args) < 2:
|
|
454
|
+
print(
|
|
455
|
+
"Usage: python pubmed_weekly.py filter_files <week_name> <file_list>"
|
|
456
|
+
)
|
|
457
|
+
sys.exit(1)
|
|
458
|
+
|
|
459
|
+
week_name = args[0]
|
|
460
|
+
file_list = args[1].split()
|
|
461
|
+
filtered = filter_files_by_date(week_name, file_list)
|
|
462
|
+
print(" ".join(filtered))
|
|
463
|
+
|
|
464
|
+
elif command == "download_file":
|
|
465
|
+
if len(args) < 2:
|
|
466
|
+
print(
|
|
467
|
+
"Usage: python pubmed_weekly.py download_file <week_name> <filename>"
|
|
468
|
+
)
|
|
469
|
+
sys.exit(1)
|
|
470
|
+
|
|
471
|
+
week_name = args[0]
|
|
472
|
+
filename = args[1]
|
|
473
|
+
sys.exit(download_file(week_name, filename))
|
|
474
|
+
|
|
475
|
+
elif command == "combine":
|
|
476
|
+
if len(args) < 1:
|
|
477
|
+
print("Usage: python pubmed_weekly.py combine <week_name>")
|
|
478
|
+
sys.exit(1)
|
|
479
|
+
|
|
480
|
+
week_name = args[0]
|
|
481
|
+
result = combine(week_name)
|
|
482
|
+
print(json.dumps(result, indent=2))
|
|
483
|
+
|
|
484
|
+
if not result.get("success"):
|
|
485
|
+
sys.exit(1)
|
|
486
|
+
|
|
487
|
+
else:
|
|
488
|
+
print(f"Unknown command: {command}")
|
|
489
|
+
sys.exit(1)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
if __name__ == "__main__":
|
|
493
|
+
main()
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: bioresearcher-python-setup-uv
|
|
3
|
+
description: "Sets up a project-local Python environment with the uv package manager: downloads uv binary from astral.sh, creates .venv, configures dependencies (pandas or scientific plotting: pymol-open-source, matplotlib, pymupdf, biopython), verifies install, and appends usage rules to AGENTS.md. Auto-races PyPI and CN mirrors. Use when uv/Python is missing, for plotting/analysis, or on requests like install uv, set up Python, prepare plotting packages, use a China mirror, or prepare .scripts/py/."
|
|
4
|
+
license: Apache-2.0
|
|
5
|
+
compatibility: "Unix-like shells (Linux, macOS, Git Bash) and Windows cmd.exe; requires curl or PowerShell for download"
|
|
6
|
+
metadata:
|
|
7
|
+
version: "1.1.0"
|
|
8
|
+
source: "opencode-bioresearcher-plugin@1.7.2"
|
|
9
|
+
allowed-tools: Bash Read
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
# Python Environment Setup with uv
|
|
13
|
+
|
|
14
|
+
This skill sets up a Python environment using the uv package manager.
|
|
15
|
+
|
|
16
|
+
## Prerequisites
|
|
17
|
+
- Internet connection for downloading uv
|
|
18
|
+
- Python 3.8+ should be available on PATH (or uv will prompt to install it)
|
|
19
|
+
|
|
20
|
+
## Steps
|
|
21
|
+
|
|
22
|
+
Follow the sequence below in order. Perform environment verification before proceeding.
|
|
23
|
+
|
|
24
|
+
### Step 1: Auto-Select the Fastest Package Index (mirror race)
|
|
25
|
+
|
|
26
|
+
Do NOT ask the user which mirror to use — probe and adopt the fastest index automatically (honor an explicit user-specified mirror if one was requested). Direct PyPI access can be extremely slow or unreachable from some networks (e.g. mainland China); regional mirrors fix this. Race PyPI, Aliyun, Tsinghua, and USTC with a timed probe and export the winner for the WHOLE session (uv reads `UV_INDEX_URL` automatically on every later `./uv pip` / `./uv venv` call):
|
|
27
|
+
|
|
28
|
+
**For Unix-like shells:**
|
|
29
|
+
```bash
|
|
30
|
+
export UV_INDEX_URL="$(
|
|
31
|
+
for idx in "https://pypi.org/simple" \
|
|
32
|
+
"https://mirrors.aliyun.com/pypi/simple" \
|
|
33
|
+
"https://pypi.tuna.tsinghua.edu.cn/simple" \
|
|
34
|
+
"https://mirrors.ustc.edu.cn/pypi/simple"; do
|
|
35
|
+
if t=$(curl -o /dev/null -sS -m 6 -w '%{time_total}' "$idx/pip/" 2>/dev/null); then
|
|
36
|
+
echo "$t $idx"
|
|
37
|
+
fi
|
|
38
|
+
done | sort -n | head -1 | cut -d' ' -f2-
|
|
39
|
+
)"
|
|
40
|
+
case "$UV_INDEX_URL" in http*) echo "Using index: $UV_INDEX_URL" ;; *) unset UV_INDEX_URL; echo "No index reachable; proceeding with defaults" ;; esac
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Only curl exit status 0 counts as a probe success (a mirror that fails fast — DNS block, connection refused, TLS error — is excluded entirely rather than winning with a bogus near-zero time).
|
|
44
|
+
|
|
45
|
+
**For Windows cmd.exe:**
|
|
46
|
+
```bash
|
|
47
|
+
powershell -NoProfile -Command "$t=@{}; foreach($u in 'https://pypi.org/simple','https://mirrors.aliyun.com/pypi/simple','https://pypi.tuna.tsinghua.edu.cn/simple','https://mirrors.ustc.edu.cn/pypi/simple'){ try { $sw=[Diagnostics.Stopwatch]::StartNew(); Invoke-WebRequest -UseBasicParsing -TimeoutSec 6 "$u/pip/" | Out-Null; $t[$sw.Elapsed.TotalSeconds]=$u } catch {} }; if($t.Count){ [Console]::WriteLine(($t.GetEnumerator() | Sort-Object Name | Select-Object -First 1).Value) }"
|
|
48
|
+
:: then: set UV_INDEX_URL=<winner> (or leave unset if PyPI is fast for you)
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Step 2: Detect Shell and Download uv Binary
|
|
52
|
+
|
|
53
|
+
First, detect your shell environment:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
# Detect shell type
|
|
57
|
+
# MSYSTEM is set by Git Bash, MINGW_PREFIX by MSYS2
|
|
58
|
+
if [ -n "$MSYSTEM" ] || [ -n "$MINGW_PREFIX" ] || command -v curl >/dev/null 2>&1; then
|
|
59
|
+
echo "Unix-like shell detected (Git Bash, bash, zsh, etc.)"
|
|
60
|
+
IS_UNIX_SHELL=true
|
|
61
|
+
else
|
|
62
|
+
echo "Windows cmd.exe detected"
|
|
63
|
+
IS_UNIX_SHELL=false
|
|
64
|
+
fi
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Download the official standalone installer script to disk, verify, and run locally into `.uv`:
|
|
68
|
+
|
|
69
|
+
**For Unix-like shells (Git Bash / macOS / Linux):**
|
|
70
|
+
```bash
|
|
71
|
+
mkdir -p .uv
|
|
72
|
+
curl -LsSf https://astral.sh/uv/install.sh -o .uv/install-uv.sh
|
|
73
|
+
UV_INSTALL_DIR="$(pwd)/.uv" sh .uv/install-uv.sh
|
|
74
|
+
rm -f .uv/install-uv.sh
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
The installer already falls back from `releases.astral.sh` to `github.com` automatically. If BOTH are slow/unreachable (mainland China), retry with a GitHub-proxy mirror via the documented `UV_DOWNLOAD_URL` override:
|
|
78
|
+
```bash
|
|
79
|
+
UV_DOWNLOAD_URL="https://ghfast.top/https://github.com/astral-sh/uv/releases/download" \
|
|
80
|
+
UV_INSTALL_DIR="$(pwd)/.uv" sh .uv/install-uv.sh
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
**For Windows cmd.exe (if Git Bash unavailable):**
|
|
84
|
+
```bash
|
|
85
|
+
powershell -NoProfile -ExecutionPolicy Bypass -Command "New-Item -ItemType Directory -Force -Path .uv | Out-Null; $env:UV_INSTALL_DIR = (Get-Location).Path + '\.uv'; Invoke-WebRequest -Uri 'https://astral.sh/uv/install.ps1' -OutFile '.uv\install-uv.ps1'; & '.uv\install-uv.ps1'; Remove-Item -Force '.uv\install-uv.ps1'"
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### Step 3: Create Symlink or Copy uv to Working Directory
|
|
89
|
+
|
|
90
|
+
**For Unix-like shells (Git Bash / macOS / Linux):**
|
|
91
|
+
```bash
|
|
92
|
+
ln -sf .uv/uv uv
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
**For Windows cmd.exe:**
|
|
96
|
+
|
|
97
|
+
Try symlink first, fall back to copy if no Admin rights:
|
|
98
|
+
```bash
|
|
99
|
+
cmd /c "(mklink uv .uv\uv.exe) 2>nul || copy /Y .uv\uv.exe uv.exe"
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### Step 4: Create Virtual Environment and Install Packages
|
|
103
|
+
|
|
104
|
+
NOTE: this step (package installation) may timeout. If timed out, ask the user whether they would like to retry package installation. If successful, do NOT ask any question and continue to Step 5.
|
|
105
|
+
|
|
106
|
+
**For Unix-like shells:**
|
|
107
|
+
```bash
|
|
108
|
+
./uv venv .venv
|
|
109
|
+
# Standard Analysis:
|
|
110
|
+
VIRTUAL_ENV="$(pwd)/.venv" ./uv pip install pandas
|
|
111
|
+
# Scientific Visualization & Plot-Making (for bioresearcher-plot-making):
|
|
112
|
+
# VIRTUAL_ENV="$(pwd)/.venv" ./uv pip install pymol-open-source matplotlib pymupdf numpy pillow biopython pandas
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
CRITICAL: always export `VIRTUAL_ENV="$(pwd)/.venv"` before `./uv pip install` and `./uv run`. Without it, an active conda/mamba environment on the host (`CONDA_PREFIX`) takes precedence over the project `./.venv`, and uv will silently install into (and mutate) the HOST environment.
|
|
116
|
+
|
|
117
|
+
All `./uv pip install` calls automatically use the `UV_INDEX_URL` selected in Step 1 (pass `--index-url "$UV_INDEX_URL"` explicitly if the env var may have been dropped between commands).
|
|
118
|
+
|
|
119
|
+
**For Windows cmd.exe:**
|
|
120
|
+
```bash
|
|
121
|
+
uv.exe venv .venv
|
|
122
|
+
# Standard Analysis:
|
|
123
|
+
uv.exe pip install --python .venv\Scripts\python.exe pandas
|
|
124
|
+
# Scientific Visualization & Plot-Making:
|
|
125
|
+
# uv.exe pip install --python .venv\Scripts\python.exe pymol-open-source matplotlib pymupdf numpy pillow biopython pandas
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
### Step 5: Verification
|
|
129
|
+
|
|
130
|
+
**For Unix-like shells:**
|
|
131
|
+
```bash
|
|
132
|
+
./uv --version
|
|
133
|
+
./.venv/bin/python -c "import pandas; print('pandas', pandas.__version__)"
|
|
134
|
+
# For visualization stack verification:
|
|
135
|
+
# ./.venv/bin/python -c "from pymol import cmd; import matplotlib, pymupdf, Bio; print('PyMOL + Plotting stack ready')"
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
**For Windows cmd.exe:**
|
|
139
|
+
```bash
|
|
140
|
+
uv.exe --version
|
|
141
|
+
.venv\Scripts\python.exe -c "import pandas; print('pandas', pandas.__version__)"
|
|
142
|
+
# For visualization stack verification:
|
|
143
|
+
# .venv\Scripts\python.exe -c "from pymol import cmd; import matplotlib, pymupdf, Bio; print('PyMOL + Plotting stack ready')"
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
### Step 6: Update Agent Instruction File
|
|
147
|
+
|
|
148
|
+
Ask the user whether they want to update AGENTS.md (or CLAUDE.md / other agent instruction file) in the WORKING DIRECTORY to "direct agents to use the installed UV Python" (options: "Yes" / "No"). If you receive no answer, continue to Step 7 (do NOT modify the instruction file NOR create directories). If you receive a "Yes" answer, follow the steps below.
|
|
149
|
+
|
|
150
|
+
1. If the agent instruction file (AGENTS.md or equivalent) is not found in WORKING DIR, create an empty AGENTS.md.
|
|
151
|
+
2. Inspect its content. If you do not see the content block below, APPEND EXACTLY AS IS to the end of the file.
|
|
152
|
+
3. Check if `./.scripts/py` exists. If not, create the directories.
|
|
153
|
+
|
|
154
|
+
Content block:
|
|
155
|
+
|
|
156
|
+
```md
|
|
157
|
+
<!-- BEGIN BIORESEARCHER UV ENVIRONMENT GUIDELINES -->
|
|
158
|
+
## Important note about Python
|
|
159
|
+
|
|
160
|
+
ALWAYS use the uv package manager available in WORKING DIRECTORY. Pin every install and invocation to the project-local virtual environment: `VIRTUAL_ENV="$(pwd)/.venv" ./uv pip ...` for package management and `./.venv/bin/python ...` to run Python scripts or package executables. NEVER run bare `uv pip install` or `uv run` — an active host conda environment (CONDA_PREFIX) would silently take precedence over `./.venv`.
|
|
161
|
+
|
|
162
|
+
ALWAYS save python scripts under path `./.scripts/py/` and run the script with `./.venv/bin/python ...` whenever your work involves executing python scripts. Your script MUST contain concise docstrings and comments and use good engineering practices including separation of concerns.
|
|
163
|
+
<!-- END BIORESEARCHER UV ENVIRONMENT GUIDELINES -->
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
### Step 7: Return summary to user (Usage After Setup)
|
|
167
|
+
|
|
168
|
+
**For Unix-like shells:**
|
|
169
|
+
```bash
|
|
170
|
+
./.venv/bin/python your_script.py
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
**For Windows cmd.exe:**
|
|
174
|
+
```bash
|
|
175
|
+
.venv\Scripts\python.exe your_script.py
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
## Notes
|
|
179
|
+
- Add `.uv/` and `.venv/` to `.gitignore`
|
|
180
|
+
- `uv run`/`uv pip` only target `./.venv` when no `--python` flag, `VIRTUAL_ENV`, or host `CONDA_PREFIX` takes precedence — pin with `VIRTUAL_ENV="$(pwd)/.venv"` or invoke `./.venv/bin/python` directly
|
|
181
|
+
- If uv must download a Python interpreter (none found on PATH), set `UV_PYTHON_INSTALL_MIRROR` to a GitHub-proxy prefix of `https://github.com/astral-sh/python-build-standalone/releases/download` on slow networks
|
|
182
|
+
- Use `VIRTUAL_ENV="$(pwd)/.venv" ./uv pip install <package>` (Unix) or `uv.exe pip install --python .venv\Scripts\python.exe <package>` (Windows cmd.exe) for additional packages; avoid `uv add` unless a `pyproject.toml` project is intended
|
|
183
|
+
- Windows with Git Bash: Follow Unix-like shell instructions
|
|
184
|
+
- Windows cmd.exe without Admin rights: `uv.exe` is copied instead of symlinked
|