@paradigma-inc/flywheel 0.1.95 → 0.1.99
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -4
- package/package.json +2 -1
- package/skills/flywheel-llm-proof-paper/SKILL.md +156 -0
- package/skills/flywheel-llm-proof-paper/agents/openai.yaml +4 -0
- package/skills/flywheel-llm-proof-paper/references/checks.md +60 -0
- package/skills/flywheel-llm-proof-paper/scripts/__pycache__/check_paper.cpython-311.pyc +0 -0
- package/skills/flywheel-llm-proof-paper/scripts/check_paper.py +1065 -0
- package/src/cli.mjs +27 -3
- package/src/public-command-metadata.mjs +44 -0
- package/src/runtime/vendor/flywheel-cli-dist/commands/_wait-polling.d.ts +21 -0
- package/src/runtime/vendor/flywheel-cli-dist/commands/_wait-polling.js +68 -2
- package/src/runtime/vendor/flywheel-cli-dist/commands/_wait-polling.js.map +1 -1
- package/src/runtime/vendor/flywheel-cli-dist/commands/compute-acquire.js +2 -8
- package/src/runtime/vendor/flywheel-cli-dist/commands/compute-acquire.js.map +1 -1
- package/src/runtime/vendor/flywheel-cli-dist/commands/feedback-create.js +4 -0
- package/src/runtime/vendor/flywheel-cli-dist/commands/feedback-create.js.map +1 -1
- package/src/runtime/vendor/flywheel-cli-dist/commands/registry/compute.js +1 -1
- package/src/runtime/vendor/flywheel-cli-dist/commands/registry/compute.js.map +1 -1
- package/src/runtime/vendor/flywheel-cli-dist/commands/registry/resources.js +16 -0
- package/src/runtime/vendor/flywheel-cli-dist/commands/registry/resources.js.map +1 -1
- package/src/runtime/vendor/manifest.json +1 -1
- package/src/setup/shared/prior-mode-detect.mjs +76 -9
- package/src/unified-cli.mjs +54 -16
- package/src/update/cache.mjs +124 -0
- package/src/update/command.mjs +438 -0
- package/src/update/install-state.mjs +135 -0
- package/src/update/refresh.mjs +393 -0
- package/src/update/registry.mjs +35 -0
- package/src/update/version.mjs +59 -0
- package/src/update/warning.mjs +109 -0
|
@@ -0,0 +1,1065 @@
|
|
|
1
|
+
#!/usr/bin/env -S uv run --script
|
|
2
|
+
# /// script
|
|
3
|
+
# requires-python = ">=3.10"
|
|
4
|
+
# dependencies = [
|
|
5
|
+
# "bibtexparser>=1.4,<2",
|
|
6
|
+
# "tenacity>=9.1.2",
|
|
7
|
+
# ]
|
|
8
|
+
# ///
|
|
9
|
+
"""Small local preflight for LaTeX papers."""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
import dataclasses
|
|
15
|
+
import difflib
|
|
16
|
+
import json
|
|
17
|
+
import re
|
|
18
|
+
import shlex
|
|
19
|
+
import shutil
|
|
20
|
+
import subprocess
|
|
21
|
+
import sys
|
|
22
|
+
import time
|
|
23
|
+
import urllib.error
|
|
24
|
+
import urllib.parse
|
|
25
|
+
import urllib.request
|
|
26
|
+
import xml.etree.ElementTree as ET
|
|
27
|
+
from collections.abc import Callable, Sequence
|
|
28
|
+
from dataclasses import dataclass
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import Literal
|
|
31
|
+
|
|
32
|
+
import bibtexparser
|
|
33
|
+
from bibtexparser.bparser import BibTexParser
|
|
34
|
+
from tenacity import (
|
|
35
|
+
retry,
|
|
36
|
+
retry_if_exception,
|
|
37
|
+
stop_after_attempt,
|
|
38
|
+
wait_exponential_jitter,
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
Severity = Literal["hard", "warning"]
|
|
42
|
+
|
|
43
|
+
USER_AGENT = "paper-llm-proof-skill/0.1 (+https://github.com)"
|
|
44
|
+
CSL_JSON_ACCEPT = "application/vnd.citationstyles.csl+json"
|
|
45
|
+
RETRYABLE_HTTP_STATUSES = {0, 408, 425, 429, 500, 502, 503, 504}
|
|
46
|
+
MIN_INTERVAL_BY_HOST = {"api.crossref.org": 0.2, "export.arxiv.org": 3.0}
|
|
47
|
+
TITLE_SIMILARITY_THRESHOLD = 0.70
|
|
48
|
+
TITLE_TOKEN_SIMILARITY_THRESHOLD = 0.85
|
|
49
|
+
TITLE_STOPWORDS = {
|
|
50
|
+
"a",
|
|
51
|
+
"an",
|
|
52
|
+
"and",
|
|
53
|
+
"are",
|
|
54
|
+
"as",
|
|
55
|
+
"at",
|
|
56
|
+
"be",
|
|
57
|
+
"been",
|
|
58
|
+
"being",
|
|
59
|
+
"by",
|
|
60
|
+
"for",
|
|
61
|
+
"from",
|
|
62
|
+
"in",
|
|
63
|
+
"is",
|
|
64
|
+
"of",
|
|
65
|
+
"on",
|
|
66
|
+
"or",
|
|
67
|
+
"the",
|
|
68
|
+
"to",
|
|
69
|
+
"was",
|
|
70
|
+
"were",
|
|
71
|
+
"with",
|
|
72
|
+
}
|
|
73
|
+
SKIP_DIRS = {
|
|
74
|
+
".git",
|
|
75
|
+
".hg",
|
|
76
|
+
".svn",
|
|
77
|
+
".venv",
|
|
78
|
+
"__pycache__",
|
|
79
|
+
"build",
|
|
80
|
+
"dist",
|
|
81
|
+
"node_modules",
|
|
82
|
+
"out",
|
|
83
|
+
}
|
|
84
|
+
BIB_RESOURCE_RE = re.compile(
|
|
85
|
+
r"\\(?:bibliography|addbibresource)(?:\s*\[[^\]]*\])?\s*\{(?P<names>[^}]+)\}",
|
|
86
|
+
re.IGNORECASE,
|
|
87
|
+
)
|
|
88
|
+
BIB_ENTRY_RE = re.compile(
|
|
89
|
+
r"@\s*(?P<type>[A-Za-z][\w-]*)\s*[{(]\s*(?P<key>[^,\s]+)",
|
|
90
|
+
re.IGNORECASE,
|
|
91
|
+
)
|
|
92
|
+
DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.IGNORECASE)
|
|
93
|
+
ARXIV_RE = re.compile(
|
|
94
|
+
r"(?P<id>(?:[a-z-]+(?:\.[A-Z]{2})?/\d{7})|(?:\d{4}\.\d{4,5}))(?P<version>v\d+)?",
|
|
95
|
+
re.IGNORECASE,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass(frozen=True)
|
|
100
|
+
class BibEntry:
|
|
101
|
+
key: str
|
|
102
|
+
fields: dict[str, str]
|
|
103
|
+
file: Path
|
|
104
|
+
line: int
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@dataclass(frozen=True)
|
|
108
|
+
class Issue:
|
|
109
|
+
severity: Severity
|
|
110
|
+
code: str
|
|
111
|
+
message: str
|
|
112
|
+
file: str | None = None
|
|
113
|
+
line: int | None = None
|
|
114
|
+
key: str | None = None
|
|
115
|
+
evidence: str | None = None
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@dataclass(frozen=True)
|
|
119
|
+
class Report:
|
|
120
|
+
project: str
|
|
121
|
+
tex_files: tuple[str, ...]
|
|
122
|
+
bib_files: tuple[str, ...]
|
|
123
|
+
checked_keys: tuple[str, ...]
|
|
124
|
+
slop_guard_requested: bool
|
|
125
|
+
issues: tuple[Issue, ...]
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
@dataclass(frozen=True)
|
|
129
|
+
class ExtractedIdentifier:
|
|
130
|
+
value: str | None
|
|
131
|
+
error: str | None = None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass(frozen=True)
|
|
135
|
+
class CatalogTitle:
|
|
136
|
+
raw: str
|
|
137
|
+
text: str
|
|
138
|
+
warning: str | None = None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
class LookupErrorWithStatus(RuntimeError):
|
|
142
|
+
def __init__(self, status: int, message: str) -> None:
|
|
143
|
+
super().__init__(message)
|
|
144
|
+
self.status = status
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
class CatalogClient:
|
|
148
|
+
def __init__(self, timeout: float) -> None:
|
|
149
|
+
self.timeout = timeout
|
|
150
|
+
self.last_request_by_host: dict[str, float] = {}
|
|
151
|
+
|
|
152
|
+
def get_json(
|
|
153
|
+
self, url: str, headers: dict[str, str] | None = None
|
|
154
|
+
) -> dict[str, object]:
|
|
155
|
+
parsed = json.loads(self.get_text(url, headers=headers))
|
|
156
|
+
if not isinstance(parsed, dict):
|
|
157
|
+
raise LookupErrorWithStatus(0, f"Expected JSON object from {url}")
|
|
158
|
+
return parsed
|
|
159
|
+
|
|
160
|
+
def get_text(self, url: str, headers: dict[str, str] | None = None) -> str:
|
|
161
|
+
return self.fetch(url, headers=headers)
|
|
162
|
+
|
|
163
|
+
@retry(
|
|
164
|
+
retry=retry_if_exception(
|
|
165
|
+
lambda exc: (
|
|
166
|
+
isinstance(exc, LookupErrorWithStatus)
|
|
167
|
+
and exc.status in RETRYABLE_HTTP_STATUSES
|
|
168
|
+
)
|
|
169
|
+
),
|
|
170
|
+
wait=wait_exponential_jitter(initial=1, max=30),
|
|
171
|
+
stop=stop_after_attempt(4),
|
|
172
|
+
reraise=True,
|
|
173
|
+
)
|
|
174
|
+
def fetch(self, url: str, headers: dict[str, str] | None = None) -> str:
|
|
175
|
+
self.wait_for_rate_limit(url)
|
|
176
|
+
request_headers = {"User-Agent": USER_AGENT}
|
|
177
|
+
if headers:
|
|
178
|
+
request_headers.update(headers)
|
|
179
|
+
request = urllib.request.Request(url, headers=request_headers)
|
|
180
|
+
try:
|
|
181
|
+
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
|
182
|
+
return response.read().decode("utf-8")
|
|
183
|
+
except urllib.error.HTTPError as exc:
|
|
184
|
+
raise LookupErrorWithStatus(exc.code, f"HTTP {exc.code} for {url}") from exc
|
|
185
|
+
except urllib.error.URLError as exc:
|
|
186
|
+
raise LookupErrorWithStatus(
|
|
187
|
+
0, f"Network error for {url}: {exc.reason}"
|
|
188
|
+
) from exc
|
|
189
|
+
|
|
190
|
+
def wait_for_rate_limit(self, url: str) -> None:
|
|
191
|
+
host = urllib.parse.urlparse(url).netloc.lower()
|
|
192
|
+
min_interval = MIN_INTERVAL_BY_HOST.get(host)
|
|
193
|
+
if min_interval is None:
|
|
194
|
+
return
|
|
195
|
+
now = time.monotonic()
|
|
196
|
+
if (
|
|
197
|
+
host in self.last_request_by_host
|
|
198
|
+
and now - self.last_request_by_host[host] < min_interval
|
|
199
|
+
):
|
|
200
|
+
time.sleep(min_interval - (now - self.last_request_by_host[host]))
|
|
201
|
+
self.last_request_by_host[host] = time.monotonic()
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
205
|
+
parser = build_parser()
|
|
206
|
+
args = parser.parse_args(argv)
|
|
207
|
+
try:
|
|
208
|
+
report = run_check(Path(args.project).expanduser().resolve(), args)
|
|
209
|
+
except Exception as exc:
|
|
210
|
+
print(f"paper-llm-proof could not complete: {exc}", file=sys.stderr)
|
|
211
|
+
return 3
|
|
212
|
+
|
|
213
|
+
hard_count = sum(issue.severity == "hard" for issue in report.issues)
|
|
214
|
+
warning_count = sum(issue.severity == "warning" for issue in report.issues)
|
|
215
|
+
exit_code = 2 if hard_count else 0
|
|
216
|
+
rendered = render_report(report, args.format)
|
|
217
|
+
if args.out:
|
|
218
|
+
out_path = Path(args.out).expanduser().resolve()
|
|
219
|
+
out_path.write_text(rendered, encoding="utf-8")
|
|
220
|
+
print(
|
|
221
|
+
f"paper-llm-proof wrote {out_path} (hard={hard_count}, warnings={warning_count}, exit={exit_code})",
|
|
222
|
+
file=sys.stderr,
|
|
223
|
+
)
|
|
224
|
+
else:
|
|
225
|
+
print(rendered)
|
|
226
|
+
return exit_code
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
230
|
+
parser = argparse.ArgumentParser(
|
|
231
|
+
description="Sanity-check a LaTeX/BibTeX paper for real references and prose lint signals.",
|
|
232
|
+
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
233
|
+
)
|
|
234
|
+
parser.add_argument("project", help="Path to the LaTeX project root")
|
|
235
|
+
parser.add_argument(
|
|
236
|
+
"--bib", action="append", default=[], help="Additional .bib file to include"
|
|
237
|
+
)
|
|
238
|
+
parser.add_argument("--format", choices=("markdown", "json"), default="markdown")
|
|
239
|
+
parser.add_argument("--out", help="Write report to this file")
|
|
240
|
+
parser.add_argument(
|
|
241
|
+
"--offline",
|
|
242
|
+
action="store_true",
|
|
243
|
+
help="Skip external catalog calls. Bibliography parsing still runs.",
|
|
244
|
+
)
|
|
245
|
+
parser.add_argument(
|
|
246
|
+
"--slop-guard",
|
|
247
|
+
action="store_true",
|
|
248
|
+
help="Run Slop Guard and report warning-only prose lint evidence.",
|
|
249
|
+
)
|
|
250
|
+
parser.add_argument(
|
|
251
|
+
"--slop-guard-command",
|
|
252
|
+
help="Command used to launch Slop Guard. The checker appends '-j -'.",
|
|
253
|
+
)
|
|
254
|
+
parser.add_argument("--timeout", type=float, default=60.0)
|
|
255
|
+
parser.add_argument("--no-cache", action="store_true", help=argparse.SUPPRESS)
|
|
256
|
+
return parser
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def run_check(project: Path, args: argparse.Namespace) -> Report:
|
|
260
|
+
if not project.exists() or not project.is_dir():
|
|
261
|
+
raise NotADirectoryError(f"Project path is not a directory: {project}")
|
|
262
|
+
|
|
263
|
+
tex_files = find_files(project, ".tex")
|
|
264
|
+
if not tex_files:
|
|
265
|
+
raise FileNotFoundError(f"No .tex files found under {project}")
|
|
266
|
+
|
|
267
|
+
bib_files = resolve_bib_files(
|
|
268
|
+
project, tex_files, resolve_explicit_bib_files(project, args.bib)
|
|
269
|
+
)
|
|
270
|
+
entries = parse_bibliography_files(bib_files)
|
|
271
|
+
entries_by_key = {entry.key: entry for entry in entries}
|
|
272
|
+
keys_to_check = tuple(sorted(entries_by_key))
|
|
273
|
+
|
|
274
|
+
issues = []
|
|
275
|
+
if not bib_files:
|
|
276
|
+
issues.append(
|
|
277
|
+
Issue(
|
|
278
|
+
"hard",
|
|
279
|
+
"no-bib-files",
|
|
280
|
+
"No BibTeX files were found or referenced by the LaTeX project.",
|
|
281
|
+
file=str(project),
|
|
282
|
+
)
|
|
283
|
+
)
|
|
284
|
+
client = None if args.offline else CatalogClient(args.timeout)
|
|
285
|
+
issues.extend(
|
|
286
|
+
verify_references([entries_by_key[key] for key in keys_to_check], client)
|
|
287
|
+
)
|
|
288
|
+
if args.slop_guard:
|
|
289
|
+
issues.extend(run_slop_guard(tex_files, args.slop_guard_command))
|
|
290
|
+
|
|
291
|
+
return Report(
|
|
292
|
+
project=str(project),
|
|
293
|
+
tex_files=tuple(str(path) for path in tex_files),
|
|
294
|
+
bib_files=tuple(str(path) for path in bib_files),
|
|
295
|
+
checked_keys=keys_to_check,
|
|
296
|
+
slop_guard_requested=bool(args.slop_guard),
|
|
297
|
+
issues=tuple(issues),
|
|
298
|
+
)
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def find_files(root: Path, suffix: str) -> tuple[Path, ...]:
|
|
302
|
+
return tuple(
|
|
303
|
+
sorted(
|
|
304
|
+
path.resolve()
|
|
305
|
+
for path in root.rglob(f"*{suffix}")
|
|
306
|
+
if not any(part in SKIP_DIRS for part in path.relative_to(root).parts[:-1])
|
|
307
|
+
)
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def resolve_explicit_bib_files(
|
|
312
|
+
project: Path, bib_paths: Sequence[str]
|
|
313
|
+
) -> tuple[Path, ...]:
|
|
314
|
+
paths = []
|
|
315
|
+
for bib_path in bib_paths:
|
|
316
|
+
resolved = Path(bib_path).expanduser()
|
|
317
|
+
resolved = resolved if resolved.is_absolute() else project / resolved
|
|
318
|
+
if not resolved.exists():
|
|
319
|
+
raise FileNotFoundError(f"Explicit BibTeX file does not exist: {resolved}")
|
|
320
|
+
if not resolved.is_file():
|
|
321
|
+
raise IsADirectoryError(f"Explicit BibTeX path is not a file: {resolved}")
|
|
322
|
+
paths.append(resolved.resolve())
|
|
323
|
+
return tuple(paths)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def resolve_bib_files(
|
|
327
|
+
project: Path, tex_files: Sequence[Path], explicit_bibs: Sequence[Path]
|
|
328
|
+
) -> tuple[Path, ...]:
|
|
329
|
+
paths = {bib for bib in explicit_bibs if bib.exists()}
|
|
330
|
+
for tex_file in tex_files:
|
|
331
|
+
visible_text = strip_latex_comments(read_text(tex_file))
|
|
332
|
+
for match in BIB_RESOURCE_RE.finditer(visible_text):
|
|
333
|
+
for raw_name in match.group("names").split(","):
|
|
334
|
+
if resolved := resolve_bib_reference(
|
|
335
|
+
project, tex_file, raw_name.strip()
|
|
336
|
+
):
|
|
337
|
+
paths.add(resolved)
|
|
338
|
+
return tuple(sorted(paths or find_files(project, ".bib")))
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def resolve_bib_reference(project: Path, tex_file: Path, name: str) -> Path | None:
|
|
342
|
+
if not name:
|
|
343
|
+
return None
|
|
344
|
+
candidate = Path(name)
|
|
345
|
+
if candidate.suffix != ".bib":
|
|
346
|
+
candidate = candidate.with_suffix(".bib")
|
|
347
|
+
for path in (
|
|
348
|
+
candidate if candidate.is_absolute() else tex_file.parent / candidate,
|
|
349
|
+
project / candidate,
|
|
350
|
+
):
|
|
351
|
+
if path.exists():
|
|
352
|
+
return path.resolve()
|
|
353
|
+
return None
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
def parse_bibliography_files(bib_files: Sequence[Path]) -> tuple[BibEntry, ...]:
|
|
357
|
+
return tuple(
|
|
358
|
+
entry
|
|
359
|
+
for bib_file in bib_files
|
|
360
|
+
for entry in parse_bibtex(read_text(bib_file), bib_file)
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def parse_bibtex(text: str, path: Path) -> list[BibEntry]:
|
|
365
|
+
parser = BibTexParser(common_strings=True)
|
|
366
|
+
parser.homogenize_fields = False
|
|
367
|
+
parser.interpolate_strings = False
|
|
368
|
+
line_numbers = entry_line_numbers(text)
|
|
369
|
+
entries = []
|
|
370
|
+
for raw in bibtexparser.loads(text, parser=parser).entries:
|
|
371
|
+
key = str(raw.get("ID", "")).strip()
|
|
372
|
+
if key:
|
|
373
|
+
fields = {
|
|
374
|
+
field.lower(): normalize_space(str(value))
|
|
375
|
+
for field, value in raw.items()
|
|
376
|
+
if field not in {"ID", "ENTRYTYPE"}
|
|
377
|
+
}
|
|
378
|
+
lines_for_key = line_numbers.get(key)
|
|
379
|
+
entries.append(
|
|
380
|
+
BibEntry(
|
|
381
|
+
key, fields, path, lines_for_key.pop(0) if lines_for_key else 1
|
|
382
|
+
)
|
|
383
|
+
)
|
|
384
|
+
return entries
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def entry_line_numbers(text: str) -> dict[str, list[int]]:
|
|
388
|
+
lines: dict[str, list[int]] = {}
|
|
389
|
+
for match in BIB_ENTRY_RE.finditer(text):
|
|
390
|
+
lines.setdefault(match.group("key").strip(), []).append(
|
|
391
|
+
line_for_offset(text, match.start())
|
|
392
|
+
)
|
|
393
|
+
return lines
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def verify_references(
|
|
397
|
+
entries: Sequence[BibEntry], client: CatalogClient | None
|
|
398
|
+
) -> list[Issue]:
|
|
399
|
+
return [issue for entry in entries for issue in verify_reference(entry, client)]
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def verify_reference(entry: BibEntry, client: CatalogClient | None) -> list[Issue]:
|
|
403
|
+
title = catalog_title(entry.fields.get("title", ""))
|
|
404
|
+
doi = extract_doi(entry)
|
|
405
|
+
arxiv_id = extract_arxiv_id(entry)
|
|
406
|
+
for identifier, code, label in (
|
|
407
|
+
(doi, "malformed-doi", "DOI"),
|
|
408
|
+
(arxiv_id, "malformed-arxiv-id", "arXiv identifier"),
|
|
409
|
+
):
|
|
410
|
+
if identifier.error:
|
|
411
|
+
return [
|
|
412
|
+
reference_issue(
|
|
413
|
+
entry,
|
|
414
|
+
"hard",
|
|
415
|
+
code,
|
|
416
|
+
f"BibTeX key '{entry.key}' has a malformed {label} field.",
|
|
417
|
+
identifier.error,
|
|
418
|
+
)
|
|
419
|
+
]
|
|
420
|
+
if not doi.value and not arxiv_id.value:
|
|
421
|
+
return [
|
|
422
|
+
reference_issue(
|
|
423
|
+
entry,
|
|
424
|
+
"warning",
|
|
425
|
+
"reference-needs-agent-check",
|
|
426
|
+
f"BibTeX key '{entry.key}' has no DOI or arXiv ID; verify the title manually.",
|
|
427
|
+
title.text or title.raw or "No title",
|
|
428
|
+
)
|
|
429
|
+
]
|
|
430
|
+
if client is None:
|
|
431
|
+
return [
|
|
432
|
+
reference_issue(
|
|
433
|
+
entry,
|
|
434
|
+
"warning",
|
|
435
|
+
"catalog-check-skipped",
|
|
436
|
+
f"Skipped catalog reality check for '{entry.key}' because --offline was used.",
|
|
437
|
+
)
|
|
438
|
+
]
|
|
439
|
+
if doi.value and not is_arxiv_doi(doi.value):
|
|
440
|
+
return verify_identifier(
|
|
441
|
+
entry,
|
|
442
|
+
"doi",
|
|
443
|
+
doi.value,
|
|
444
|
+
title,
|
|
445
|
+
lambda: lookup_doi(doi.value, client),
|
|
446
|
+
"doi-not-found",
|
|
447
|
+
"doi-title-mismatch",
|
|
448
|
+
)
|
|
449
|
+
return verify_identifier(
|
|
450
|
+
entry,
|
|
451
|
+
"arxiv",
|
|
452
|
+
arxiv_id.value or "",
|
|
453
|
+
title,
|
|
454
|
+
lambda: lookup_arxiv(arxiv_id.value or "", client),
|
|
455
|
+
"arxiv-not-found",
|
|
456
|
+
"arxiv-title-mismatch",
|
|
457
|
+
)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def verify_identifier(
|
|
461
|
+
entry: BibEntry,
|
|
462
|
+
method: str,
|
|
463
|
+
identifier: str,
|
|
464
|
+
title: CatalogTitle,
|
|
465
|
+
lookup: Callable[[], tuple[str, str] | None],
|
|
466
|
+
not_found_code: str,
|
|
467
|
+
mismatch_code: str,
|
|
468
|
+
) -> list[Issue]:
|
|
469
|
+
try:
|
|
470
|
+
record = lookup()
|
|
471
|
+
except LookupErrorWithStatus as exc:
|
|
472
|
+
return [
|
|
473
|
+
reference_issue(
|
|
474
|
+
entry,
|
|
475
|
+
"hard",
|
|
476
|
+
"catalog-request-failed",
|
|
477
|
+
f"Could not complete {method} lookup for '{entry.key}'.",
|
|
478
|
+
str(exc),
|
|
479
|
+
)
|
|
480
|
+
]
|
|
481
|
+
if record is None:
|
|
482
|
+
return [
|
|
483
|
+
reference_issue(
|
|
484
|
+
entry,
|
|
485
|
+
"hard",
|
|
486
|
+
not_found_code,
|
|
487
|
+
f"{method} for BibTeX key '{entry.key}' did not resolve.",
|
|
488
|
+
identifier,
|
|
489
|
+
)
|
|
490
|
+
]
|
|
491
|
+
source, remote_title = record
|
|
492
|
+
if not remote_title:
|
|
493
|
+
return [
|
|
494
|
+
reference_issue(
|
|
495
|
+
entry,
|
|
496
|
+
"warning",
|
|
497
|
+
"title-needs-agent-check",
|
|
498
|
+
f"{method} for BibTeX key '{entry.key}' resolves, "
|
|
499
|
+
"but no catalog title was available for automatic comparison.",
|
|
500
|
+
no_catalog_title_evidence(title, source),
|
|
501
|
+
)
|
|
502
|
+
]
|
|
503
|
+
if title.warning:
|
|
504
|
+
return [
|
|
505
|
+
reference_issue(
|
|
506
|
+
entry,
|
|
507
|
+
"warning",
|
|
508
|
+
"title-needs-agent-check",
|
|
509
|
+
f"{method} for BibTeX key '{entry.key}' resolves, but the BibTeX title needs agent review.",
|
|
510
|
+
title_warning_evidence(title, source, remote_title),
|
|
511
|
+
)
|
|
512
|
+
]
|
|
513
|
+
score = title_similarity(title.text, remote_title) if remote_title else None
|
|
514
|
+
token_score = title_token_similarity(title.text, remote_title)
|
|
515
|
+
if score is not None and (
|
|
516
|
+
score < TITLE_SIMILARITY_THRESHOLD
|
|
517
|
+
or token_score < TITLE_TOKEN_SIMILARITY_THRESHOLD
|
|
518
|
+
):
|
|
519
|
+
if title_prefix_variant(title.text, remote_title):
|
|
520
|
+
return [
|
|
521
|
+
reference_issue(
|
|
522
|
+
entry,
|
|
523
|
+
"warning",
|
|
524
|
+
"title-needs-agent-check",
|
|
525
|
+
f"{method} for BibTeX key '{entry.key}' resolves, "
|
|
526
|
+
"but the title needs agent review.",
|
|
527
|
+
title_variant_evidence(
|
|
528
|
+
title.text, source, remote_title, score, token_score
|
|
529
|
+
),
|
|
530
|
+
)
|
|
531
|
+
]
|
|
532
|
+
evidence = (
|
|
533
|
+
f"BibTeX: {title.text} | {source}: {remote_title} | "
|
|
534
|
+
f"score={score:.2f} | token_score={token_score:.2f}"
|
|
535
|
+
)
|
|
536
|
+
return [
|
|
537
|
+
reference_issue(
|
|
538
|
+
entry,
|
|
539
|
+
"hard",
|
|
540
|
+
mismatch_code,
|
|
541
|
+
f"{method} for BibTeX key '{entry.key}' resolves, but the title does not match.",
|
|
542
|
+
evidence,
|
|
543
|
+
)
|
|
544
|
+
]
|
|
545
|
+
return []
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def title_warning_evidence(title: CatalogTitle, source: str, remote_title: str) -> str:
|
|
549
|
+
parts = [title.warning or "Title could not be compared automatically."]
|
|
550
|
+
if title.raw:
|
|
551
|
+
parts.append(f"BibTeX: {title.raw}")
|
|
552
|
+
parts.append(f"{source}: {remote_title}")
|
|
553
|
+
return " | ".join(parts)
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def no_catalog_title_evidence(title: CatalogTitle, source: str) -> str:
|
|
557
|
+
title_text = title.raw or title.text or "No BibTeX title"
|
|
558
|
+
return (
|
|
559
|
+
f"{source} resolved without title metadata. "
|
|
560
|
+
f"BibTeX title needs agent review: {title_text}"
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
def title_variant_evidence(
|
|
565
|
+
local_title: str,
|
|
566
|
+
source: str,
|
|
567
|
+
remote_title: str,
|
|
568
|
+
score: float,
|
|
569
|
+
token_score: float,
|
|
570
|
+
) -> str:
|
|
571
|
+
return (
|
|
572
|
+
"Title differs by prefix/subtitle-like extra text. "
|
|
573
|
+
f"BibTeX: {local_title} | {source}: {remote_title} | "
|
|
574
|
+
f"score={score:.2f} | token_score={token_score:.2f}"
|
|
575
|
+
)
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def reference_issue(
|
|
579
|
+
entry: BibEntry,
|
|
580
|
+
severity: Severity,
|
|
581
|
+
code: str,
|
|
582
|
+
message: str,
|
|
583
|
+
evidence: str | None = None,
|
|
584
|
+
) -> Issue:
|
|
585
|
+
return Issue(
|
|
586
|
+
severity,
|
|
587
|
+
code,
|
|
588
|
+
message,
|
|
589
|
+
file=str(entry.file),
|
|
590
|
+
line=entry.line,
|
|
591
|
+
key=entry.key,
|
|
592
|
+
evidence=evidence,
|
|
593
|
+
)
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
def lookup_doi(doi: str, client: CatalogClient) -> tuple[str, str] | None:
|
|
597
|
+
try:
|
|
598
|
+
data = client.get_json(
|
|
599
|
+
f"https://api.crossref.org/works/{urllib.parse.quote(doi, safe='')}"
|
|
600
|
+
)
|
|
601
|
+
except LookupErrorWithStatus as exc:
|
|
602
|
+
if exc.status == 404:
|
|
603
|
+
return lookup_doi_resolver(doi, client)
|
|
604
|
+
raise
|
|
605
|
+
message = data.get("message")
|
|
606
|
+
title = first_string(message.get("title", []) if isinstance(message, dict) else [])
|
|
607
|
+
return ("Crossref", title or "")
|
|
608
|
+
|
|
609
|
+
|
|
610
|
+
def lookup_doi_resolver(doi: str, client: CatalogClient) -> tuple[str, str] | None:
|
|
611
|
+
try:
|
|
612
|
+
text = client.get_text(
|
|
613
|
+
f"https://doi.org/{urllib.parse.quote(doi, safe='/')}",
|
|
614
|
+
headers={"Accept": CSL_JSON_ACCEPT},
|
|
615
|
+
)
|
|
616
|
+
except LookupErrorWithStatus as exc:
|
|
617
|
+
if exc.status in {404, 410}:
|
|
618
|
+
return None
|
|
619
|
+
raise
|
|
620
|
+
try:
|
|
621
|
+
data = json.loads(text)
|
|
622
|
+
except json.JSONDecodeError:
|
|
623
|
+
return ("DOI resolver", "")
|
|
624
|
+
if not isinstance(data, dict):
|
|
625
|
+
return ("DOI resolver", "")
|
|
626
|
+
return ("DOI resolver", csl_title(data.get("title")) or "")
|
|
627
|
+
|
|
628
|
+
|
|
629
|
+
def lookup_arxiv(arxiv_id: str, client: CatalogClient) -> tuple[str, str] | None:
|
|
630
|
+
url = "https://export.arxiv.org/api/query?" + urllib.parse.urlencode(
|
|
631
|
+
{"id_list": arxiv_id}
|
|
632
|
+
)
|
|
633
|
+
root = ET.fromstring(client.get_text(url))
|
|
634
|
+
entry = root.find("{http://www.w3.org/2005/Atom}entry")
|
|
635
|
+
title = (
|
|
636
|
+
entry.findtext("{http://www.w3.org/2005/Atom}title")
|
|
637
|
+
if entry is not None
|
|
638
|
+
else None
|
|
639
|
+
)
|
|
640
|
+
return ("arXiv", clean_bib_value(title)) if title else None
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def extract_doi(entry: BibEntry) -> ExtractedIdentifier:
|
|
644
|
+
doi_field = clean_bib_value(entry.fields.get("doi", ""))
|
|
645
|
+
candidates = [doi_field] if doi_field else []
|
|
646
|
+
candidates.extend(
|
|
647
|
+
value
|
|
648
|
+
for field in ("url", "note", "eprint")
|
|
649
|
+
if (value := entry.fields.get(field, "")) and looks_like_doi_source(value)
|
|
650
|
+
)
|
|
651
|
+
for raw in candidates:
|
|
652
|
+
if normalized := normalize_doi(raw):
|
|
653
|
+
return ExtractedIdentifier(normalized)
|
|
654
|
+
return (
|
|
655
|
+
ExtractedIdentifier(None, candidates[0])
|
|
656
|
+
if candidates
|
|
657
|
+
else ExtractedIdentifier(None)
|
|
658
|
+
)
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
def looks_like_doi_source(raw: str) -> bool:
|
|
662
|
+
cleaned = clean_bib_value(raw)
|
|
663
|
+
return bool(
|
|
664
|
+
DOI_RE.search(cleaned)
|
|
665
|
+
or re.search(r"^(?:https?://)?(?:dx\.)?doi\.org/", cleaned, re.IGNORECASE)
|
|
666
|
+
or re.search(r"\bdoi\s*:", cleaned, re.IGNORECASE)
|
|
667
|
+
)
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def normalize_doi(raw: str) -> str | None:
|
|
671
|
+
cleaned = clean_bib_value(raw).strip()
|
|
672
|
+
cleaned = re.sub(
|
|
673
|
+
r"^(?:https?://)?(?:dx\.)?doi\.org/", "", cleaned, flags=re.IGNORECASE
|
|
674
|
+
)
|
|
675
|
+
cleaned = re.sub(r"^doi\s*:\s*", "", cleaned, flags=re.IGNORECASE)
|
|
676
|
+
match = DOI_RE.search(cleaned)
|
|
677
|
+
return match.group(0).rstrip(".,;:)]}").lower() if match else None
|
|
678
|
+
|
|
679
|
+
|
|
680
|
+
def is_arxiv_doi(doi: str) -> bool:
|
|
681
|
+
return doi.lower().startswith("10.48550/arxiv.")
|
|
682
|
+
|
|
683
|
+
|
|
684
|
+
def extract_arxiv_id(entry: BibEntry) -> ExtractedIdentifier:
|
|
685
|
+
eprint = clean_bib_value(entry.fields.get("eprint", ""))
|
|
686
|
+
archive = clean_bib_value(entry.fields.get("archiveprefix", "")).lower()
|
|
687
|
+
eprint_type = clean_bib_value(entry.fields.get("eprinttype", "")).lower()
|
|
688
|
+
candidates = (
|
|
689
|
+
[eprint] if eprint and (archive == "arxiv" or eprint_type == "arxiv") else []
|
|
690
|
+
)
|
|
691
|
+
candidates.extend(
|
|
692
|
+
value
|
|
693
|
+
for field in ("eprint", "url", "note", "doi")
|
|
694
|
+
if (value := entry.fields.get(field, "")) and looks_like_arxiv_source(value)
|
|
695
|
+
)
|
|
696
|
+
for raw in candidates:
|
|
697
|
+
if arxiv_id := normalize_arxiv_id(raw):
|
|
698
|
+
return ExtractedIdentifier(arxiv_id)
|
|
699
|
+
return (
|
|
700
|
+
ExtractedIdentifier(None, candidates[0])
|
|
701
|
+
if candidates
|
|
702
|
+
else ExtractedIdentifier(None)
|
|
703
|
+
)
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def looks_like_arxiv_source(raw: str) -> bool:
|
|
707
|
+
cleaned = clean_bib_value(raw)
|
|
708
|
+
lower = cleaned.lower()
|
|
709
|
+
return bool(
|
|
710
|
+
"arxiv.org/" in lower
|
|
711
|
+
or "10.48550/arxiv." in lower
|
|
712
|
+
or re.search(r"\barxiv\s*:", cleaned, re.IGNORECASE)
|
|
713
|
+
)
|
|
714
|
+
|
|
715
|
+
|
|
716
|
+
def normalize_arxiv_id(raw: str) -> str | None:
|
|
717
|
+
cleaned = re.sub(r"^arxiv\s*:\s*", "", clean_bib_value(raw), flags=re.IGNORECASE)
|
|
718
|
+
cleaned = re.sub(
|
|
719
|
+
r"^https?://arxiv\.org/(?:abs|pdf)/", "", cleaned, flags=re.IGNORECASE
|
|
720
|
+
).removesuffix(".pdf")
|
|
721
|
+
match = ARXIV_RE.search(cleaned)
|
|
722
|
+
if match:
|
|
723
|
+
return f"{match.group('id')}{match.group('version') or ''}"
|
|
724
|
+
doi_match = re.search(r"10\.48550/arxiv\.([0-9.]+)", cleaned, flags=re.IGNORECASE)
|
|
725
|
+
return doi_match.group(1) if doi_match else None
|
|
726
|
+
|
|
727
|
+
|
|
728
|
+
def run_slop_guard(
|
|
729
|
+
tex_files: Sequence[Path], command_override: str | None
|
|
730
|
+
) -> list[Issue]:
|
|
731
|
+
text = slop_guard_text(tex_files)
|
|
732
|
+
if not text:
|
|
733
|
+
return []
|
|
734
|
+
try:
|
|
735
|
+
completed = subprocess.run(
|
|
736
|
+
slop_guard_command(command_override),
|
|
737
|
+
input=text,
|
|
738
|
+
capture_output=True,
|
|
739
|
+
check=False,
|
|
740
|
+
text=True,
|
|
741
|
+
timeout=120.0,
|
|
742
|
+
)
|
|
743
|
+
if completed.returncode not in (0, 1):
|
|
744
|
+
raise RuntimeError(
|
|
745
|
+
normalize_space(completed.stderr or completed.stdout)[:500]
|
|
746
|
+
)
|
|
747
|
+
data = json.loads(completed.stdout)
|
|
748
|
+
if not isinstance(data, dict):
|
|
749
|
+
raise TypeError("Slop Guard JSON payload was not an object.")
|
|
750
|
+
except (
|
|
751
|
+
FileNotFoundError,
|
|
752
|
+
OSError,
|
|
753
|
+
RuntimeError,
|
|
754
|
+
TypeError,
|
|
755
|
+
subprocess.TimeoutExpired,
|
|
756
|
+
json.JSONDecodeError,
|
|
757
|
+
) as exc:
|
|
758
|
+
return [
|
|
759
|
+
Issue(
|
|
760
|
+
"warning",
|
|
761
|
+
"slop-guard-error",
|
|
762
|
+
"Slop Guard could not complete.",
|
|
763
|
+
evidence=str(exc),
|
|
764
|
+
)
|
|
765
|
+
]
|
|
766
|
+
|
|
767
|
+
score = as_float(data.get("score"))
|
|
768
|
+
return (
|
|
769
|
+
[
|
|
770
|
+
Issue(
|
|
771
|
+
"warning",
|
|
772
|
+
"slop-guard-signal",
|
|
773
|
+
"Slop Guard reported prose-lint findings. Treat as triage, not proof.",
|
|
774
|
+
evidence=slop_guard_evidence(score, data),
|
|
775
|
+
)
|
|
776
|
+
]
|
|
777
|
+
if score is not None and score < 80.0
|
|
778
|
+
else []
|
|
779
|
+
)
|
|
780
|
+
|
|
781
|
+
|
|
782
|
+
def slop_guard_text(tex_files: Sequence[Path]) -> str:
|
|
783
|
+
return normalize_space(
|
|
784
|
+
"\n\n".join(
|
|
785
|
+
latex_to_plain_text(
|
|
786
|
+
strip_latex_boilerplate(strip_latex_comments(read_text(path)))
|
|
787
|
+
)
|
|
788
|
+
for path in tex_files
|
|
789
|
+
)
|
|
790
|
+
)[:100_000]
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
def slop_guard_command(command_override: str | None) -> list[str]:
|
|
794
|
+
if command_override:
|
|
795
|
+
command = shlex.split(command_override)
|
|
796
|
+
if not command:
|
|
797
|
+
raise FileNotFoundError("--slop-guard-command was empty")
|
|
798
|
+
return [*command, "-j", "-"]
|
|
799
|
+
if installed := shutil.which("sg"):
|
|
800
|
+
return [installed, "-j", "-"]
|
|
801
|
+
if uvx := shutil.which("uvx"):
|
|
802
|
+
return [uvx, "--from", "slop-guard", "sg", "-j", "-"]
|
|
803
|
+
raise FileNotFoundError("Install `sg` or `uvx` as an available shell command.")
|
|
804
|
+
|
|
805
|
+
|
|
806
|
+
def slop_guard_evidence(score: float, data: dict[str, object]) -> str:
|
|
807
|
+
band = data.get("band") if isinstance(data.get("band"), str) else "unknown"
|
|
808
|
+
parts = [f"score={score:.0f}", f"band={band}"]
|
|
809
|
+
violations = data.get("violations")
|
|
810
|
+
if isinstance(violations, list):
|
|
811
|
+
hits = []
|
|
812
|
+
for item in violations[:5]:
|
|
813
|
+
if not isinstance(item, dict):
|
|
814
|
+
continue
|
|
815
|
+
rule = item.get("rule")
|
|
816
|
+
match = item.get("match")
|
|
817
|
+
if isinstance(rule, str) and isinstance(match, str):
|
|
818
|
+
hits.append(f"{rule}: {normalize_space(match)[:80]}")
|
|
819
|
+
if hits:
|
|
820
|
+
parts.append("hits=" + "; ".join(hits))
|
|
821
|
+
return " | ".join(parts)
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def render_report(report: Report, report_format: str) -> str:
|
|
825
|
+
if report_format == "json":
|
|
826
|
+
data = dataclasses.asdict(report)
|
|
827
|
+
data["counts"] = {
|
|
828
|
+
"hard": sum(issue.severity == "hard" for issue in report.issues),
|
|
829
|
+
"warning": sum(issue.severity == "warning" for issue in report.issues),
|
|
830
|
+
"checked": len(report.checked_keys),
|
|
831
|
+
}
|
|
832
|
+
return json.dumps(data, indent=2, sort_keys=True)
|
|
833
|
+
return render_markdown(report)
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
def render_markdown(report: Report) -> str:
|
|
837
|
+
hard = [issue for issue in report.issues if issue.severity == "hard"]
|
|
838
|
+
warnings = [issue for issue in report.issues if issue.severity == "warning"]
|
|
839
|
+
lines = [
|
|
840
|
+
"# Paper LLM-Proof Report",
|
|
841
|
+
"",
|
|
842
|
+
f"- Project: `{report.project}`",
|
|
843
|
+
f"- TeX files: {len(report.tex_files)}",
|
|
844
|
+
f"- Bib files: {len(report.bib_files)}",
|
|
845
|
+
f"- Checked references: {len(report.checked_keys)}",
|
|
846
|
+
f"- Hard failures: {len(hard)}",
|
|
847
|
+
f"- Warnings: {len(warnings)}",
|
|
848
|
+
f"- Slop Guard: {'requested' if report.slop_guard_requested else 'not requested'}",
|
|
849
|
+
"",
|
|
850
|
+
]
|
|
851
|
+
lines.extend(render_issue_section("Hard Failures", hard))
|
|
852
|
+
lines.extend(render_issue_section("Warnings", warnings))
|
|
853
|
+
return "\n".join(lines)
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def render_issue_section(title: str, issues: Sequence[Issue]) -> list[str]:
|
|
857
|
+
lines = [f"## {title}", ""]
|
|
858
|
+
if not issues:
|
|
859
|
+
return [*lines, "None.", ""]
|
|
860
|
+
for issue in sorted(
|
|
861
|
+
issues, key=lambda item: (item.file or "", item.line or 0, item.code)
|
|
862
|
+
):
|
|
863
|
+
location = f" `{issue.file}`" if issue.file else ""
|
|
864
|
+
location += f":{issue.line}" if issue.file and issue.line else ""
|
|
865
|
+
key = f" `{issue.key}`" if issue.key else ""
|
|
866
|
+
lines.append(f"- `{issue.code}`{key}{location}: {issue.message}")
|
|
867
|
+
if issue.evidence:
|
|
868
|
+
lines.append(f" Evidence: {issue.evidence}")
|
|
869
|
+
return [*lines, ""]
|
|
870
|
+
|
|
871
|
+
|
|
872
|
+
def read_text(path: Path) -> str:
|
|
873
|
+
try:
|
|
874
|
+
return path.read_text(encoding="utf-8")
|
|
875
|
+
except UnicodeDecodeError:
|
|
876
|
+
return path.read_text(encoding="latin-1")
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def strip_latex_comments(text: str) -> str:
|
|
880
|
+
return "".join(
|
|
881
|
+
strip_latex_comment_line(line) for line in text.splitlines(keepends=True)
|
|
882
|
+
)
|
|
883
|
+
|
|
884
|
+
|
|
885
|
+
def strip_latex_comment_line(line: str) -> str:
|
|
886
|
+
for index, char in enumerate(line):
|
|
887
|
+
if char == "%" and (index == 0 or line[index - 1] != "\\"):
|
|
888
|
+
return line[:index] + ("\n" if line.endswith("\n") else "")
|
|
889
|
+
return line
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def strip_latex_boilerplate(text: str) -> str:
|
|
893
|
+
text = re.sub(r"\\documentclass(?:\s*\[[^\]]*\])?\s*\{[^{}]*\}", " ", text)
|
|
894
|
+
text = re.sub(r"\\(?:begin|end)\s*\{document\}", " ", text)
|
|
895
|
+
return re.sub(
|
|
896
|
+
r"\\(?:bibliography|bibliographystyle|addbibresource)(?:\s*\[[^\]]*\])?\s*\{[^{}]*\}",
|
|
897
|
+
" ",
|
|
898
|
+
text,
|
|
899
|
+
)
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
def latex_to_plain_text(text: str) -> str:
|
|
903
|
+
text = re.sub(
|
|
904
|
+
r"\\[A-Za-z]*[Cc]ite[A-Za-z]*\*?(?:\s*\[[^\]]*\]){0,3}\s*\{[^{}]*\}",
|
|
905
|
+
" [CITATION] ",
|
|
906
|
+
text,
|
|
907
|
+
)
|
|
908
|
+
text = re.sub(r"\\(?:ref|eqref|label)\s*\{[^{}]*\}", " ", text)
|
|
909
|
+
text = re.sub(r"\\(?:url|href)\s*\{([^{}]*)\}", r" \1 ", text)
|
|
910
|
+
text = re.sub(r"\\[a-zA-Z]+\*?(?:\s*\[[^\]]*\])?", " ", text)
|
|
911
|
+
return normalize_space(
|
|
912
|
+
re.sub(
|
|
913
|
+
r"[{}$^_~]",
|
|
914
|
+
" ",
|
|
915
|
+
text.replace("\\&", "&").replace("\\%", "%").replace("\\_", "_"),
|
|
916
|
+
)
|
|
917
|
+
)
|
|
918
|
+
|
|
919
|
+
|
|
920
|
+
# Boundary: title comparison is deliberately not a LaTeX interpreter. It handles
|
|
921
|
+
# obvious plain text and simple formatting macros; anything still command-like
|
|
922
|
+
# becomes an agent-review warning after the DOI/arXiv identifier resolves.
|
|
923
|
+
def catalog_title(text: str | None) -> CatalogTitle:
|
|
924
|
+
raw = normalize_space(text or "")
|
|
925
|
+
if not raw:
|
|
926
|
+
return CatalogTitle(raw, "", "BibTeX title is empty.")
|
|
927
|
+
cleaned = unwrap_simple_latex_text(raw)
|
|
928
|
+
commands = remaining_latex_commands(cleaned)
|
|
929
|
+
if commands:
|
|
930
|
+
return CatalogTitle(
|
|
931
|
+
raw,
|
|
932
|
+
"",
|
|
933
|
+
"BibTeX title contains LaTeX command(s) the checker does not interpret: "
|
|
934
|
+
+ ", ".join(commands[:5]),
|
|
935
|
+
)
|
|
936
|
+
cleaned = strip_bibtex_markup(cleaned)
|
|
937
|
+
if not cleaned:
|
|
938
|
+
return CatalogTitle(raw, "", "BibTeX title did not leave comparable text.")
|
|
939
|
+
return CatalogTitle(raw, cleaned)
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
def clean_bib_value(text: str | None) -> str:
|
|
943
|
+
if text is None:
|
|
944
|
+
return ""
|
|
945
|
+
text = unwrap_simple_latex_text(text)
|
|
946
|
+
text = re.sub(r"\\[a-zA-Z]+\*?", " ", text)
|
|
947
|
+
return strip_bibtex_markup(text)
|
|
948
|
+
|
|
949
|
+
|
|
950
|
+
def unwrap_simple_latex_text(text: str) -> str:
|
|
951
|
+
text = re.sub(r"\\['`^\"~=.]\s*\{?([A-Za-z])\}?", r"\1", text)
|
|
952
|
+
text = re.sub(r"\\[uvHcdbtr](?=\s|\{)\s*\{?([A-Za-z])\}?", r"\1", text)
|
|
953
|
+
text = re.sub(
|
|
954
|
+
r"\\(?:href|hyperref)\*?(?:\s*\[[^\]]*\])?\s*\{[^{}]*\}\s*\{([^{}]*)\}",
|
|
955
|
+
r"\1",
|
|
956
|
+
text,
|
|
957
|
+
)
|
|
958
|
+
while True:
|
|
959
|
+
updated = re.sub(
|
|
960
|
+
r"\\[a-zA-Z]+\*?(?:\s*\[[^\]]*\])?\s*\{([^{}]*)\}",
|
|
961
|
+
r"\1",
|
|
962
|
+
text,
|
|
963
|
+
)
|
|
964
|
+
if updated == text:
|
|
965
|
+
break
|
|
966
|
+
text = updated
|
|
967
|
+
return text
|
|
968
|
+
|
|
969
|
+
|
|
970
|
+
def remaining_latex_commands(text: str) -> tuple[str, ...]:
|
|
971
|
+
return tuple(
|
|
972
|
+
sorted({match.group(0) for match in re.finditer(r"\\[a-zA-Z]+\*?", text)})
|
|
973
|
+
)
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def strip_bibtex_markup(text: str) -> str:
|
|
977
|
+
return normalize_space(
|
|
978
|
+
text.replace("{", "")
|
|
979
|
+
.replace("}", "")
|
|
980
|
+
.replace("\\&", "&")
|
|
981
|
+
.replace("\\_", "_")
|
|
982
|
+
.replace("\\%", "%")
|
|
983
|
+
)
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
def normalize_space(text: str) -> str:
|
|
987
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def normalize_for_match(text: str) -> str:
|
|
991
|
+
return normalize_space(re.sub(r"[^a-z0-9]+", " ", text.lower()))
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
def title_similarity(left: str, right: str) -> float:
|
|
995
|
+
left_norm = normalize_for_match(left)
|
|
996
|
+
right_norm = normalize_for_match(right)
|
|
997
|
+
if not left_norm or not right_norm:
|
|
998
|
+
return 0.0
|
|
999
|
+
return difflib.SequenceMatcher(None, left_norm, right_norm).ratio()
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def title_token_similarity(left: str, right: str) -> float:
|
|
1003
|
+
left_tokens = set(significant_title_tokens(left))
|
|
1004
|
+
right_tokens = set(significant_title_tokens(right))
|
|
1005
|
+
if not left_tokens or not right_tokens:
|
|
1006
|
+
return 0.0
|
|
1007
|
+
return len(left_tokens & right_tokens) / len(left_tokens | right_tokens)
|
|
1008
|
+
|
|
1009
|
+
|
|
1010
|
+
# Boundary: prefix/subtitle variants are not clear contradictions. They still
|
|
1011
|
+
# need an agent check, but they should not block like different titles do.
|
|
1012
|
+
def title_prefix_variant(left: str, right: str) -> bool:
|
|
1013
|
+
left_tokens = tuple(normalize_for_match(left).split())
|
|
1014
|
+
right_tokens = tuple(normalize_for_match(right).split())
|
|
1015
|
+
if not left_tokens or not right_tokens or left_tokens == right_tokens:
|
|
1016
|
+
return False
|
|
1017
|
+
shorter, longer = (
|
|
1018
|
+
(left_tokens, right_tokens)
|
|
1019
|
+
if len(left_tokens) < len(right_tokens)
|
|
1020
|
+
else (right_tokens, left_tokens)
|
|
1021
|
+
)
|
|
1022
|
+
return longer[: len(shorter)] == shorter
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
def significant_title_tokens(text: str) -> tuple[str, ...]:
|
|
1026
|
+
tokens = tuple(
|
|
1027
|
+
token
|
|
1028
|
+
for token in normalize_for_match(text).split()
|
|
1029
|
+
if token not in TITLE_STOPWORDS
|
|
1030
|
+
)
|
|
1031
|
+
return tokens or tuple(normalize_for_match(text).split())
|
|
1032
|
+
|
|
1033
|
+
|
|
1034
|
+
def line_for_offset(text: str, offset: int) -> int:
|
|
1035
|
+
return text.count("\n", 0, offset) + 1
|
|
1036
|
+
|
|
1037
|
+
|
|
1038
|
+
def first_string(values: object) -> str | None:
|
|
1039
|
+
if not isinstance(values, list):
|
|
1040
|
+
return None
|
|
1041
|
+
for value in values:
|
|
1042
|
+
if isinstance(value, str) and value.strip():
|
|
1043
|
+
return clean_bib_value(value)
|
|
1044
|
+
return None
|
|
1045
|
+
|
|
1046
|
+
|
|
1047
|
+
def csl_title(value: object) -> str | None:
|
|
1048
|
+
if isinstance(value, str) and value.strip():
|
|
1049
|
+
return clean_bib_value(value)
|
|
1050
|
+
return first_string(value)
|
|
1051
|
+
|
|
1052
|
+
|
|
1053
|
+
def as_float(value: object) -> float | None:
|
|
1054
|
+
if isinstance(value, (int, float)):
|
|
1055
|
+
return float(value)
|
|
1056
|
+
if isinstance(value, str):
|
|
1057
|
+
try:
|
|
1058
|
+
return float(value)
|
|
1059
|
+
except ValueError:
|
|
1060
|
+
return None
|
|
1061
|
+
return None
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
if __name__ == "__main__":
|
|
1065
|
+
raise SystemExit(main())
|