@aiwg/cli 2026.10.15 → 2026.10.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/providers/pi/aiwg-bridge.ts +45 -7
- package/dist/src/artifacts/citation-parser.js +7 -6
- package/dist/src/artifacts/corpus-tools/citation-densify.js +4 -2
- package/dist/src/artifacts/corpus-tools/cli.js +8 -3
- package/dist/src/artifacts/corpus-tools/discovery-log.js +26 -24
- package/dist/src/artifacts/corpus-tools/expand-cli.js +3 -2
- package/dist/src/artifacts/corpus-tools/expand-dates.js +107 -0
- package/dist/src/artifacts/corpus-tools/expand-queue.js +4 -1
- package/dist/src/artifacts/corpus-tools/expand-review.js +104 -0
- package/dist/src/artifacts/corpus-tools/expand-state.js +17 -1
- package/dist/src/artifacts/corpus-tools/expand.js +16 -17
- package/dist/src/artifacts/corpus-tools/induction-audit.js +7 -6
- package/dist/src/artifacts/corpus-tools/integrity-scan.js +3 -3
- package/dist/src/artifacts/corpus-tools/radar-shared.js +7 -9
- package/dist/src/artifacts/corpus-tools/ref-files.js +11 -0
- package/dist/src/artifacts/corpus-tools/resolve-cli.js +11 -3
- package/dist/src/artifacts/corpus-tools/resolve.js +42 -11
- package/dist/src/artifacts/corpus-tools/retrieval-lab.js +2 -1
- package/dist/src/artifacts/corpus-tools/sidecar-lint.js +4 -4
- package/dist/src/artifacts/corpus-tools/sidecar-repair.js +2 -1
- package/dist/src/artifacts/corpus-views/ref-parser.js +3 -17
- package/dist/src/artifacts/graph-query.js +5 -3
- package/dist/src/artifacts/hybrid-query.js +11 -2
- package/dist/src/artifacts/index-refresh.js +2 -0
- package/dist/src/artifacts/query-engine-corpus.js +217 -0
- package/dist/src/artifacts/query-engine-discover-output.js +229 -0
- package/dist/src/artifacts/query-engine-discover.js +285 -0
- package/dist/src/artifacts/query-engine-entries.js +188 -0
- package/dist/src/artifacts/query-engine-index.js +249 -0
- package/dist/src/artifacts/query-engine-matching.js +143 -0
- package/dist/src/artifacts/query-engine-options.js +2 -0
- package/dist/src/artifacts/query-engine-rules.js +202 -0
- package/dist/src/artifacts/query-engine-scoring.js +436 -0
- package/dist/src/artifacts/query-engine-show-card.js +194 -0
- package/dist/src/artifacts/query-engine-show-load.js +211 -0
- package/dist/src/artifacts/query-engine-show.js +300 -0
- package/dist/src/artifacts/query-engine.js +7 -2576
- package/dist/src/cli/handlers/daemon.js +17 -1
- package/dist/src/cli/handlers/doctor-args.js +10 -5
- package/dist/src/cli/handlers/doctor-help.js +4 -1
- package/dist/src/cli/handlers/index.js +2 -2
- package/dist/src/cli/handlers/mc-agents.js +111 -0
- package/dist/src/cli/handlers/mc-args.js +54 -0
- package/dist/src/cli/handlers/mc-control.js +227 -0
- package/dist/src/cli/handlers/mc-dispatch.js +203 -0
- package/dist/src/cli/handlers/mc-help.js +171 -0
- package/dist/src/cli/handlers/mc-run.js +343 -0
- package/dist/src/cli/handlers/mc-session.js +138 -0
- package/dist/src/cli/handlers/mc-status.js +258 -0
- package/dist/src/cli/handlers/mc-summary.js +43 -0
- package/dist/src/cli/handlers/mc-types.js +3 -0
- package/dist/src/cli/handlers/mc-worker.js +205 -0
- package/dist/src/cli/handlers/mc.js +12 -1665
- package/dist/src/cli/handlers/models-refusal.js +13 -0
- package/dist/src/cli/handlers/models-resource-sample.js +10 -0
- package/dist/src/cli/handlers/models.js +5 -3
- package/dist/src/cli/handlers/regenerate.js +8 -16
- package/dist/src/cli/handlers/remove-bundled.js +133 -0
- package/dist/src/cli/handlers/remove-options.js +31 -0
- package/dist/src/cli/handlers/remove-user.js +174 -0
- package/dist/src/cli/handlers/remove.js +201 -0
- package/dist/src/cli/handlers/sessions.js +7 -7
- package/dist/src/cli/handlers/subcommands-bundles.js +257 -0
- package/dist/src/cli/handlers/subcommands-discovery.js +165 -0
- package/dist/src/cli/handlers/subcommands-list-format.js +129 -0
- package/dist/src/cli/handlers/subcommands-list.js +250 -0
- package/dist/src/cli/handlers/subcommands-plugins.js +213 -0
- package/dist/src/cli/handlers/subcommands-quickref.js +58 -0
- package/dist/src/cli/handlers/subcommands-rlm.js +113 -0
- package/dist/src/cli/handlers/subcommands-services.js +66 -0
- package/dist/src/cli/handlers/subcommands-storage.js +259 -0
- package/dist/src/cli/handlers/subcommands.js +18 -1976
- package/dist/src/cli/handlers/update.js +157 -0
- package/dist/src/cli/handlers/use.js +14 -4
- package/dist/src/cli/handlers/utilities.js +3 -560
- package/dist/src/cli/handlers/utility-doctor.js +412 -0
- package/dist/src/cli/scope-resolver.js +1 -142
- package/dist/src/cli/services/deployment-verification.js +0 -1
- package/dist/src/cli/user-scope-mirror.js +173 -0
- package/dist/src/config/aiwg-config.js +2 -1
- package/dist/src/config/authorization-cli.js +7 -3
- package/dist/src/config/dispatch-host-thresholds.js +2 -0
- package/dist/src/config/dispatch-thresholds-runtime.d.mts +3 -2
- package/dist/src/config/dispatch-thresholds-runtime.mjs +21 -7
- package/dist/src/config/framework-aliases.js +12 -0
- package/dist/src/config/host-resource-report.d.mts +24 -0
- package/dist/src/config/host-resource-report.mjs +110 -0
- package/dist/src/config/host-resource-runtime.d.mts +37 -0
- package/dist/src/config/host-resource-runtime.mjs +296 -0
- package/dist/src/config/installed-selection.js +22 -0
- package/dist/src/config/worker-invocation-marker.d.mts +1 -0
- package/dist/src/config/worker-invocation-marker.mjs +2 -0
- package/dist/src/decision/ensemble-study/artifacts.js +7 -0
- package/dist/src/decision/ensemble-study/gate-policy.js +58 -0
- package/dist/src/decision/ensemble-study/gates.js +78 -0
- package/dist/src/decision/ensemble-study/protocol.js +18 -21
- package/dist/src/decision/ensemble-study/score.js +3 -2
- package/dist/src/decision/ensemble-study/statistics.js +7 -26
- package/dist/src/decision/heldout/access.js +74 -0
- package/dist/src/decision/heldout/calibration.js +1 -0
- package/dist/src/decision/heldout/collector.js +8 -1
- package/dist/src/decision/sensitivity/comparative-gates.js +135 -0
- package/dist/src/decision/sensitivity/comparative.js +18 -41
- package/dist/src/effects/verifiers/file.js +22 -5
- package/dist/src/extensions/claude-attribution.js +82 -0
- package/dist/src/extensions/claude-hooks-installer.js +133 -69
- package/dist/src/extensions/commands/daemon.js +109 -0
- package/dist/src/extensions/commands/definitions.js +5 -89
- package/dist/src/gates/evaluate.js +2 -358
- package/dist/src/gates/heldout.js +17 -7
- package/dist/src/gates/observations.js +363 -0
- package/dist/src/gates/pack-composition.js +270 -0
- package/dist/src/gates/pack-tightening.js +197 -0
- package/dist/src/gates/pack-validation.js +262 -0
- package/dist/src/gates/registry-error.js +5 -0
- package/dist/src/gates/registry.js +13 -685
- package/dist/src/lint/check-patterns.js +163 -0
- package/dist/src/lint/checks.js +268 -0
- package/dist/src/lint/reference-resolution.js +54 -0
- package/dist/src/lint/runner.js +3 -465
- package/dist/src/mcp/helpers.mjs +11 -1
- package/dist/src/mcp/json-document.mjs +29 -0
- package/dist/src/mcp/muse-native-config.mjs +1 -28
- package/dist/src/memory/project-registry.js +44 -4
- package/dist/src/mission-protocol/worker-review-gate.js +7 -0
- package/dist/src/models/worker-plan.js +36 -5
- package/dist/src/models/wrapper-deployment.js +25 -12
- package/dist/src/models/wrapper-enforcement.js +47 -7
- package/dist/src/network-analysis/research.js +2 -1
- package/dist/src/plugin/command-ownership.js +34 -0
- package/dist/src/plugin/skill-command-frontmatter.js +197 -0
- package/dist/src/plugin/skill-command-translator.js +6 -232
- package/dist/src/policy/hitl-check.js +14 -4
- package/dist/src/policy/hitl-consume.js +54 -0
- package/dist/src/policy/hitl-receipt.js +2 -0
- package/dist/src/policy/hitl.js +11 -3
- package/dist/src/providers/context-output-ownership.js +26 -0
- package/dist/src/providers/deployment-ownership.d.mts +19 -0
- package/dist/src/providers/deployment-ownership.mjs +226 -0
- package/dist/src/providers/deployment-removal.js +200 -0
- package/dist/src/providers/provider-context-contracts.js +162 -0
- package/dist/src/providers/provider-definition-schema.js +136 -0
- package/dist/src/providers/provider-definitions.js +8 -1298
- package/dist/src/providers/provider-seeds-1.js +360 -0
- package/dist/src/providers/provider-seeds-2.js +387 -0
- package/dist/src/providers/provider-seeds-3.js +262 -0
- package/dist/src/providers/removal-transaction.js +86 -0
- package/dist/src/providers/settings-document.d.mts +3 -0
- package/dist/src/providers/settings-document.mjs +47 -0
- package/dist/src/providers/settings-removal.js +190 -0
- package/dist/src/providers/skill-ownership.d.mts +19 -2
- package/dist/src/providers/skill-ownership.mjs +12 -4
- package/dist/src/providers/transformation-receipt-integration.js +15 -271
- package/dist/src/providers/transformation-receipt-outputs.js +99 -0
- package/dist/src/providers/transformation-receipt-source.js +194 -0
- package/dist/src/research/clients/arxiv.js +4 -30
- package/dist/src/research/clients/base.js +48 -22
- package/dist/src/research/identity.js +42 -4
- package/dist/src/research/query-cli.js +3 -87
- package/dist/src/research/query-options.js +87 -0
- package/dist/src/research/reference-match.js +105 -24
- package/dist/src/research/services/acquisition.js +9 -7
- package/dist/src/research/services/citation.js +5 -2
- package/dist/src/research/types.js +2 -0
- package/dist/src/rules/activation.mjs +15 -6
- package/dist/src/rules/core-startup.mjs +2 -2
- package/dist/src/serve/mission-conductor.js +52 -4
- package/dist/src/serve/mission-summary.js +56 -0
- package/dist/src/sessions/adapters/grok-build-native.js +23 -0
- package/dist/src/sessions/adapters/grok-build.js +14 -7
- package/dist/src/sessions/workspace-discovery.js +5 -5
- package/dist/src/smiths/context-pipeline/aiwg-md.js +39 -46
- package/dist/src/smiths/context-pipeline/claude-hook.js +1 -1
- package/dist/src/smiths/context-pipeline/code-regions.js +28 -0
- package/dist/src/smiths/context-pipeline/context-file-safety.js +3 -0
- package/dist/src/smiths/context-pipeline/finalization.js +81 -51
- package/dist/src/smiths/context-pipeline/generator.js +10 -11
- package/dist/src/smiths/context-pipeline/managed-hook.js +1 -1
- package/dist/src/smiths/context-pipeline/muse-bridge.js +2 -2
- package/dist/src/smiths/context-pipeline/provider-policy.js +6 -12
- package/dist/src/smiths/context-pipeline/workspace-audit.js +299 -0
- package/dist/src/smiths/context-pipeline/workspace-bootstrap.js +148 -0
- package/dist/src/smiths/context-pipeline/workspace-context-shared.js +111 -0
- package/dist/src/smiths/context-pipeline/workspace-context.js +8 -1373
- package/dist/src/smiths/context-pipeline/workspace-doctor.js +192 -0
- package/dist/src/smiths/context-pipeline/workspace-document.js +201 -0
- package/dist/src/smiths/context-pipeline/workspace-extraction.js +205 -0
- package/dist/src/smiths/context-pipeline/workspace-migration.js +250 -0
- package/dist/src/smiths/hook-bridge/index.js +9 -0
- package/package.json +13 -1
- package/schemas/decision/D17StudyProtocol.v1.schema.json +39 -67
- package/schemas/decision/D29Study.v10.schema.json +65 -0
- package/tools/agents/deploy-agents.mjs +24 -675
- package/tools/agents/deploy-command-mirrors.mjs +229 -0
- package/tools/agents/deploy-options.mjs +258 -0
- package/tools/agents/deploy-support.mjs +165 -0
- package/tools/agents/providers/base-addon-discovery.mjs +139 -0
- package/tools/agents/providers/base-behaviors.mjs +168 -0
- package/tools/agents/providers/base-context.mjs +205 -0
- package/tools/agents/providers/base-contract.mjs +116 -0
- package/tools/agents/providers/base-files.mjs +252 -0
- package/tools/agents/providers/base-flat-artifacts.mjs +233 -0
- package/tools/agents/providers/base-framework-discovery.mjs +309 -0
- package/tools/agents/providers/base-frontmatter.mjs +196 -0
- package/tools/agents/providers/base-inventory.mjs +269 -0
- package/tools/agents/providers/base-kernel-skills.mjs +354 -0
- package/tools/agents/providers/base-migrations.mjs +258 -0
- package/tools/agents/providers/base-models-platforms.mjs +273 -0
- package/tools/agents/providers/base-rule-index.mjs +450 -0
- package/tools/agents/providers/base-runtime.mjs +60 -0
- package/tools/agents/providers/base-skill-copy.mjs +273 -0
- package/tools/agents/providers/base.mjs +16 -3364
- package/tools/agents/providers/claude-post-deploy.mjs +20 -1
- package/tools/agents/providers/context-write.mjs +4 -0
- package/tools/agents/providers/hermes-startup.mjs +0 -5
- package/tools/agents/providers/muse.mjs +7 -2
- package/tools/agents/providers/pi.mjs +3 -1
- package/tools/agents/providers/rule-delivery-surface.mjs +359 -0
- package/tools/agents/providers/rule-delivery.mjs +20 -1144
- package/tools/agents/providers/rule-deployment-commit.mjs +148 -0
- package/tools/agents/providers/rule-deployment-prepare.mjs +461 -0
- package/tools/agents/providers/rule-deployment-state.mjs +300 -0
- package/tools/agents/providers/rule-projection.mjs +3 -0
- package/tools/daemon/README.md +511 -0
- package/tools/daemon/adapters/base-adapter.mjs +84 -0
- package/tools/daemon/adapters/webhook-adapter.mjs +107 -0
- package/tools/daemon/agent-supervisor.mjs +299 -0
- package/tools/daemon/automation-engine.mjs +310 -0
- package/tools/daemon/autonomous-engine.mjs +346 -0
- package/tools/daemon/behavior-loader.mjs +316 -0
- package/tools/daemon/behavior.mjs +174 -0
- package/tools/daemon/concierge/intent-router.mjs +401 -0
- package/tools/daemon/concierge/orchestrator.mjs +272 -0
- package/tools/daemon/concierge/response-translator.mjs +462 -0
- package/tools/daemon/config.mjs +377 -0
- package/tools/daemon/cron-scheduler.mjs +178 -0
- package/tools/daemon/daemon-main.mjs +1169 -0
- package/tools/daemon/event-router.mjs +84 -0
- package/tools/daemon/file-watcher.mjs +383 -0
- package/tools/daemon/index.mjs +1181 -0
- package/tools/daemon/init-profile.mjs +124 -0
- package/tools/daemon/ipc-client.mjs +197 -0
- package/tools/daemon/ipc-ops-handlers.mjs +236 -0
- package/tools/daemon/ipc-server.mjs +275 -0
- package/tools/daemon/maintenance-owner.mjs +79 -0
- package/tools/daemon/memory-manager.mjs +386 -0
- package/tools/daemon/message-router.mjs +194 -0
- package/tools/daemon/provider-watcher.mjs +585 -0
- package/tools/daemon/pty-adapter.mjs +420 -0
- package/tools/daemon/repl-chat.mjs +359 -0
- package/tools/daemon/resource-bounds.mjs +88 -0
- package/tools/daemon/sandbox-transport.mjs +295 -0
- package/tools/daemon/scheduled-task-runner.mjs +535 -0
- package/tools/daemon/task-store.mjs +450 -0
- package/tools/daemon/tmux-manager.mjs +324 -0
- package/tools/daemon/update-policy.mjs +517 -0
- package/tools/daemon/web-server.mjs +399 -0
- package/tools/daemon/web-ui/index.html +585 -0
- package/tools/daemon/web-ui/styles.css +156 -0
- package/tools/docker/Dockerfile.daemon +35 -0
- package/tools/docker/container-manager.mjs +291 -0
- package/tools/messaging/adapter-registry.mjs +230 -0
- package/tools/messaging/adapters/base.mjs +326 -0
- package/tools/messaging/adapters/discord.mjs +462 -0
- package/tools/messaging/adapters/slack.mjs +402 -0
- package/tools/messaging/adapters/telegram.mjs +552 -0
- package/tools/messaging/chat-handler.mjs +297 -0
- package/tools/messaging/command-router.mjs +269 -0
- package/tools/messaging/event-bus.mjs +199 -0
- package/tools/messaging/index.mjs +456 -0
- package/tools/messaging/message-formatter.mjs +320 -0
- package/tools/messaging/room-manager.mjs +277 -0
- package/tools/messaging/types.mjs +94 -0
- package/tools/providers/pi-bridge-policy.mjs +4 -0
- package/tools/ralph-external/daemon-supervisor.mjs +394 -0
- package/tools/ralph-external/lib/behavior-registry.mjs +270 -0
- package/tools/ralph-external/lib/worker-invocation-processes.mjs +7 -4
- package/tools/ralph-external/lib/worker-isolation-args.mjs +217 -0
- package/tools/ralph-external/lib/worker-isolation-status.mjs +19 -0
- package/tools/ralph-external/lib/worker-isolation.mjs +11 -237
- package/tools/ralph-external/lib/worker-overflow.mjs +10 -2
- package/tools/ralph-external/lib/worker-termination.mjs +15 -8
- package/tools/ralph-external/lib/worker-unit-cleanup.mjs +72 -0
- package/tools/ralph-external/session-launcher.mjs +7 -7
- package/tools/scripts/resource-preflight.mjs +3 -242
- package/vscode-extension/schemas/aiwg.config.v1.json +2 -4
|
@@ -4,6 +4,8 @@
|
|
|
4
4
|
* @module research/clients/base
|
|
5
5
|
*/
|
|
6
6
|
import { ResearchError, ResearchErrorCode, } from '../types.js';
|
|
7
|
+
// Bound server-directed cooldowns for both retries and later shared-client requests.
|
|
8
|
+
const MAX_RETRY_AFTER_MS = 30_000;
|
|
7
9
|
/**
|
|
8
10
|
* Token bucket rate limiter
|
|
9
11
|
*/
|
|
@@ -11,6 +13,8 @@ export class RateLimiter {
|
|
|
11
13
|
config;
|
|
12
14
|
tokens;
|
|
13
15
|
lastRefill;
|
|
16
|
+
blockedUntil = 0;
|
|
17
|
+
queue = Promise.resolve();
|
|
14
18
|
constructor(config) {
|
|
15
19
|
this.config = config;
|
|
16
20
|
this.tokens = config.currentTokens;
|
|
@@ -29,19 +33,25 @@ export class RateLimiter {
|
|
|
29
33
|
/**
|
|
30
34
|
* Acquire a token, waiting if necessary
|
|
31
35
|
*/
|
|
32
|
-
|
|
33
|
-
this.
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
36
|
+
acquire() {
|
|
37
|
+
const pending = this.queue.then(() => this.acquireToken());
|
|
38
|
+
this.queue = pending.catch(() => { });
|
|
39
|
+
return pending;
|
|
40
|
+
}
|
|
41
|
+
/** Share a server cooldown with subsequent requests, including exhausted retries. */
|
|
42
|
+
defer(ms) {
|
|
43
|
+
this.blockedUntil = Math.max(this.blockedUntil, Date.now() + ms);
|
|
44
|
+
}
|
|
45
|
+
async acquireToken() {
|
|
46
|
+
for (;;) {
|
|
47
|
+
this.refill();
|
|
48
|
+
const waitMs = Math.max(this.blockedUntil - Date.now(), (1 - this.tokens) / this.config.refillRate * 1000);
|
|
49
|
+
if (waitMs <= 0) {
|
|
50
|
+
this.tokens -= 1;
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
await this.sleep(Math.ceil(waitMs));
|
|
37
54
|
}
|
|
38
|
-
// Calculate wait time
|
|
39
|
-
const deficit = 1 - this.tokens;
|
|
40
|
-
const waitMs = (deficit / this.config.refillRate) * 1000;
|
|
41
|
-
await this.sleep(waitMs);
|
|
42
|
-
// Refill again after waiting
|
|
43
|
-
this.refill();
|
|
44
|
-
this.tokens -= 1;
|
|
45
55
|
}
|
|
46
56
|
sleep(ms) {
|
|
47
57
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -73,10 +83,14 @@ export class BaseClient {
|
|
|
73
83
|
// Execute with retry
|
|
74
84
|
return this.executeWithRetry(url, options);
|
|
75
85
|
}
|
|
86
|
+
/** XML clients use the same limiter, timeout and HTTP retry policy as JSON. */
|
|
87
|
+
requestText(url, options = {}) {
|
|
88
|
+
return this.executeWithRetry(url, options, 0, undefined, response => response.text());
|
|
89
|
+
}
|
|
76
90
|
/**
|
|
77
91
|
* Execute request with exponential backoff retry
|
|
78
92
|
*/
|
|
79
|
-
async executeWithRetry(url, options, attempt = 0, firstError) {
|
|
93
|
+
async executeWithRetry(url, options, attempt = 0, firstError, read = response => response.json()) {
|
|
80
94
|
// Every attempt consumes a token, after any retry backoff and before timeout.
|
|
81
95
|
await this.rateLimiter.acquire();
|
|
82
96
|
const controller = new AbortController();
|
|
@@ -104,7 +118,7 @@ export class BaseClient {
|
|
|
104
118
|
const error = await this.handleHttpError(response);
|
|
105
119
|
throw error;
|
|
106
120
|
}
|
|
107
|
-
return
|
|
121
|
+
return await read(response);
|
|
108
122
|
}
|
|
109
123
|
catch (error) {
|
|
110
124
|
// Clear timeout on error
|
|
@@ -117,13 +131,18 @@ export class BaseClient {
|
|
|
117
131
|
}
|
|
118
132
|
// Track the first error for better error reporting
|
|
119
133
|
const currentFirstError = firstError || (error instanceof ResearchError ? error : undefined);
|
|
120
|
-
//
|
|
134
|
+
// A throttle cooldown survives exhaustion so the next batch line also waits.
|
|
135
|
+
const delay = Math.max(this.calculateBackoff(attempt), error instanceof ResearchError ? Math.min(error.retryAfterMs ?? 0, MAX_RETRY_AFTER_MS) : 0);
|
|
136
|
+
if (error instanceof ResearchError && error.code === ResearchErrorCode.RF_103) {
|
|
137
|
+
this.rateLimiter.defer(delay);
|
|
138
|
+
}
|
|
139
|
+
// Retry transient HTTP failures with the same limiter on every attempt.
|
|
121
140
|
if (error instanceof ResearchError) {
|
|
122
141
|
// Retry on transient errors
|
|
123
142
|
if (this.isRetriable(error) && attempt < this.retryConfig.maxRetries) {
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
return this.executeWithRetry(url, options, attempt + 1, currentFirstError);
|
|
143
|
+
if (error.code !== ResearchErrorCode.RF_103)
|
|
144
|
+
await this.sleep(delay);
|
|
145
|
+
return this.executeWithRetry(url, options, attempt + 1, currentFirstError, read);
|
|
127
146
|
}
|
|
128
147
|
throw error;
|
|
129
148
|
}
|
|
@@ -131,7 +150,7 @@ export class BaseClient {
|
|
|
131
150
|
if (attempt < this.retryConfig.maxRetries) {
|
|
132
151
|
const delay = this.calculateBackoff(attempt);
|
|
133
152
|
await this.sleep(delay);
|
|
134
|
-
return this.executeWithRetry(url, options, attempt + 1, currentFirstError);
|
|
153
|
+
return this.executeWithRetry(url, options, attempt + 1, currentFirstError, read);
|
|
135
154
|
}
|
|
136
155
|
// If we have a first error from an HTTP response, throw that instead of generic network error
|
|
137
156
|
if (currentFirstError) {
|
|
@@ -149,7 +168,14 @@ export class BaseClient {
|
|
|
149
168
|
return new ResearchError(ResearchErrorCode.RF_300, 'Resource not found');
|
|
150
169
|
}
|
|
151
170
|
if (status === 429) {
|
|
152
|
-
|
|
171
|
+
const error = new ResearchError(ResearchErrorCode.RF_103, 'Rate limit exceeded');
|
|
172
|
+
const value = response.headers?.get('retry-after');
|
|
173
|
+
if (value) {
|
|
174
|
+
const ms = /^\d+(?:\.\d+)?$/.test(value.trim()) ? Number(value) * 1000 : Date.parse(value) - Date.now();
|
|
175
|
+
if (Number.isFinite(ms))
|
|
176
|
+
error.retryAfterMs = Math.max(0, ms);
|
|
177
|
+
}
|
|
178
|
+
return error;
|
|
153
179
|
}
|
|
154
180
|
if (status === 401 || status === 403) {
|
|
155
181
|
return new ResearchError(ResearchErrorCode.RF_102, 'Invalid API key or unauthorized');
|
|
@@ -164,8 +190,8 @@ export class BaseClient {
|
|
|
164
190
|
*/
|
|
165
191
|
isRetriable(error) {
|
|
166
192
|
if (error instanceof ResearchError) {
|
|
167
|
-
//
|
|
168
|
-
return error.code === ResearchErrorCode.RF_200;
|
|
193
|
+
// Rate limits and server errors are transient.
|
|
194
|
+
return error.code === ResearchErrorCode.RF_200 || error.code === ResearchErrorCode.RF_103;
|
|
169
195
|
}
|
|
170
196
|
// Retry on network errors
|
|
171
197
|
return true;
|
|
@@ -67,19 +67,57 @@ export function normalizeIdentity(value) {
|
|
|
67
67
|
.map(v => typeof v === 'number' && Number.isFinite(v) ? String(v) : v)
|
|
68
68
|
.find(v => typeof v === 'string' && v.trim());
|
|
69
69
|
const arxiv = parseArxivId(str('arxivId', 'arxiv_id', 'arxiv') ?? '') ?? parseArxivId(str('doi') ?? '');
|
|
70
|
+
const alternateDois = Array.isArray(fields.alternateDois)
|
|
71
|
+
? [...new Set(fields.alternateDois.flatMap(value => typeof value === 'string' ? normalizeDoi(value) ?? [] : []))] : [];
|
|
70
72
|
return {
|
|
71
73
|
...(arxiv ? { arxivId: arxiv.id, arxivVersion: arxiv.version ?? (typeof fields.arxivVersion === 'number' ? fields.arxivVersion : undefined) } : {}),
|
|
72
74
|
doi: normalizeDoi(str('doi', 'DOI') ?? ''),
|
|
75
|
+
...(alternateDois.length ? { alternateDois } : {}),
|
|
73
76
|
s2PaperId: str('s2PaperId', 's2_paper_id', 'paperId')?.toLowerCase(),
|
|
74
|
-
openReviewId: str('openReviewId', 'openreview_id'),
|
|
75
|
-
aclId: str('aclId', 'acl_id'),
|
|
77
|
+
openReviewId: str('openReviewId', 'openreview_id')?.toLowerCase(),
|
|
78
|
+
aclId: str('aclId', 'acl_id')?.toLowerCase(),
|
|
76
79
|
title: str('title') ? normalizeTitle(str('title')) : undefined,
|
|
77
80
|
};
|
|
78
81
|
}
|
|
79
82
|
/** Stable keys for matching any identifier or a normalized title across corpus sources. */
|
|
80
83
|
export function identityKeys(value) {
|
|
81
84
|
const identity = normalizeIdentity(value);
|
|
82
|
-
return [
|
|
83
|
-
|
|
85
|
+
return [...new Set([
|
|
86
|
+
...['arxivId', 'doi', 's2PaperId', 'openReviewId', 'aclId', 'title']
|
|
87
|
+
.flatMap(key => identity[key] ? [`${key}:${identity[key]}`] : []),
|
|
88
|
+
...(identity.alternateDois ?? []).map(doi => `doi:${doi}`),
|
|
89
|
+
])];
|
|
90
|
+
}
|
|
91
|
+
/** REF spelling is case-sensitive, including the optional legacy suffix letter. */
|
|
92
|
+
export const REF_ID_SOURCE = 'REF-\\d+[a-zA-Z]?';
|
|
93
|
+
/** Match whole tokens in prose and paths without accepting a malformed ID prefix. */
|
|
94
|
+
export const REF_ID_PATTERN = `(?<![A-Za-z0-9_])${REF_ID_SOURCE}(?![A-Za-z0-9_])`;
|
|
95
|
+
const REF_ID_FULL = new RegExp(`^${REF_ID_SOURCE}$`);
|
|
96
|
+
export function refIdsInText(text) {
|
|
97
|
+
return text.match(new RegExp(REF_ID_PATTERN, 'g')) ?? [];
|
|
98
|
+
}
|
|
99
|
+
/** Exact lexical REF identity: preserve zero padding and suffix case. */
|
|
100
|
+
export function isRefId(value) {
|
|
101
|
+
return value.trim() === value && REF_ID_FULL.test(value);
|
|
102
|
+
}
|
|
103
|
+
/** Read the REF token before a slug or extension; never normalize its spelling. */
|
|
104
|
+
export function refIdFromFilename(filename, extension = '.md') {
|
|
105
|
+
if (!filename.endsWith(extension))
|
|
106
|
+
return null;
|
|
107
|
+
const stem = filename.slice(0, -extension.length);
|
|
108
|
+
const slug = stem.indexOf('-', 4);
|
|
109
|
+
const ref = slug === -1 ? stem : stem.slice(0, slug);
|
|
110
|
+
return isRefId(ref) ? ref : null;
|
|
111
|
+
}
|
|
112
|
+
/** Sort key for REF ids: [numeric, alpha-suffix]. Mirrors build.py ref_sort_key. */
|
|
113
|
+
export function refSortKey(refId) {
|
|
114
|
+
const m = isRefId(refId) ? refId.match(/^REF-(\d+)([a-zA-Z]?)$/) : null;
|
|
115
|
+
return m ? [parseInt(m[1], 10), m[2]] : [999999, refId];
|
|
116
|
+
}
|
|
117
|
+
/** Compare two REF ids by (numeric, suffix). */
|
|
118
|
+
export function compareRefId(a, b) {
|
|
119
|
+
const [an, as_] = refSortKey(a);
|
|
120
|
+
const [bn, bs] = refSortKey(b);
|
|
121
|
+
return an !== bn ? an - bn : as_ < bs ? -1 : as_ > bs ? 1 : 0;
|
|
84
122
|
}
|
|
85
123
|
//# sourceMappingURL=identity.js.map
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import fs from "node:fs";
|
|
2
|
+
import { parseArgs, usage } from "./query-options.js";
|
|
3
|
+
import { refIdsInText } from "./identity.js";
|
|
2
4
|
import path from "node:path";
|
|
3
5
|
import { parseFrontmatter } from "../artifacts/index-builder.js";
|
|
4
6
|
import { loadGraphIndexFile } from "../artifacts/index-reader.js";
|
|
@@ -36,92 +38,6 @@ const STOPWORDS = new Set([
|
|
|
36
38
|
"say",
|
|
37
39
|
"evidence",
|
|
38
40
|
]);
|
|
39
|
-
function usage() {
|
|
40
|
-
return [
|
|
41
|
-
"Usage: aiwg research-query <question> [--backend fortemi-core|local] [--graph <name>]",
|
|
42
|
-
" [--depth quick|thorough] [--sources-only] [--max-sources N]",
|
|
43
|
-
" [--include-diagnostics] [--json] [--save]",
|
|
44
|
-
].join("\n");
|
|
45
|
-
}
|
|
46
|
-
function flagValue(args, flag, errorMessage) {
|
|
47
|
-
const index = args.indexOf(flag);
|
|
48
|
-
if (index < 0)
|
|
49
|
-
return undefined;
|
|
50
|
-
const value = args[index + 1];
|
|
51
|
-
if (!value || value.startsWith("--")) {
|
|
52
|
-
throw new Error(errorMessage);
|
|
53
|
-
}
|
|
54
|
-
return value;
|
|
55
|
-
}
|
|
56
|
-
function positiveIntegerFlag(args, flag, defaultValue, errorMessage) {
|
|
57
|
-
const value = flagValue(args, flag, errorMessage);
|
|
58
|
-
if (value === undefined)
|
|
59
|
-
return defaultValue;
|
|
60
|
-
const parsed = Number.parseInt(value, 10);
|
|
61
|
-
if (!Number.isInteger(parsed) || parsed < 1 || String(parsed) !== value) {
|
|
62
|
-
throw new Error(errorMessage);
|
|
63
|
-
}
|
|
64
|
-
return parsed;
|
|
65
|
-
}
|
|
66
|
-
function hasFlag(args, flag) {
|
|
67
|
-
return args.includes(flag);
|
|
68
|
-
}
|
|
69
|
-
function stripFlags(args) {
|
|
70
|
-
const valueFlags = new Set([
|
|
71
|
-
"--backend",
|
|
72
|
-
"--graph",
|
|
73
|
-
"--depth",
|
|
74
|
-
"--max-sources",
|
|
75
|
-
]);
|
|
76
|
-
const bareFlags = new Set([
|
|
77
|
-
"--sources-only",
|
|
78
|
-
"--include-diagnostics",
|
|
79
|
-
"--json",
|
|
80
|
-
"--save",
|
|
81
|
-
]);
|
|
82
|
-
const question = [];
|
|
83
|
-
for (let index = 0; index < args.length; index++) {
|
|
84
|
-
const arg = args[index];
|
|
85
|
-
if (valueFlags.has(arg)) {
|
|
86
|
-
index++;
|
|
87
|
-
continue;
|
|
88
|
-
}
|
|
89
|
-
if (bareFlags.has(arg))
|
|
90
|
-
continue;
|
|
91
|
-
question.push(arg);
|
|
92
|
-
}
|
|
93
|
-
return question;
|
|
94
|
-
}
|
|
95
|
-
function parseArgs(args) {
|
|
96
|
-
if (hasFlag(args, "--help") || hasFlag(args, "-h")) {
|
|
97
|
-
console.log(usage());
|
|
98
|
-
process.exit(0);
|
|
99
|
-
}
|
|
100
|
-
const backend = flagValue(args, "--backend", "--backend must be local or fortemi-core");
|
|
101
|
-
if (backend && backend !== "local" && backend !== "fortemi-core") {
|
|
102
|
-
throw new Error("--backend must be local or fortemi-core");
|
|
103
|
-
}
|
|
104
|
-
const depth = (flagValue(args, "--depth", "--depth must be quick or thorough") ?? "thorough");
|
|
105
|
-
if (depth !== "quick" && depth !== "thorough") {
|
|
106
|
-
throw new Error("--depth must be quick or thorough");
|
|
107
|
-
}
|
|
108
|
-
const maxSources = positiveIntegerFlag(args, "--max-sources", 10, "--max-sources must be a positive integer");
|
|
109
|
-
const question = stripFlags(args).join(" ").trim();
|
|
110
|
-
if (!question)
|
|
111
|
-
throw new Error("question is required");
|
|
112
|
-
return {
|
|
113
|
-
question,
|
|
114
|
-
backend: backend ?? "fortemi-core",
|
|
115
|
-
graph: (flagValue(args, "--graph", "--graph requires a graph name") ??
|
|
116
|
-
"project"),
|
|
117
|
-
depth,
|
|
118
|
-
maxSources,
|
|
119
|
-
sourcesOnly: hasFlag(args, "--sources-only"),
|
|
120
|
-
includeDiagnostics: hasFlag(args, "--include-diagnostics"),
|
|
121
|
-
json: hasFlag(args, "--json"),
|
|
122
|
-
save: hasFlag(args, "--save"),
|
|
123
|
-
};
|
|
124
|
-
}
|
|
125
41
|
function tokens(text) {
|
|
126
42
|
return text
|
|
127
43
|
.toLowerCase()
|
|
@@ -141,7 +57,7 @@ function readEntryBody(cwd, entryPath) {
|
|
|
141
57
|
}
|
|
142
58
|
function entryId(entry) {
|
|
143
59
|
const text = [entry.name, entry.title, entry.path].filter(Boolean).join(" ");
|
|
144
|
-
return (text
|
|
60
|
+
return (refIdsInText(text)[0] ??
|
|
145
61
|
text.match(/\bPROF-[A-Z0-9-]+/i)?.[0]?.toUpperCase() ??
|
|
146
62
|
path.basename(entry.path).replace(/\.[^.]+$/, ""));
|
|
147
63
|
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
export function usage() {
|
|
2
|
+
return [
|
|
3
|
+
"Usage: aiwg research-query <question> [--backend fortemi-core|local] [--graph <name>]",
|
|
4
|
+
" [--depth quick|thorough] [--sources-only] [--max-sources N]",
|
|
5
|
+
" [--include-diagnostics] [--json] [--save]",
|
|
6
|
+
].join("\n");
|
|
7
|
+
}
|
|
8
|
+
function flagValue(args, flag, errorMessage) {
|
|
9
|
+
const index = args.indexOf(flag);
|
|
10
|
+
if (index < 0)
|
|
11
|
+
return undefined;
|
|
12
|
+
const value = args[index + 1];
|
|
13
|
+
if (!value || value.startsWith("--")) {
|
|
14
|
+
throw new Error(errorMessage);
|
|
15
|
+
}
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
function positiveIntegerFlag(args, flag, defaultValue, errorMessage) {
|
|
19
|
+
const value = flagValue(args, flag, errorMessage);
|
|
20
|
+
if (value === undefined)
|
|
21
|
+
return defaultValue;
|
|
22
|
+
const parsed = Number.parseInt(value, 10);
|
|
23
|
+
if (!Number.isInteger(parsed) || parsed < 1 || String(parsed) !== value) {
|
|
24
|
+
throw new Error(errorMessage);
|
|
25
|
+
}
|
|
26
|
+
return parsed;
|
|
27
|
+
}
|
|
28
|
+
function hasFlag(args, flag) {
|
|
29
|
+
return args.includes(flag);
|
|
30
|
+
}
|
|
31
|
+
function stripFlags(args) {
|
|
32
|
+
const valueFlags = new Set([
|
|
33
|
+
"--backend",
|
|
34
|
+
"--graph",
|
|
35
|
+
"--depth",
|
|
36
|
+
"--max-sources",
|
|
37
|
+
]);
|
|
38
|
+
const bareFlags = new Set([
|
|
39
|
+
"--sources-only",
|
|
40
|
+
"--include-diagnostics",
|
|
41
|
+
"--json",
|
|
42
|
+
"--save",
|
|
43
|
+
]);
|
|
44
|
+
const question = [];
|
|
45
|
+
for (let index = 0; index < args.length; index++) {
|
|
46
|
+
const arg = args[index];
|
|
47
|
+
if (valueFlags.has(arg)) {
|
|
48
|
+
index++;
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
if (bareFlags.has(arg))
|
|
52
|
+
continue;
|
|
53
|
+
question.push(arg);
|
|
54
|
+
}
|
|
55
|
+
return question;
|
|
56
|
+
}
|
|
57
|
+
export function parseArgs(args) {
|
|
58
|
+
if (hasFlag(args, "--help") || hasFlag(args, "-h")) {
|
|
59
|
+
console.log(usage());
|
|
60
|
+
process.exit(0);
|
|
61
|
+
}
|
|
62
|
+
const backend = flagValue(args, "--backend", "--backend must be local or fortemi-core");
|
|
63
|
+
if (backend && backend !== "local" && backend !== "fortemi-core") {
|
|
64
|
+
throw new Error("--backend must be local or fortemi-core");
|
|
65
|
+
}
|
|
66
|
+
const depth = (flagValue(args, "--depth", "--depth must be quick or thorough") ?? "thorough");
|
|
67
|
+
if (depth !== "quick" && depth !== "thorough") {
|
|
68
|
+
throw new Error("--depth must be quick or thorough");
|
|
69
|
+
}
|
|
70
|
+
const maxSources = positiveIntegerFlag(args, "--max-sources", 10, "--max-sources must be a positive integer");
|
|
71
|
+
const question = stripFlags(args).join(" ").trim();
|
|
72
|
+
if (!question)
|
|
73
|
+
throw new Error("question is required");
|
|
74
|
+
return {
|
|
75
|
+
question,
|
|
76
|
+
backend: backend ?? "fortemi-core",
|
|
77
|
+
graph: (flagValue(args, "--graph", "--graph requires a graph name") ??
|
|
78
|
+
"project"),
|
|
79
|
+
depth,
|
|
80
|
+
maxSources,
|
|
81
|
+
sourcesOnly: hasFlag(args, "--sources-only"),
|
|
82
|
+
includeDiagnostics: hasFlag(args, "--include-diagnostics"),
|
|
83
|
+
json: hasFlag(args, "--json"),
|
|
84
|
+
save: hasFlag(args, "--save"),
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=query-options.js.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Parse partial citations and score distinct candidate identities without choosing close matches.
|
|
2
|
-
import { normalizeIdentity, normalizeTitle, parseResearchIdentifier } from './identity.js';
|
|
2
|
+
import { normalizeIdentity, normalizeTitle, parseArxivId, parseResearchIdentifier } from './identity.js';
|
|
3
3
|
/** Common quoted, APA-style and title-plus-year citations; retain the untouched input. */
|
|
4
4
|
export function parsePartialReference(input, year) {
|
|
5
5
|
const identifier = parseResearchIdentifier(input);
|
|
@@ -22,12 +22,52 @@ export function parsePartialReference(input, year) {
|
|
|
22
22
|
rest = rest.replace(/^[\s.,;:()[\]{}"']+|[\s.,;:()[\]{}"']+$/g, '').trim();
|
|
23
23
|
return { input, identifier, title: normalizeTitle(rest) ? rest : undefined, year: year ?? (inferredYear ? Number(inferredYear) : undefined), firstAuthor };
|
|
24
24
|
}
|
|
25
|
-
/**
|
|
25
|
+
/** Token coverage discounted by sequence order; ordered prefixes retain coverage scores. */
|
|
26
26
|
export function titleSimilarity(left, right) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
const
|
|
30
|
-
|
|
27
|
+
if (normalizeTitle(left) && normalizeTitle(left) === normalizeTitle(right))
|
|
28
|
+
return 1;
|
|
29
|
+
const a = normalizeTitle(left).split(' ').filter(Boolean);
|
|
30
|
+
const b = normalizeTitle(right).split(' ').filter(Boolean);
|
|
31
|
+
if (!a.length || !b.length)
|
|
32
|
+
return 0;
|
|
33
|
+
const uniqueA = new Set(a), uniqueB = new Set(b);
|
|
34
|
+
const coverage = [...uniqueA].filter(token => uniqueB.has(token)).length / new Set([...a, ...b]).size;
|
|
35
|
+
const row = Array(b.length + 1).fill(0);
|
|
36
|
+
for (const token of a) {
|
|
37
|
+
let diagonal = 0;
|
|
38
|
+
for (let j = 1; j <= b.length; j++) {
|
|
39
|
+
const previous = row[j];
|
|
40
|
+
row[j] = token === b[j - 1] ? diagonal + 1 : Math.max(row[j], row[j - 1]);
|
|
41
|
+
diagonal = previous;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
// Repeated tokens must also reduce similarity even when their token sets agree.
|
|
45
|
+
const repetition = Math.min(uniqueA.size / a.length, uniqueB.size / b.length);
|
|
46
|
+
return coverage * row[b.length] / Math.min(a.length, b.length) * repetition;
|
|
47
|
+
}
|
|
48
|
+
/** Caution needs competing candidates or an unqualified generic topic, never length alone with metadata. */
|
|
49
|
+
export function needsTitleDisambiguation(reference, candidates, incomplete) {
|
|
50
|
+
if (!reference.title || !candidates.length || candidates[0]?.score_parts.identifier === 1)
|
|
51
|
+
return false;
|
|
52
|
+
const unqualified = reference.year === undefined && !reference.firstAuthor;
|
|
53
|
+
const competing = candidates.slice(1).some(candidate => classifyCandidates([candidate]) !== 'unresolved');
|
|
54
|
+
// A supplied year that the only candidate contradicts is not corroboration.
|
|
55
|
+
if (reference.year !== undefined && candidates[0]?.score_parts.year === 0)
|
|
56
|
+
return true;
|
|
57
|
+
if (!competing && !unqualified)
|
|
58
|
+
return false;
|
|
59
|
+
const title = normalizeTitle(reference.title);
|
|
60
|
+
const tokens = title.split(' ');
|
|
61
|
+
const informative = tokens.filter(token => !/^(?:a|an|the|of|for|in|on|and|with|to|is|all|you)$/.test(token));
|
|
62
|
+
const exact = candidates.filter(candidate => normalizeTitle(candidate.paper.title) === title).length;
|
|
63
|
+
// Overview headings and common topic phrases remain generic even after a complete search.
|
|
64
|
+
// Single names/acronyms and distinctive technical phrases keep their complete-search behavior.
|
|
65
|
+
const overview = /^(?:(?:a|an|the) )?(?:tutorial|theory)\b/.test(title);
|
|
66
|
+
const commonTopic = informative.every(token => /^(?:deep|learning|scaling|laws|logical|physical|clocks)$/.test(token));
|
|
67
|
+
const generic = tokens.length > 1 && (tokens.length <= 2 ||
|
|
68
|
+
(informative.length <= 4 && (overview || commonTopic)));
|
|
69
|
+
return (incomplete && (tokens.length <= 4 || informative.length <= 4 || exact > 1)) ||
|
|
70
|
+
(generic && (unqualified || competing));
|
|
31
71
|
}
|
|
32
72
|
function surname(name) {
|
|
33
73
|
return normalizeTitle(name.includes(',') ? name.split(',')[0] : name.trim().split(/\s+/).at(-1) ?? '');
|
|
@@ -60,29 +100,70 @@ export function classifyCandidates(candidates) {
|
|
|
60
100
|
const margin = best - (candidates[1]?.score ?? 0);
|
|
61
101
|
return best + 1e-12 >= 0.92 && margin + 1e-12 >= 0.05 ? 'resolved' : 'ambiguous';
|
|
62
102
|
}
|
|
63
|
-
/**
|
|
103
|
+
/** Shared IDs or corroborated preprint/publication metadata establish a work, never title alone. */
|
|
64
104
|
export function mergePapers(papers) {
|
|
105
|
+
const ids = ['arxivId', 'doi', 's2PaperId', 'openReviewId', 'aclId'];
|
|
106
|
+
const preprintDoi = (doi) => Boolean(parseArxivId(doi) || /^10\.2139\/ssrn\./i.test(doi));
|
|
107
|
+
const published = (paper) => Boolean(paper.doi && !preprintDoi(paper.doi));
|
|
108
|
+
const preprint = (paper) => !published(paper) && Boolean(paper.arxivId || (paper.doi && preprintDoi(paper.doi)));
|
|
109
|
+
const sameWorkMetadata = (left, right) => {
|
|
110
|
+
if (!((preprint(left) && published(right)) || (preprint(right) && published(left))))
|
|
111
|
+
return false;
|
|
112
|
+
if (!Number.isInteger(left.year) || !Number.isInteger(right.year) || Math.abs(left.year - right.year) > 2)
|
|
113
|
+
return false;
|
|
114
|
+
if (!normalizeTitle(left.title) || normalizeTitle(left.title) !== normalizeTitle(right.title))
|
|
115
|
+
return false;
|
|
116
|
+
const names = new Set(left.authors.map(author => normalizeTitle(author.name)).filter(Boolean));
|
|
117
|
+
return right.authors.some(author => names.has(normalizeTitle(author.name)));
|
|
118
|
+
};
|
|
119
|
+
const linked = (left, right) => {
|
|
120
|
+
const a = normalizeIdentity(left), b = normalizeIdentity(right);
|
|
121
|
+
if (titleSimilarity(a.title ?? '', b.title ?? '') < 0.5)
|
|
122
|
+
return false;
|
|
123
|
+
if (published(left) && published(right) && a.doi !== b.doi)
|
|
124
|
+
return false;
|
|
125
|
+
const metadataMatch = sameWorkMetadata(left, right);
|
|
126
|
+
const sharedDoi = [a.doi, ...(a.alternateDois ?? [])].some(doi => doi && [b.doi, ...(b.alternateDois ?? [])].includes(doi));
|
|
127
|
+
if (ids.some(key => a[key] && b[key] && a[key] !== b[key] &&
|
|
128
|
+
!(key === 'doi' && (metadataMatch || sharedDoi ||
|
|
129
|
+
(a.arxivId && a.arxivId === b.arxivId && (parseArxivId(a.doi) || parseArxivId(b.doi)))))))
|
|
130
|
+
return false;
|
|
131
|
+
return metadataMatch || sharedDoi || ids.some(key => a[key] && a[key] === b[key]);
|
|
132
|
+
};
|
|
133
|
+
const combine = (left, right) => {
|
|
134
|
+
// A published CrossRef record owns proceedings metadata; other sources fill gaps.
|
|
135
|
+
const preferRight = (published(right) && !published(left)) ||
|
|
136
|
+
(published(right) && right.source === 'crossref' && left.source !== 'crossref');
|
|
137
|
+
const [primary, alternate] = preferRight ? [right, left] : [left, right];
|
|
138
|
+
const result = { ...primary };
|
|
139
|
+
for (const [key, value] of Object.entries(alternate)) {
|
|
140
|
+
if (value !== undefined && (!result[key] ||
|
|
141
|
+
(Array.isArray(result[key]) && result[key].length === 0))) {
|
|
142
|
+
result[key] = value;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const alternateDois = [...new Set([primary.doi, alternate.doi,
|
|
146
|
+
...(primary.alternateDois ?? []), ...(alternate.alternateDois ?? [])])]
|
|
147
|
+
.filter((doi) => Boolean(doi && doi !== result.doi));
|
|
148
|
+
if (alternateDois.length)
|
|
149
|
+
result.alternateDois = alternateDois;
|
|
150
|
+
return result;
|
|
151
|
+
};
|
|
65
152
|
const merged = [];
|
|
66
153
|
for (const paper of papers) {
|
|
67
|
-
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
if (!previous) {
|
|
76
|
-
merged.push({ ...paper });
|
|
77
|
-
continue;
|
|
78
|
-
}
|
|
79
|
-
// Source order determines metadata precedence; later sources only fill missing fields.
|
|
80
|
-
for (const [key, value] of Object.entries(paper)) {
|
|
81
|
-
if (value !== undefined && (!previous[key] ||
|
|
82
|
-
(Array.isArray(previous[key]) && previous[key].length === 0))) {
|
|
83
|
-
previous[key] = value;
|
|
154
|
+
// Comparison keys are canonical; OpenReview/ACL spelling remains source metadata.
|
|
155
|
+
const { title: _title, openReviewId: _review, aclId: _acl, ...identity } = normalizeIdentity(paper);
|
|
156
|
+
let work = { ...paper, ...Object.fromEntries(Object.entries(identity).filter(([, value]) => value !== undefined)) };
|
|
157
|
+
// A late S2 bridge may join two components already encountered in source order.
|
|
158
|
+
for (let i = 0; i < merged.length;) {
|
|
159
|
+
if (!linked(work, merged[i])) {
|
|
160
|
+
i++;
|
|
161
|
+
continue;
|
|
84
162
|
}
|
|
163
|
+
work = combine(merged.splice(i, 1)[0], work);
|
|
164
|
+
i = 0;
|
|
85
165
|
}
|
|
166
|
+
merged.push(work);
|
|
86
167
|
}
|
|
87
168
|
return merged;
|
|
88
169
|
}
|
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { promises as fs } from 'fs';
|
|
7
7
|
import { join, dirname } from 'path';
|
|
8
8
|
import { createHash } from 'crypto';
|
|
9
|
+
import { isRefId, refIdFromFilename, refSortKey } from '../identity.js';
|
|
9
10
|
import { UnpaywallClient } from '../clients/unpaywall.js';
|
|
10
11
|
/**
|
|
11
12
|
* Acquisition service for downloading research papers
|
|
@@ -87,8 +88,8 @@ export class AcquisitionService {
|
|
|
87
88
|
}
|
|
88
89
|
// Extract numbers from REF-XXX format
|
|
89
90
|
const refNumbers = refs
|
|
90
|
-
.filter(
|
|
91
|
-
.map((ref) =>
|
|
91
|
+
.filter(isRefId)
|
|
92
|
+
.map((ref) => refSortKey(ref)[0])
|
|
92
93
|
.filter((n) => !isNaN(n));
|
|
93
94
|
// Find next available number
|
|
94
95
|
const maxNumber = refNumbers.length > 0 ? Math.max(...refNumbers) : 0;
|
|
@@ -138,7 +139,7 @@ export class AcquisitionService {
|
|
|
138
139
|
{
|
|
139
140
|
id: 'F1',
|
|
140
141
|
description: 'Assigned globally unique identifier (REF-XXX)',
|
|
141
|
-
met:
|
|
142
|
+
met: isRefId(source.refId),
|
|
142
143
|
},
|
|
143
144
|
{
|
|
144
145
|
id: 'F2',
|
|
@@ -283,12 +284,13 @@ export class AcquisitionService {
|
|
|
283
284
|
async scanExistingRefs() {
|
|
284
285
|
const refs = [];
|
|
285
286
|
try {
|
|
286
|
-
const files = await fs.readdir(this.downloadDir);
|
|
287
|
+
const files = await fs.readdir(this.downloadDir, { withFileTypes: true });
|
|
287
288
|
for (const file of files) {
|
|
288
|
-
if (file.
|
|
289
|
-
|
|
289
|
+
if (!file.isFile())
|
|
290
|
+
continue;
|
|
291
|
+
const refId = refIdFromFilename(file.name, '.pdf');
|
|
292
|
+
if (refId)
|
|
290
293
|
refs.push(refId);
|
|
291
|
-
}
|
|
292
294
|
}
|
|
293
295
|
}
|
|
294
296
|
catch (error) {
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* @module research/services/citation
|
|
5
5
|
*/
|
|
6
6
|
import { promises as fs } from 'fs';
|
|
7
|
+
import { isRefId, refIdFromFilename } from '../identity.js';
|
|
7
8
|
/**
|
|
8
9
|
* Citation service for managing citations and networks
|
|
9
10
|
*/
|
|
@@ -37,10 +38,12 @@ export class CitationService {
|
|
|
37
38
|
* Validate that a citation exists in corpus
|
|
38
39
|
*/
|
|
39
40
|
async validateCitation(refId, corpusPath) {
|
|
41
|
+
if (!isRefId(refId))
|
|
42
|
+
return false;
|
|
40
43
|
const path = corpusPath || this.corpusPath;
|
|
41
44
|
try {
|
|
42
|
-
const files = await fs.readdir(path);
|
|
43
|
-
return files.some((file) => file.
|
|
45
|
+
const files = await fs.readdir(path, { withFileTypes: true });
|
|
46
|
+
return files.some((file) => file.isFile() && refIdFromFilename(file.name) === refId);
|
|
44
47
|
}
|
|
45
48
|
catch (error) {
|
|
46
49
|
return false;
|
|
@@ -37,6 +37,8 @@ export var ResearchErrorCode;
|
|
|
37
37
|
export class ResearchError extends Error {
|
|
38
38
|
code;
|
|
39
39
|
cause;
|
|
40
|
+
/** Server-directed cooldown, shared by all requests through the client limiter. */
|
|
41
|
+
retryAfterMs;
|
|
40
42
|
constructor(code, message, cause) {
|
|
41
43
|
super(message);
|
|
42
44
|
this.code = code;
|