@aiwg/cli 2026.10.12 → 2026.10.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/addons/aiwg-utils/rules/anti-laziness.core.md +13 -0
- package/agentic/code/addons/aiwg-utils/rules/anti-laziness.md +118 -0
- package/agentic/code/providers/pi/aiwg-bridge.ts +44 -1
- package/dist/src/artifacts/corpus-tools/cli.js +32 -0
- package/dist/src/artifacts/corpus-tools/discovery-log.js +1 -1
- package/dist/src/artifacts/corpus-tools/expand-cli.js +89 -0
- package/dist/src/artifacts/corpus-tools/expand-queue.js +38 -0
- package/dist/src/artifacts/corpus-tools/expand-state.js +40 -0
- package/dist/src/artifacts/corpus-tools/expand.js +125 -0
- package/dist/src/artifacts/corpus-tools/resolve-cli.js +91 -0
- package/dist/src/artifacts/corpus-tools/resolve-corpus.js +41 -0
- package/dist/src/artifacts/corpus-tools/resolve.js +105 -0
- package/dist/src/cli/handlers/doctor-args.js +40 -0
- package/dist/src/cli/handlers/doctor-help.js +4 -2
- package/dist/src/cli/handlers/use-bundle-counts.js +32 -6
- package/dist/src/cli/handlers/use-skill-assets.js +207 -0
- package/dist/src/cli/handlers/use.js +30 -193
- package/dist/src/cli/handlers/utilities.js +5 -31
- package/dist/src/config/authorization-cli.js +11 -3
- package/dist/src/config/worker-context.js +33 -4
- package/dist/src/decision/context-pruning-common.js +41 -0
- package/dist/src/decision/context-pruning-evaluation.js +366 -0
- package/dist/src/decision/context-pruning-pilot.js +431 -0
- package/dist/src/decision/context-pruning-types.js +19 -0
- package/dist/src/decision/context-pruning.js +5 -847
- package/dist/src/decision/ensemble/contract.js +4 -24
- package/dist/src/decision/heldout/collector.js +1 -1
- package/dist/src/decision/qualification/release.js +5 -62
- package/dist/src/extensions/manifest.js +8 -1
- package/dist/src/extensions/project-local-doctor.js +78 -2
- package/dist/src/extensions/validation.js +10 -11
- package/dist/src/features/catalog.js +1 -1
- package/dist/src/gates/evaluate.js +5 -15
- package/dist/src/gates/heldout.js +42 -0
- package/dist/src/gates/index.js +2 -1
- package/dist/src/gates/upstream-ceiling.js +111 -0
- package/dist/src/models/provider-models.js +5 -8
- package/dist/src/models/worker-installation-store.mjs +228 -0
- package/dist/src/models/worker-installation.mjs +7 -162
- package/dist/src/models/wrapper-definitions.js +14 -0
- package/dist/src/models/wrapper-deployment.js +48 -25
- package/dist/src/models/wrapper-enforcement.js +72 -0
- package/dist/src/policy/hitl-check.js +80 -0
- package/dist/src/providers/provider-definitions.js +10 -10
- package/dist/src/research/clients/arxiv.js +10 -4
- package/dist/src/research/clients/base.js +5 -4
- package/dist/src/research/clients/crossref.js +9 -1
- package/dist/src/research/clients/semantic-scholar.js +73 -44
- package/dist/src/research/identity.js +85 -0
- package/dist/src/research/index.js +1 -0
- package/dist/src/research/reference-match.js +89 -0
- package/dist/src/rules/activation.d.mts +2 -0
- package/dist/src/rules/activation.mjs +23 -0
- package/dist/src/rules/core-startup.d.mts +2 -0
- package/dist/src/rules/core-startup.mjs +31 -0
- package/dist/src/skills/adapters/local.js +32 -9
- package/dist/src/skills/agent-skills.js +2 -0
- package/dist/src/skills/cli-display.js +33 -0
- package/dist/src/skills/cli.js +2 -26
- package/dist/src/skills/collision-discovery.js +84 -0
- package/dist/src/skills/deployer.js +20 -75
- package/dist/src/skills/deployment-ownership.js +74 -0
- package/dist/src/skills/doctor.js +2 -2
- package/dist/src/skills/importer.js +4 -26
- package/dist/src/skills/registry.js +22 -7
- package/dist/src/smiths/context-pipeline/agents-md.js +34 -0
- package/dist/src/smiths/context-pipeline/aiwg-md.js +3 -0
- package/dist/src/smiths/context-pipeline/bridge-ownership.js +50 -0
- package/dist/src/smiths/context-pipeline/generator.js +40 -104
- package/dist/src/smiths/context-pipeline/managed-hook.js +10 -0
- package/dist/src/smiths/context-pipeline/parallelism-section.js +7 -3
- package/package.json +6 -2
- package/tools/agents/deploy-agents.mjs +2 -123
- package/tools/agents/providers/antigravity.mjs +4 -6
- package/tools/agents/providers/base.mjs +3 -3
- package/tools/agents/providers/claude.mjs +3 -5
- package/tools/agents/providers/codex.mjs +3 -5
- package/tools/agents/providers/context-write.mjs +12 -0
- package/tools/agents/providers/copilot.mjs +3 -5
- package/tools/agents/providers/cursor.mjs +3 -5
- package/tools/agents/providers/factory.mjs +3 -5
- package/tools/agents/providers/grok-build.mjs +17 -12
- package/tools/agents/providers/hermes-skill-lifecycle.mjs +144 -0
- package/tools/agents/providers/hermes-startup.mjs +209 -0
- package/tools/agents/providers/hermes.mjs +9 -359
- package/tools/agents/providers/model-wrapper.mjs +90 -0
- package/tools/agents/providers/muse.mjs +3 -3
- package/tools/agents/providers/omp.mjs +3 -1
- package/tools/agents/providers/openclaw.mjs +3 -5
- package/tools/agents/providers/opencode.mjs +3 -5
- package/tools/agents/providers/pi.mjs +1 -1
- package/tools/agents/providers/provider-paths-staging.mjs +9 -0
- package/tools/agents/providers/rule-delivery.mjs +26 -228
- package/tools/agents/providers/rule-presentation.mjs +13 -4
- package/tools/agents/providers/rule-projection.mjs +119 -0
- package/tools/agents/providers/rule-startup.mjs +70 -0
- package/tools/agents/providers/stale-artifact-prune.mjs +128 -0
- package/tools/agents/providers/warp.mjs +5 -7
- package/tools/agents/providers/windsurf.mjs +14 -62
- package/tools/plugin/package-plugins.mjs +2 -2
- package/tools/plugin/plugin-links.mjs +3 -0
- package/tools/skills/codex-skill-transform.mjs +232 -0
- package/tools/skills/deploy-skills-codex.mjs +8 -249
- package/tools/warp/warp-content.mjs +548 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Describes controls on deployed wrapper artifacts, distinct from headless CLI controls.
|
|
2
|
+
import { compileModelPolicy, loadProviderModelCatalog } from './catalog-policy.js';
|
|
3
|
+
import { MODEL_WRAPPER_DEFINITIONS } from './wrapper-definitions.js';
|
|
4
|
+
const NATIVE_CONTROLS = {
|
|
5
|
+
claude: { model: 'model', effort: 'effort' },
|
|
6
|
+
codex: { model: 'model', effort: 'model_reasoning_effort' },
|
|
7
|
+
copilot: { model: 'model' }, cursor: { model: 'model' },
|
|
8
|
+
factory: { model: 'model', effort: 'reasoningEffort' },
|
|
9
|
+
opencode: { model: 'model' }, omp: { model: 'model', effort: 'thinkingLevel' },
|
|
10
|
+
};
|
|
11
|
+
const GLOBAL_ONLY = new Set(['warp', 'hermes', 'grokbot', 'muse']);
|
|
12
|
+
export function wrapperControls(provider) {
|
|
13
|
+
return NATIVE_CONTROLS[provider] ?? {};
|
|
14
|
+
}
|
|
15
|
+
export function isConcreteWrapperModel(model) {
|
|
16
|
+
return Boolean(model && !/^(?:configured\/|Hint\(|auto$|inherit(?:ed)?$|default$|profile-selected$|global-delegation-model$)/i.test(model));
|
|
17
|
+
}
|
|
18
|
+
export function resolveWrapperPin(provider, wrapper, options = {}) {
|
|
19
|
+
const controls = wrapperControls(provider);
|
|
20
|
+
const catalog = options.catalog ?? loadProviderModelCatalog();
|
|
21
|
+
let tier = wrapper.tier;
|
|
22
|
+
let role = wrapper.role;
|
|
23
|
+
const compile = () => compileModelPolicy({ provider, artifact: 'agent', catalog,
|
|
24
|
+
policy: { role, tier, effort: options.effort ?? wrapper.effort,
|
|
25
|
+
...(options.override ? { override: options.override } : {}) } });
|
|
26
|
+
let policy = compile();
|
|
27
|
+
// Auto can select a flagship; every economy wrapper needs the concrete fallback.
|
|
28
|
+
if (controls.model && policy.effectiveModel === 'auto' && !options.override) {
|
|
29
|
+
tier = 'standard';
|
|
30
|
+
role = 'coding';
|
|
31
|
+
policy = compile();
|
|
32
|
+
}
|
|
33
|
+
if (controls.model && options.override && !isConcreteWrapperModel(options.override)) {
|
|
34
|
+
throw new Error(`${provider}: wrapper override must be a concrete model pin`);
|
|
35
|
+
}
|
|
36
|
+
const model = controls.model && isConcreteWrapperModel(policy.effectiveModel) ? policy.effectiveModel : undefined;
|
|
37
|
+
const effort = model && controls.effort ? policy.fields[controls.effort] : undefined;
|
|
38
|
+
if (model && controls.effort && !effort)
|
|
39
|
+
throw new Error(`${provider}: unsupported wrapper effort`);
|
|
40
|
+
return { tier, role, model, effort, controls,
|
|
41
|
+
enforcement: (GLOBAL_ONLY.has(provider) ? 'global-only' : model ? 'pinned' : 'inherited') };
|
|
42
|
+
}
|
|
43
|
+
/** Static artifact defaults; this does not inspect live sessions or assert runtime enforcement. */
|
|
44
|
+
export function wrapperEnforcementReport(provider) {
|
|
45
|
+
const catalog = loadProviderModelCatalog();
|
|
46
|
+
const controls = wrapperControls(provider);
|
|
47
|
+
const known = provider in catalog.providers;
|
|
48
|
+
const states = known ? MODEL_WRAPPER_DEFINITIONS.map(wrapper => resolveWrapperPin(provider, wrapper, { catalog }).enforcement) : [];
|
|
49
|
+
const state = GLOBAL_ONLY.has(provider) ? 'global-only'
|
|
50
|
+
: states.length > 0 && states.every(value => value === 'pinned') ? 'pinned' : 'inherited';
|
|
51
|
+
const effort = state === 'pinned' && controls.effort
|
|
52
|
+
? `pinned via ${controls.effort} (high/medium/low by wrapper)` : 'not enforced per wrapper';
|
|
53
|
+
const details = {
|
|
54
|
+
claude: 'Use AIWG wrappers or an explicit invocation model. Unpinned general-purpose agents inherit; CLAUDE_CODE_SUBAGENT_MODEL supplies a fallback and CLAUDE_CODE_SUBAGENT_MODEL_FORCE=1 can override pins.',
|
|
55
|
+
omp: 'Native model and thinkingLevel fields can pin exact operator role models; configured/* catalog entries are placeholders.',
|
|
56
|
+
'grok-build': 'Not enforced: native-agent model translation remains gated by #2577; configured selectors do not establish tier enforcement.',
|
|
57
|
+
openclaw: 'Markdown wrapper mirrors do not set native subagents.model or per-spawn overrides.',
|
|
58
|
+
openhuman: 'User harness wrappers carry semantic model hints; project wrappers omit model. Neither is an exact pin.',
|
|
59
|
+
antigravity: 'Wrapper mirrors do not project native CLI model/effort flags.',
|
|
60
|
+
windsurf: 'Portable wrapper model selection is unsupported; the main UI session controls selection.',
|
|
61
|
+
pi: 'Agents are reference skills; CLI model/thinking flags are separate from deployed wrappers.',
|
|
62
|
+
'deepseek-harness': 'Agent references do not pin the configured Harness route.',
|
|
63
|
+
hermes: 'All delegated workers share the provider delegation configuration.',
|
|
64
|
+
warp: 'Model selection follows the Warp run/profile.',
|
|
65
|
+
grokbot: 'No verified AIWG-owned per-worker model setting is available.',
|
|
66
|
+
muse: 'Session --model or settings.json model controls selection; per-agent pins are unverified.',
|
|
67
|
+
};
|
|
68
|
+
return `Tier enforcement (generated wrapper defaults): ${state}; effort: ${effort}. `
|
|
69
|
+
+ `${details[provider] ?? (known ? 'Native custom subagents carry explicit catalog model selectors.' : 'Provider wrapper controls are unverified.')} `
|
|
70
|
+
+ 'Explicit operator pins are preserved where supported. This is not a live resolved-model audit; provider/account policy may substitute models.';
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=wrapper-enforcement.js.map
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// Checks host-classified effects and requires a verified signed receipt before suppressing a workflow prompt.
|
|
2
|
+
import { lstat, realpath } from 'node:fs/promises';
|
|
3
|
+
import { basename, dirname, isAbsolute, relative, resolve } from 'node:path';
|
|
4
|
+
import { effectSession, openLedger } from '../effects/host.js';
|
|
5
|
+
import { loadHitlLayers } from './hitl-config.js';
|
|
6
|
+
import { recordHitlAuthorization } from './hitl-receipt.js';
|
|
7
|
+
import { resolveHitlAuthorization, validHitlAction } from './hitl.js';
|
|
8
|
+
/** Resolve existing ancestors as well as new leaves; dangling links fail closed. */
|
|
9
|
+
async function canonical(path) {
|
|
10
|
+
try {
|
|
11
|
+
return await realpath(path);
|
|
12
|
+
}
|
|
13
|
+
catch (error) {
|
|
14
|
+
if (error.code !== 'ENOENT')
|
|
15
|
+
throw error;
|
|
16
|
+
try {
|
|
17
|
+
await lstat(path);
|
|
18
|
+
throw new Error('Unresolved action symlink');
|
|
19
|
+
}
|
|
20
|
+
catch (statError) {
|
|
21
|
+
if (statError.code !== 'ENOENT')
|
|
22
|
+
throw statError;
|
|
23
|
+
}
|
|
24
|
+
const parent = dirname(path);
|
|
25
|
+
if (parent === path)
|
|
26
|
+
throw error;
|
|
27
|
+
return resolve(await canonical(parent), basename(path));
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
const policyPath = (path) => /(^|\/)(aiwg\.config|AGENTS\.md|WORKSPACE\.md|AIWG\.md|CLAUDE\.md)$/.test(path)
|
|
31
|
+
|| /(^|\/)(rules|policies|hooks)(\/|$)/.test(path)
|
|
32
|
+
|| /^(\.git|\.claude|\.codex|\.agents|\.github|\.gitea)(\/|$)/.test(path);
|
|
33
|
+
const secretPath = (path) => /(^|\/)(\.env(?:\.[^/]*)?|\.ssh|\.aws|credentials?|tokens?|secrets?)(\/|$)/i.test(path);
|
|
34
|
+
/** Input is a host descriptor, never an issue, model response, or authorization claim. */
|
|
35
|
+
export async function canonicalHitlAction(projectDir, input) {
|
|
36
|
+
if (!validHitlAction(input))
|
|
37
|
+
return input;
|
|
38
|
+
const root = await realpath(projectDir);
|
|
39
|
+
const resources = [];
|
|
40
|
+
for (const path of input.resources) {
|
|
41
|
+
const resource = relative(root, await canonical(resolve(root, path))).split('\\').join('/');
|
|
42
|
+
if (!resource || resource === '..' || resource.startsWith('../') || isAbsolute(resource))
|
|
43
|
+
return null;
|
|
44
|
+
resources.push(resource);
|
|
45
|
+
}
|
|
46
|
+
const action = { ...input, resources: [...new Set(resources)].sort() };
|
|
47
|
+
// Check both spellings: an alias must not hide a protected resource.
|
|
48
|
+
const paths = [...input.resources, ...resources];
|
|
49
|
+
if (paths.some(secretPath))
|
|
50
|
+
action.category = 'secret.access';
|
|
51
|
+
else if (action.category !== 'repo.read' && paths.some(policyPath))
|
|
52
|
+
action.category = 'policy.override';
|
|
53
|
+
return action;
|
|
54
|
+
}
|
|
55
|
+
/** No receipt, no automatic action. Errors retain the caller's existing review path. */
|
|
56
|
+
export async function checkHitlAuthorization(projectDir, input, options = {}) {
|
|
57
|
+
let context = { now: new Date().toISOString(), layers: [], ...options };
|
|
58
|
+
try {
|
|
59
|
+
const loaded = await loadHitlLayers(projectDir);
|
|
60
|
+
context = { ...context, layers: loaded.layers };
|
|
61
|
+
const action = await canonicalHitlAction(loaded.projectDir, input);
|
|
62
|
+
const decision = resolveHitlAuthorization(action, context);
|
|
63
|
+
if (decision.decision !== 'auto-authorized')
|
|
64
|
+
return decision;
|
|
65
|
+
// Reload policies and the clock at consumption time, including expiry and revocation.
|
|
66
|
+
context = { ...context, now: new Date().toISOString(), layers: (await loadHitlLayers(projectDir)).layers,
|
|
67
|
+
expectedPolicyVersion: options.expectedPolicyVersion ?? decision.policyVersion };
|
|
68
|
+
const ledger = openLedger(await effectSession(loaded.projectDir, 'custom'));
|
|
69
|
+
const recorded = await recordHitlAuthorization(ledger, action, context);
|
|
70
|
+
if (recorded.resolution.decision !== 'auto-authorized')
|
|
71
|
+
return recorded.resolution;
|
|
72
|
+
if (recorded.reconciled.result.verification.result !== 'present')
|
|
73
|
+
throw new Error('Unverified HITL receipt');
|
|
74
|
+
return { ...recorded.resolution, receipt: { file: recorded.file, effectId: recorded.intent.effectId } };
|
|
75
|
+
}
|
|
76
|
+
catch {
|
|
77
|
+
return { ...resolveHitlAuthorization(null, context), decision: 'escalate', reason: 'authorization-check-unavailable' };
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=hitl-check.js.map
|
|
@@ -336,7 +336,7 @@ const BUILT_IN_SEEDS = [
|
|
|
336
336
|
paths: {
|
|
337
337
|
deployTarget: 'project',
|
|
338
338
|
artifacts: {
|
|
339
|
-
agents: '.agents/agents', commands: null, skills: '.agents/.aiwg/skills', rules:
|
|
339
|
+
agents: '.agents/agents', commands: null, skills: '.agents/.aiwg/skills', rules: '.antigravity/rules', behaviors: null,
|
|
340
340
|
},
|
|
341
341
|
kernelSkills: '.agents/skills',
|
|
342
342
|
contextDiscovery: { agents: '.agents/agents', skills: '.agents/skills', rules: null, behaviors: null },
|
|
@@ -344,7 +344,7 @@ const BUILT_IN_SEEDS = [
|
|
|
344
344
|
contextFiles: { aiwgMd: true, agentsMd: true, claudeMdHook: false, hookFile: null, contextFile: 'AGENTS.md' },
|
|
345
345
|
},
|
|
346
346
|
smithPaths: {
|
|
347
|
-
agents: '.agents/agents', commands: null, skills: '.agents/skills', rules:
|
|
347
|
+
agents: '.agents/agents', commands: null, skills: '.agents/skills', rules: '.antigravity/rules',
|
|
348
348
|
fileExtension: '.md', configFile: 'AGENTS.md', aggregated: false,
|
|
349
349
|
},
|
|
350
350
|
skillNamespace: {
|
|
@@ -704,7 +704,7 @@ const BUILT_IN_SEEDS = [
|
|
|
704
704
|
agents: '.grok/agents',
|
|
705
705
|
commands: null,
|
|
706
706
|
skills: '.grok/skills',
|
|
707
|
-
rules:
|
|
707
|
+
rules: '.grok/rules',
|
|
708
708
|
behaviors: null,
|
|
709
709
|
},
|
|
710
710
|
kernelSkills: '.grok/skills',
|
|
@@ -721,7 +721,7 @@ const BUILT_IN_SEEDS = [
|
|
|
721
721
|
agents: '.grok/agents',
|
|
722
722
|
commands: null,
|
|
723
723
|
skills: '.grok/skills',
|
|
724
|
-
rules:
|
|
724
|
+
rules: '.grok/rules',
|
|
725
725
|
fileExtension: '.md',
|
|
726
726
|
configFile: '.grok/config.toml',
|
|
727
727
|
aggregated: false,
|
|
@@ -768,7 +768,7 @@ const BUILT_IN_SEEDS = [
|
|
|
768
768
|
agents: null,
|
|
769
769
|
commands: null,
|
|
770
770
|
skills: resolveHermesHomePath('skills'),
|
|
771
|
-
rules:
|
|
771
|
+
rules: '.hermes/rules',
|
|
772
772
|
fileExtension: '.md',
|
|
773
773
|
configFile: 'AGENTS.md',
|
|
774
774
|
aggregated: false,
|
|
@@ -810,12 +810,12 @@ const BUILT_IN_SEEDS = [
|
|
|
810
810
|
paths: {
|
|
811
811
|
deployTarget: 'mixed',
|
|
812
812
|
artifacts: {
|
|
813
|
-
// Skills are
|
|
814
|
-
// indexed
|
|
813
|
+
// Skills are native; rule files are reached through the AGENTS.md
|
|
814
|
+
// hook. Agents and commands remain indexed through discover/show.
|
|
815
815
|
agents: null,
|
|
816
816
|
commands: null,
|
|
817
817
|
skills: '.agents/skills',
|
|
818
|
-
rules:
|
|
818
|
+
rules: '.muse/rules',
|
|
819
819
|
behaviors: null,
|
|
820
820
|
},
|
|
821
821
|
kernelSkills: '.agents/skills',
|
|
@@ -832,7 +832,7 @@ const BUILT_IN_SEEDS = [
|
|
|
832
832
|
agents: null,
|
|
833
833
|
commands: null,
|
|
834
834
|
skills: '.agents/skills',
|
|
835
|
-
rules:
|
|
835
|
+
rules: '.muse/rules',
|
|
836
836
|
fileExtension: '.md',
|
|
837
837
|
configFile: null,
|
|
838
838
|
aggregated: false,
|
|
@@ -1099,7 +1099,7 @@ const BUILT_IN_SEEDS = [
|
|
|
1099
1099
|
agents: '.agents/skills',
|
|
1100
1100
|
commands: '.pi/prompts',
|
|
1101
1101
|
skills: '.pi/skills',
|
|
1102
|
-
rules:
|
|
1102
|
+
rules: '.pi/rules',
|
|
1103
1103
|
fileExtension: '.md',
|
|
1104
1104
|
configFile: 'AGENTS.md',
|
|
1105
1105
|
aggregated: false,
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* @module research/clients/arxiv
|
|
5
5
|
*/
|
|
6
6
|
import { BaseClient } from './base.js';
|
|
7
|
+
import { normalizeDoi, parseArxivId } from '../identity.js';
|
|
7
8
|
import { ResearchError, ResearchErrorCode, } from '../types.js';
|
|
8
9
|
/**
|
|
9
10
|
* arXiv API client
|
|
@@ -33,7 +34,7 @@ export class ArxivClient extends BaseClient {
|
|
|
33
34
|
*/
|
|
34
35
|
async getPaperById(arxivId) {
|
|
35
36
|
// Normalize arXiv ID (remove version if present)
|
|
36
|
-
const normalizedId = arxivId
|
|
37
|
+
const normalizedId = parseArxivId(arxivId)?.id ?? arxivId;
|
|
37
38
|
const url = this.buildUrl('/query', {
|
|
38
39
|
id_list: normalizedId,
|
|
39
40
|
max_results: 1,
|
|
@@ -125,7 +126,10 @@ export class ArxivClient extends BaseClient {
|
|
|
125
126
|
});
|
|
126
127
|
};
|
|
127
128
|
const id = extractTag('id');
|
|
128
|
-
const
|
|
129
|
+
const identity = parseArxivId(id);
|
|
130
|
+
if (!identity)
|
|
131
|
+
throw new ResearchError(ResearchErrorCode.RF_301, 'Invalid arXiv entry identifier');
|
|
132
|
+
const arxivId = identity.id;
|
|
129
133
|
const title = extractTag('title').replace(/\s+/g, ' ');
|
|
130
134
|
const summary = extractTag('summary').replace(/\s+/g, ' ');
|
|
131
135
|
const published = extractTag('published');
|
|
@@ -133,8 +137,8 @@ export class ArxivClient extends BaseClient {
|
|
|
133
137
|
const authorNames = extractAllTags('name');
|
|
134
138
|
const authors = authorNames.map((name) => ({ name }));
|
|
135
139
|
// Extract DOI if present
|
|
136
|
-
const doiMatch = entry.match(
|
|
137
|
-
const doi = doiMatch ? doiMatch[1] : undefined;
|
|
140
|
+
const doiMatch = entry.match(/<arxiv:doi[^>]*>([^<]+)<\/arxiv:doi>/i);
|
|
141
|
+
const doi = doiMatch ? normalizeDoi(doiMatch[1]) : undefined;
|
|
138
142
|
// Extract PDF link
|
|
139
143
|
const pdfLinkMatch = entry.match(/<link[^>]*title="pdf"[^>]*href="([^"]+)"/i);
|
|
140
144
|
const pdfUrl = pdfLinkMatch ? pdfLinkMatch[1] : undefined;
|
|
@@ -145,6 +149,8 @@ export class ArxivClient extends BaseClient {
|
|
|
145
149
|
year,
|
|
146
150
|
abstract: summary,
|
|
147
151
|
arxivId,
|
|
152
|
+
arxivVersion: identity.version,
|
|
153
|
+
publicationDate: published || undefined,
|
|
148
154
|
doi,
|
|
149
155
|
pdfUrl,
|
|
150
156
|
type: 'preprint',
|
|
@@ -70,8 +70,6 @@ export class BaseClient {
|
|
|
70
70
|
* Execute an HTTP request with rate limiting and retry logic
|
|
71
71
|
*/
|
|
72
72
|
async request(url, options = {}) {
|
|
73
|
-
// Apply rate limiting
|
|
74
|
-
await this.rateLimiter.acquire();
|
|
75
73
|
// Execute with retry
|
|
76
74
|
return this.executeWithRetry(url, options);
|
|
77
75
|
}
|
|
@@ -79,6 +77,8 @@ export class BaseClient {
|
|
|
79
77
|
* Execute request with exponential backoff retry
|
|
80
78
|
*/
|
|
81
79
|
async executeWithRetry(url, options, attempt = 0, firstError) {
|
|
80
|
+
// Every attempt consumes a token, after any retry backoff and before timeout.
|
|
81
|
+
await this.rateLimiter.acquire();
|
|
82
82
|
const controller = new AbortController();
|
|
83
83
|
let timeoutId;
|
|
84
84
|
try {
|
|
@@ -184,8 +184,9 @@ export class BaseClient {
|
|
|
184
184
|
/**
|
|
185
185
|
* Build URL with query parameters
|
|
186
186
|
*/
|
|
187
|
-
buildUrl(path, params) {
|
|
188
|
-
|
|
187
|
+
buildUrl(path, params, baseUrl = this.config.baseUrl) {
|
|
188
|
+
// API endpoints are relative to the complete base path, even with a slash.
|
|
189
|
+
const url = new URL(path.replace(/^\/+/, ''), `${baseUrl.replace(/\/+$/, '')}/`);
|
|
189
190
|
Object.entries(params).forEach(([key, value]) => {
|
|
190
191
|
if (value !== undefined) {
|
|
191
192
|
url.searchParams.append(key, String(value));
|
|
@@ -39,9 +39,16 @@ export class CrossRefClient extends BaseClient {
|
|
|
39
39
|
* Search papers by query
|
|
40
40
|
*/
|
|
41
41
|
async search(query, options = {}) {
|
|
42
|
+
return this.queryWorks('query', query, options);
|
|
43
|
+
}
|
|
44
|
+
/** Resolve a partial citation using CrossRef's bibliographic search field. */
|
|
45
|
+
async searchBibliographic(query, options = {}) {
|
|
46
|
+
return this.queryWorks('query.bibliographic', query, options);
|
|
47
|
+
}
|
|
48
|
+
async queryWorks(field, query, options) {
|
|
42
49
|
const { limit = 10, offset = 0 } = options;
|
|
43
50
|
const url = this.buildUrl('/works', {
|
|
44
|
-
query,
|
|
51
|
+
[field]: query,
|
|
45
52
|
rows: limit,
|
|
46
53
|
offset,
|
|
47
54
|
});
|
|
@@ -80,6 +87,7 @@ export class CrossRefClient extends BaseClient {
|
|
|
80
87
|
title,
|
|
81
88
|
authors,
|
|
82
89
|
year,
|
|
90
|
+
publicationDate: response.published?.['date-parts']?.[0]?.map((part, i) => String(part).padStart(i === 0 ? 4 : 2, '0')).join('-'),
|
|
83
91
|
abstract: response.abstract,
|
|
84
92
|
doi: response.DOI,
|
|
85
93
|
venue,
|
|
@@ -4,18 +4,21 @@
|
|
|
4
4
|
* @module research/clients/semantic-scholar
|
|
5
5
|
*/
|
|
6
6
|
import { BaseClient } from './base.js';
|
|
7
|
+
import { ResearchError, ResearchErrorCode, } from '../types.js';
|
|
8
|
+
const IDENTITY_FIELDS = 'paperId,title,abstract,year,authors,citationCount,venue,publicationDate,publicationVenue,externalIds,openAccessPdf';
|
|
7
9
|
/**
|
|
8
10
|
* Semantic Scholar API client
|
|
9
11
|
*/
|
|
10
12
|
export class SemanticScholarClient extends BaseClient {
|
|
13
|
+
recommendationsBaseUrl;
|
|
11
14
|
constructor(config) {
|
|
12
15
|
super({
|
|
13
16
|
baseUrl: 'https://api.semanticscholar.org/graph/v1',
|
|
14
17
|
timeout: 30000,
|
|
15
18
|
rateLimit: {
|
|
16
|
-
maxTokens:
|
|
19
|
+
maxTokens: 1,
|
|
17
20
|
refillRate: 1, // 1 request per second
|
|
18
|
-
currentTokens:
|
|
21
|
+
currentTokens: 1,
|
|
19
22
|
lastRefill: Date.now(),
|
|
20
23
|
},
|
|
21
24
|
retry: {
|
|
@@ -25,65 +28,86 @@ export class SemanticScholarClient extends BaseClient {
|
|
|
25
28
|
backoffMultiplier: 2,
|
|
26
29
|
},
|
|
27
30
|
...config,
|
|
31
|
+
apiKey: config?.apiKey ?? process.env.S2_API_KEY,
|
|
28
32
|
});
|
|
33
|
+
this.recommendationsBaseUrl = config?.recommendationsBaseUrl ??
|
|
34
|
+
'https://api.semanticscholar.org/recommendations/v1';
|
|
35
|
+
}
|
|
36
|
+
/** Request raw JSON in an explicit namespace; baseUrl overrides graph only. */
|
|
37
|
+
async requestApi(namespace, path, params = {}) {
|
|
38
|
+
const baseUrl = namespace === 'graph'
|
|
39
|
+
? this.config.baseUrl
|
|
40
|
+
: this.recommendationsBaseUrl;
|
|
41
|
+
return this.request(this.buildUrl(path, params, baseUrl));
|
|
42
|
+
}
|
|
43
|
+
/** Attach credentials once for all endpoints and keep transport causes private. */
|
|
44
|
+
async request(url, options = {}) {
|
|
45
|
+
const key = this.config.apiKey;
|
|
46
|
+
try {
|
|
47
|
+
const headers = new Headers(options.headers);
|
|
48
|
+
if (key)
|
|
49
|
+
headers.set('x-api-key', key);
|
|
50
|
+
return await super.request(url, { ...options, headers });
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
// A transport error can echo request headers, including in its cause/stack.
|
|
54
|
+
if (key) {
|
|
55
|
+
throw new ResearchError(error instanceof ResearchError ? error.code : ResearchErrorCode.RF_500, error instanceof ResearchError
|
|
56
|
+
? error.message.replaceAll(key, '[REDACTED]')
|
|
57
|
+
: 'Semantic Scholar request failed');
|
|
58
|
+
}
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
29
61
|
}
|
|
30
62
|
/**
|
|
31
63
|
* Get paper by DOI
|
|
32
64
|
*/
|
|
33
65
|
async getPaperByDoi(doi) {
|
|
34
|
-
|
|
35
|
-
'paperId',
|
|
36
|
-
'title',
|
|
37
|
-
'abstract',
|
|
38
|
-
'year',
|
|
39
|
-
'authors',
|
|
40
|
-
'citationCount',
|
|
41
|
-
'venue',
|
|
42
|
-
'externalIds',
|
|
43
|
-
'openAccessPdf',
|
|
44
|
-
];
|
|
45
|
-
const url = this.buildUrl(`/paper/DOI:${encodeURIComponent(doi)}`, {
|
|
46
|
-
fields: fields.join(','),
|
|
47
|
-
});
|
|
48
|
-
const response = await this.request(url);
|
|
49
|
-
return this.transformResponse(response);
|
|
66
|
+
return this.getPaperById(`DOI:${doi}`);
|
|
50
67
|
}
|
|
51
68
|
/**
|
|
52
69
|
* Get paper by arXiv ID
|
|
53
70
|
*/
|
|
54
71
|
async getPaperByArxivId(arxivId) {
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
'externalIds',
|
|
64
|
-
'openAccessPdf',
|
|
65
|
-
];
|
|
66
|
-
const url = this.buildUrl(`/paper/ARXIV:${encodeURIComponent(arxivId)}`, {
|
|
67
|
-
fields: fields.join(','),
|
|
68
|
-
});
|
|
69
|
-
const response = await this.request(url);
|
|
72
|
+
return this.getPaperById(`ARXIV:${arxivId}`);
|
|
73
|
+
}
|
|
74
|
+
/** Graph lookup by paperId, DOI:, ARXIV:, ACL:, CorpusId: or URL: identifier. */
|
|
75
|
+
async getPaperById(identifier) {
|
|
76
|
+
const colon = identifier.indexOf(':');
|
|
77
|
+
const encoded = colon < 0 ? encodeURIComponent(identifier)
|
|
78
|
+
: `${identifier.slice(0, colon + 1)}${encodeURIComponent(identifier.slice(colon + 1))}`;
|
|
79
|
+
const response = await this.requestApi('graph', `/paper/${encoded}`, { fields: IDENTITY_FIELDS });
|
|
70
80
|
return this.transformResponse(response);
|
|
71
81
|
}
|
|
82
|
+
/** Title matching uses the graph match endpoint, retaining every returned candidate. */
|
|
83
|
+
async matchTitle(query) {
|
|
84
|
+
const response = await this.requestApi('graph', '/paper/search/match', {
|
|
85
|
+
query, fields: IDENTITY_FIELDS,
|
|
86
|
+
});
|
|
87
|
+
return response.data.map(paper => this.transformResponse(paper));
|
|
88
|
+
}
|
|
89
|
+
/** Similar papers share the normal transport, key header, limiter and retries. */
|
|
90
|
+
async recommendations(identifier, limit = 20, from = 'recent') {
|
|
91
|
+
const colon = identifier.indexOf(':');
|
|
92
|
+
const encoded = `${identifier.slice(0, colon + 1)}${encodeURIComponent(identifier.slice(colon + 1))}`;
|
|
93
|
+
const response = await this.requestApi('recommendations', `/papers/forpaper/${encoded}`, { limit, from, fields: IDENTITY_FIELDS });
|
|
94
|
+
if (!Array.isArray(response.recommendedPapers) || response.recommendedPapers.some(p => !p?.paperId || !p.title)) {
|
|
95
|
+
throw new ResearchError(ResearchErrorCode.RF_301, 'Invalid recommendations response');
|
|
96
|
+
}
|
|
97
|
+
return response.recommendedPapers.map(paper => this.transformResponse(paper));
|
|
98
|
+
}
|
|
99
|
+
/** One graph lookup for each candidate selected for optional citation overlap. */
|
|
100
|
+
async recommendationReferences(paperId) {
|
|
101
|
+
const response = await this.requestApi('graph', `/paper/${encodeURIComponent(paperId)}`, { fields: 'references.paperId,references.title,references.externalIds' });
|
|
102
|
+
if (!Array.isArray(response.references))
|
|
103
|
+
throw new ResearchError(ResearchErrorCode.RF_301, 'Invalid references response');
|
|
104
|
+
return response.references.filter(paper => paper?.paperId).map(paper => this.transformResponse(paper));
|
|
105
|
+
}
|
|
72
106
|
/**
|
|
73
107
|
* Search papers by query
|
|
74
108
|
*/
|
|
75
109
|
async search(query, options = {}) {
|
|
76
|
-
const { limit = 10, offset = 0, fields =
|
|
77
|
-
'paperId',
|
|
78
|
-
'title',
|
|
79
|
-
'abstract',
|
|
80
|
-
'year',
|
|
81
|
-
'authors',
|
|
82
|
-
'citationCount',
|
|
83
|
-
'venue',
|
|
84
|
-
'externalIds',
|
|
85
|
-
'openAccessPdf',
|
|
86
|
-
], } = options;
|
|
110
|
+
const { limit = 10, offset = 0, fields = IDENTITY_FIELDS.split(','), } = options;
|
|
87
111
|
const url = this.buildUrl('/paper/search', {
|
|
88
112
|
query,
|
|
89
113
|
limit,
|
|
@@ -109,12 +133,17 @@ export class SemanticScholarClient extends BaseClient {
|
|
|
109
133
|
}));
|
|
110
134
|
return {
|
|
111
135
|
id: response.paperId,
|
|
136
|
+
s2PaperId: response.paperId,
|
|
112
137
|
title: response.title,
|
|
113
138
|
authors,
|
|
114
139
|
year: response.year || 0,
|
|
115
140
|
abstract: response.abstract,
|
|
116
141
|
doi: response.externalIds?.DOI,
|
|
117
142
|
arxivId: response.externalIds?.ArXiv,
|
|
143
|
+
aclId: response.externalIds?.ACL,
|
|
144
|
+
openReviewId: response.externalIds?.OpenReview,
|
|
145
|
+
publicationDate: response.publicationDate ?? undefined,
|
|
146
|
+
publicationVenue: response.publicationVenue ?? undefined,
|
|
118
147
|
venue: response.venue,
|
|
119
148
|
citationCount: response.citationCount,
|
|
120
149
|
pdfUrl: response.openAccessPdf?.url,
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
const ARXIV = '(?:\\d{4}\\.\\d{4,5}|[a-z][a-z.-]*(?:\\.[A-Z]{2})?/\\d{7})(?:v\\d+)?';
|
|
2
|
+
/** Parse abs/pdf/html URLs, arXiv DOIs, new/old IDs and an optional version. */
|
|
3
|
+
export function parseArxivId(input) {
|
|
4
|
+
const text = input.trim().replace(/\.pdf(?:[?#].*)?$/i, '').replace(/[?#].*$/, '');
|
|
5
|
+
const match = text.match(new RegExp(`^(?:(?:https?://(?:export\\.)?arxiv\\.org/(?:abs|pdf|html)/)|(?:https?://(?:dx\\.)?doi\\.org/)?10\\.48550/arxiv\\.|arxiv:\\s*)?(${ARXIV})$`, 'i'));
|
|
6
|
+
if (!match)
|
|
7
|
+
return undefined;
|
|
8
|
+
const version = match[1].match(/v(\d+)$/i);
|
|
9
|
+
return { id: match[1].replace(/v\d+$/i, '').toLowerCase(), ...(version ? { version: Number(version[1]) } : {}) };
|
|
10
|
+
}
|
|
11
|
+
/** Canonical DOI spelling, including DOI URLs; trailing citation punctuation is ignored. */
|
|
12
|
+
export function normalizeDoi(input) {
|
|
13
|
+
let text = input.trim().replace(/^(?:https?:\/\/(?:dx\.)?doi\.org\/|doi:\s*)/i, '');
|
|
14
|
+
try {
|
|
15
|
+
text = decodeURIComponent(text);
|
|
16
|
+
}
|
|
17
|
+
catch { /* Keep a literal DOI on malformed escapes. */ }
|
|
18
|
+
text = text.replace(/[.,;]+$/, '');
|
|
19
|
+
while (text.endsWith(')') && (text.match(/\)/g)?.length ?? 0) > (text.match(/\(/g)?.length ?? 0))
|
|
20
|
+
text = text.slice(0, -1);
|
|
21
|
+
return /^10\.\d{4,9}\/\S+$/i.test(text) ? text.toLowerCase() : undefined;
|
|
22
|
+
}
|
|
23
|
+
/** Parse a recognized identifier even when embedded in a bibliography line. */
|
|
24
|
+
export function parseResearchIdentifier(input) {
|
|
25
|
+
const arxiv = input.match(new RegExp(`(?:https?://(?:export\\.)?arxiv\\.org/(?:abs|pdf|html)/|(?:https?://(?:dx\\.)?doi\\.org/)?10\\.48550/arxiv\\.|arxiv:\\s*)${ARXIV}(?:\\.pdf)?(?:[?#][^\\s<>]*)?|(?<![\\w./])${ARXIV}(?![\\w.])`, 'i'));
|
|
26
|
+
if (arxiv) {
|
|
27
|
+
const parsed = parseArxivId(arxiv[0]);
|
|
28
|
+
if (parsed)
|
|
29
|
+
return { kind: 'arxiv', ...parsed, matched: arxiv[0] };
|
|
30
|
+
}
|
|
31
|
+
const doi = input.match(/(?:https?:\/\/(?:dx\.)?doi\.org\/|doi:\s*)?10\.\d{4,9}\/[^\s<>"\]]+/i);
|
|
32
|
+
if (doi) {
|
|
33
|
+
const id = normalizeDoi(doi[0]);
|
|
34
|
+
if (id)
|
|
35
|
+
return { kind: 'doi', id, matched: doi[0] };
|
|
36
|
+
}
|
|
37
|
+
const review = input.match(/https?:\/\/(?:www\.)?openreview\.net\/(?:forum|pdf)\?[^\s<>]*/i);
|
|
38
|
+
if (review) {
|
|
39
|
+
const id = new URL(review[0]).searchParams.get('id');
|
|
40
|
+
if (id)
|
|
41
|
+
return { kind: 'openreview', id, matched: review[0] };
|
|
42
|
+
}
|
|
43
|
+
const reviewId = input.match(/\bopenreview:\s*([\w-]+)/i);
|
|
44
|
+
if (reviewId)
|
|
45
|
+
return { kind: 'openreview', id: reviewId[1], matched: reviewId[0] };
|
|
46
|
+
const acl = input.match(/https?:\/\/(?:www\.)?aclanthology\.org\/([\w.-]+)\/?(?:[?#][^\s<>]*)?|\bacl:\s*([\w.-]+)|\b((?:\d{4}\.[a-z][\w-]*\.\d+|[A-Z]\d{2}-\d{4}))\b/i);
|
|
47
|
+
if (acl)
|
|
48
|
+
return { kind: 'acl', id: (acl[1] ?? acl[2] ?? acl[3]).replace(/\.pdf$/i, ''), matched: acl[0] };
|
|
49
|
+
const s2 = input.match(/https?:\/\/(?:www\.)?semanticscholar\.org\/paper\/(?:[^\s/]+\/)?([a-f\d]{40})\b|\b(?:s2|semanticscholar):\s*([a-f\d]{40})\b|\b(CorpusId:\d+)\b/i);
|
|
50
|
+
if (s2)
|
|
51
|
+
return { kind: 's2', id: (s2[1] ?? s2[2] ?? s2[3]).toLowerCase().replace(/^corpusid:/, 'CorpusId:'), matched: s2[0] };
|
|
52
|
+
return undefined;
|
|
53
|
+
}
|
|
54
|
+
/** Remove LaTeX formatting while retaining its text, then normalize Unicode and punctuation. */
|
|
55
|
+
export function normalizeTitle(input) {
|
|
56
|
+
return input.normalize('NFKC')
|
|
57
|
+
.replace(/\\(?:begin|end)\{[^}]*\}/g, ' ')
|
|
58
|
+
.replace(/\\[a-zA-Z]+\*?/g, '')
|
|
59
|
+
.replace(/\\([^a-zA-Z])/g, '$1')
|
|
60
|
+
.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, ' ')
|
|
61
|
+
.replace(/\s+/g, ' ').trim();
|
|
62
|
+
}
|
|
63
|
+
/** Accept both corpus frontmatter spellings and canonical client metadata. */
|
|
64
|
+
export function normalizeIdentity(value) {
|
|
65
|
+
const fields = value;
|
|
66
|
+
const str = (...keys) => keys.map(k => fields[k])
|
|
67
|
+
.map(v => typeof v === 'number' && Number.isFinite(v) ? String(v) : v)
|
|
68
|
+
.find(v => typeof v === 'string' && v.trim());
|
|
69
|
+
const arxiv = parseArxivId(str('arxivId', 'arxiv_id', 'arxiv') ?? '') ?? parseArxivId(str('doi') ?? '');
|
|
70
|
+
return {
|
|
71
|
+
...(arxiv ? { arxivId: arxiv.id, arxivVersion: arxiv.version ?? (typeof fields.arxivVersion === 'number' ? fields.arxivVersion : undefined) } : {}),
|
|
72
|
+
doi: normalizeDoi(str('doi', 'DOI') ?? ''),
|
|
73
|
+
s2PaperId: str('s2PaperId', 's2_paper_id', 'paperId')?.toLowerCase(),
|
|
74
|
+
openReviewId: str('openReviewId', 'openreview_id'),
|
|
75
|
+
aclId: str('aclId', 'acl_id'),
|
|
76
|
+
title: str('title') ? normalizeTitle(str('title')) : undefined,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/** Stable keys for matching any identifier or a normalized title across corpus sources. */
|
|
80
|
+
export function identityKeys(value) {
|
|
81
|
+
const identity = normalizeIdentity(value);
|
|
82
|
+
return ['arxivId', 'doi', 's2PaperId', 'openReviewId', 'aclId', 'title']
|
|
83
|
+
.flatMap(key => identity[key] ? [`${key}:${identity[key]}`] : []);
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=identity.js.map
|