@inneranimalmedia/agentsam-sdk 2.5.0 → 2.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTSAM.md +55 -0
- package/README.md +12 -8
- package/bin/agentsam +2 -0
- package/docs/AGENTSAM_ASTRA_OPENAI_INTEGRATION.md +1363 -0
- package/docs/CLI_SHELL.md +163 -53
- package/docs/RELEASES.md +16 -7
- package/package.json +20 -8
- package/packages/connectors/cloudflare/package.json +10 -0
- package/packages/connectors/cloudflare/src/index.js +127 -0
- package/packages/connectors/cloudflare/src/owner.js +76 -0
- package/packages/connectors/cloudflare/src/routes.js +223 -0
- package/packages/connectors/cloudflare/src/vault.js +80 -0
- package/packages/connectors/cloudflare/tests/connector.test.mjs +44 -0
- package/packages/identity/package.json +2 -2
- package/packages/identity/src/contracts/auth-config.js +18 -7
- package/packages/identity/tests/auth-config.test.mjs +9 -5
- package/packages/identity/tests/oauth-credentials.test.mjs +4 -4
- package/protocol/README.md +1 -0
- package/protocol/capabilities/cloudflare-cpu-audit-input.schema.json +19 -0
- package/protocol/capabilities/cloudflare-cpu-profile-input.schema.json +13 -0
- package/protocol/capabilities/cloudflare-wrangler-native-input.schema.json +19 -0
- package/protocol/capabilities/manifest.json +47 -0
- package/protocol/context/context-budget.schema.json +10 -15
- package/protocol/context/context-item.schema.json +4 -5
- package/protocol/context/resolved-context-pack.schema.json +19 -14
- package/protocol/models/README.md +373 -0
- package/protocol/models/model-inventory-v2.schema.json +212 -0
- package/skills/agentsam-cloudflare-workers/SKILL.md +53 -0
- package/skills/agentsam-cloudflare-workers/references/cpu-profiling.md +16 -0
- package/skills/agentsam-cloudflare-workers/references/errors-and-observability.md +29 -0
- package/skills/agentsam-cloudflare-workers/references/wrangler-native-map.md +28 -0
- package/skills/catalog.json +18 -0
- package/src/agent/capability-adapter.js +25 -13
- package/src/agent/index.js +1 -0
- package/src/agent/responses-runner.js +325 -0
- package/src/cli.js +98 -28
- package/src/cloudflare/cpu-profile.js +115 -0
- package/src/cloudflare/index.js +14 -0
- package/src/cloudflare/wrangler.js +132 -0
- package/src/commands/account-auth.js +47 -0
- package/src/commands/cloudflare.js +58 -0
- package/src/commands/connections.js +93 -0
- package/src/commands/context-economics.js +114 -0
- package/src/commands/deploy.js +39 -3
- package/src/commands/eval.js +63 -0
- package/src/commands/interactive.js +2 -5
- package/src/commands/models.js +85 -40
- package/src/commands/preferences.js +101 -59
- package/src/commands/resume.js +67 -0
- package/src/commands/security.js +5 -3
- package/src/commands/shell.js +370 -109
- package/src/commands/tunnel.js +2 -2
- package/src/commands/whoami.js +86 -0
- package/src/context/budget.js +68 -6
- package/src/context/index.js +3 -1
- package/src/context/rehydrate.js +35 -0
- package/src/context/resolve.js +44 -12
- package/src/errors/diagnostic.js +160 -0
- package/src/errors/index.js +9 -0
- package/src/eval/context.js +191 -0
- package/src/eval/index.js +1 -0
- package/src/index.js +55 -1
- package/src/lib/account-session.js +98 -0
- package/src/lib/agent-instructions.js +73 -0
- package/src/lib/auth.js +4 -0
- package/src/lib/cli-preferences.js +28 -24
- package/src/lib/deploy/git-guard.js +69 -0
- package/src/lib/deploy/health.js +57 -0
- package/src/lib/deploy/local-studio.js +283 -0
- package/src/lib/deploy/secret-scan.js +65 -0
- package/src/lib/detect-context.js +2 -2
- package/src/lib/execution-approvals.js +59 -0
- package/src/lib/local-sessions.js +127 -0
- package/src/lib/provider-credentials.js +83 -0
- package/src/lib/scaffold/templates/worker-api/index.js +101 -20
- package/src/lib/scaffold/wizards/worker-api.js +27 -11
- package/src/lib/slash-commands.js +22 -16
- package/src/models/catalog.js +135 -0
- package/src/models/index.js +7 -0
- package/src/providers/index.js +5 -0
- package/src/providers/openai-responses.js +275 -0
- package/src/security/process.js +35 -9
- package/src/telemetry/contracts.js +203 -0
- package/src/telemetry/events.js +48 -0
- package/src/telemetry/index.js +8 -0
- package/src/tools/hydrate.js +35 -0
- package/src/tools/index.js +1 -0
- package/src/ui/boot.js +15 -17
- package/test/account-session.test.mjs +36 -0
- package/test/cli-preferences.test.mjs +26 -5
- package/test/cloudflare-connector.test.mjs +96 -0
- package/test/cloudflare-runtime.test.mjs +75 -0
- package/test/context.test.mjs +61 -12
- package/test/deploy-health-scan.test.mjs +67 -0
- package/test/error-diagnostics.test.mjs +59 -0
- package/test/eval-context.test.mjs +37 -0
- package/test/execution-approvals.test.mjs +27 -0
- package/test/local-sessions.test.mjs +42 -0
- package/test/local-studio-deploy.test.mjs +83 -0
- package/test/model-catalog.test.mjs +43 -0
- package/test/models.test.mjs +30 -16
- package/test/npm10-lock.test.mjs +29 -0
- package/test/openai-responses.test.mjs +95 -0
- package/test/provider-credentials.test.mjs +52 -0
- package/test/rehydrate.test.mjs +25 -0
- package/test/release-hygiene.test.mjs +4 -4
- package/test/responses-runner.test.mjs +148 -0
- package/test/shell.test.mjs +47 -20
- package/test/smoke.mjs +4 -1
- package/test/telemetry.test.mjs +79 -0
- package/test/tools-search.test.mjs +14 -1
- package/test/whoami-resume.test.mjs +56 -0
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Operator diagnostics for AgentSam connections (Cloudflare account grant).
|
|
3
|
+
* Identity (who is this user?) is separate from the Cloudflare connector.
|
|
4
|
+
*/
|
|
5
|
+
import {
|
|
6
|
+
CLOUDFLARE_CALLBACK_PATH,
|
|
7
|
+
CLOUDFLARE_FIXTURE_CLIENT_ID,
|
|
8
|
+
cloudflareConnectionSafeStatus,
|
|
9
|
+
resolveCloudflareOAuthClient,
|
|
10
|
+
} from '../../packages/connectors/cloudflare/src/index.js';
|
|
11
|
+
import { resolveIamIssuer } from '../../packages/identity/src/contracts/auth-config.js';
|
|
12
|
+
|
|
13
|
+
const PRODUCTION_CALLBACK = `https://agentsam.inneranimalmedia.com${CLOUDFLARE_CALLBACK_PATH}`;
|
|
14
|
+
|
|
15
|
+
function doctor(env = process.env) {
|
|
16
|
+
const iam = {
|
|
17
|
+
issuer: resolveIamIssuer(env),
|
|
18
|
+
clientId: Boolean(String(env.IAM_CLIENT_ID || '').trim()),
|
|
19
|
+
serverSecret: Boolean(String(env.IAM_CLIENT_SECRET || '').trim()),
|
|
20
|
+
originAlias: Boolean(String(env.IAM_ORIGIN || '').trim()),
|
|
21
|
+
};
|
|
22
|
+
const cf = cloudflareConnectionSafeStatus(env);
|
|
23
|
+
const client = resolveCloudflareOAuthClient(env);
|
|
24
|
+
return { identity: iam, cloudflare: cf, client };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function printSetup(env = process.env) {
|
|
28
|
+
const client = resolveCloudflareOAuthClient(env);
|
|
29
|
+
console.log('Cloudflare connector setup');
|
|
30
|
+
console.log('');
|
|
31
|
+
console.log(` callback: ${PRODUCTION_CALLBACK}`);
|
|
32
|
+
console.log(' authorize: https://dash.cloudflare.com/oauth2/auth');
|
|
33
|
+
console.log(' token: https://dash.cloudflare.com/oauth2/token');
|
|
34
|
+
console.log(' revoke: https://dash.cloudflare.com/oauth2/revoke');
|
|
35
|
+
console.log('');
|
|
36
|
+
console.log('This CLI does not mint a real Cloudflare OAuth client.');
|
|
37
|
+
console.log('Fixture client id ' + CLOUDFLARE_FIXTURE_CLIENT_ID + ' is local-only and must never be installed on production.');
|
|
38
|
+
if (client.fixture) {
|
|
39
|
+
console.log('STOP: fixture credentials are loaded. OAuth start will return 503.');
|
|
40
|
+
} else if (client.status === 'not_configured') {
|
|
41
|
+
console.log('status: not_configured — production may deploy; connector stays optional.');
|
|
42
|
+
} else {
|
|
43
|
+
console.log(`status: ${client.status}`);
|
|
44
|
+
}
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function runConnections(args = []) {
|
|
49
|
+
const argv = args.filter((a) => a !== '--json');
|
|
50
|
+
const jsonMode = args.includes('--json');
|
|
51
|
+
const env = process.env;
|
|
52
|
+
if (argv[0] === 'cloudflare' && argv[1] === 'setup') {
|
|
53
|
+
if (jsonMode) {
|
|
54
|
+
console.log(JSON.stringify({
|
|
55
|
+
callback: PRODUCTION_CALLBACK,
|
|
56
|
+
authorize: 'https://dash.cloudflare.com/oauth2/auth',
|
|
57
|
+
token: 'https://dash.cloudflare.com/oauth2/token',
|
|
58
|
+
revoke: 'https://dash.cloudflare.com/oauth2/revoke',
|
|
59
|
+
mintsRealClient: false,
|
|
60
|
+
client: resolveCloudflareOAuthClient(env),
|
|
61
|
+
}, null, 2));
|
|
62
|
+
return 0;
|
|
63
|
+
}
|
|
64
|
+
return printSetup(env);
|
|
65
|
+
}
|
|
66
|
+
const report = doctor(env);
|
|
67
|
+
if (jsonMode) {
|
|
68
|
+
console.log(JSON.stringify({ identity: report.identity, cloudflare: report.cloudflare }, null, 2));
|
|
69
|
+
return 0;
|
|
70
|
+
}
|
|
71
|
+
const iam = report.identity;
|
|
72
|
+
const cf = report.cloudflare;
|
|
73
|
+
const client = report.client;
|
|
74
|
+
console.log('AgentSam Identity');
|
|
75
|
+
console.log('');
|
|
76
|
+
console.log('IAM client');
|
|
77
|
+
console.log(` ${iam.clientId ? '✓' : '•'} client id`);
|
|
78
|
+
console.log(` ✓ issuer ${iam.issuer}`);
|
|
79
|
+
console.log(` ${iam.serverSecret ? '✓' : '•'} server secret configured`);
|
|
80
|
+
if (iam.originAlias) console.log('Compatibility\n IAM_ORIGIN -> deprecated alias');
|
|
81
|
+
console.log('');
|
|
82
|
+
console.log('Cloudflare connection');
|
|
83
|
+
console.log('');
|
|
84
|
+
console.log('OAuth client');
|
|
85
|
+
console.log(` client id: ${cf.clientId}`);
|
|
86
|
+
console.log(` secret: ${cf.secret}`);
|
|
87
|
+
console.log(` status: ${client.status}`);
|
|
88
|
+
if (client.fixture) {
|
|
89
|
+
console.log(' OAuth client fixture configured');
|
|
90
|
+
console.log(' real Cloudflare OAuth client still required');
|
|
91
|
+
}
|
|
92
|
+
return 0;
|
|
93
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { createContextBudget, assessContextUsage } from '../context/index.js';
|
|
2
|
+
import { getModelRecord } from '../models/index.js';
|
|
3
|
+
import { readCliPreferences } from '../lib/cli-preferences.js';
|
|
4
|
+
|
|
5
|
+
function formatInteger(value) {
|
|
6
|
+
return Number(value).toLocaleString('en-US');
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function percent(value) {
|
|
10
|
+
return `${(Number(value) * 100).toFixed(1)}%`;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function buildContextEconomicsReport(cwd, options = {}) {
|
|
14
|
+
const preferences = options.preferences || readCliPreferences(cwd) || {};
|
|
15
|
+
const model = getModelRecord(preferences.modelPreference);
|
|
16
|
+
if (!model) {
|
|
17
|
+
return Object.freeze({
|
|
18
|
+
model: preferences.modelPreference || 'auto',
|
|
19
|
+
resolved: false,
|
|
20
|
+
reason: 'Select an exact catalog model with /model before Agent Sam can calculate model-specific context economics.',
|
|
21
|
+
reasoning_effort: preferences.reasoningEffort || 'auto',
|
|
22
|
+
service_tier: preferences.serviceTier || 'default',
|
|
23
|
+
active_input_tokens: null,
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const policy = model.context_policy || {};
|
|
28
|
+
const budget = createContextBudget({
|
|
29
|
+
windowTokens: model.context_window,
|
|
30
|
+
targetInputTokens: policy.target_input_tokens,
|
|
31
|
+
compactAtTokens: policy.compact_at_tokens,
|
|
32
|
+
interveneAtTokens: policy.intervene_at_tokens,
|
|
33
|
+
maxNormalInputTokens: policy.max_normal_input_tokens,
|
|
34
|
+
pricingThresholdTokens: policy.pricing_threshold_tokens,
|
|
35
|
+
safetyMarginTokens: policy.safety_margin_tokens,
|
|
36
|
+
});
|
|
37
|
+
const active = Number.isFinite(options.activeInputTokens) && options.activeInputTokens >= 0
|
|
38
|
+
? Math.floor(options.activeInputTokens)
|
|
39
|
+
: null;
|
|
40
|
+
const pressure = active == null ? null : assessContextUsage(active, budget);
|
|
41
|
+
|
|
42
|
+
return Object.freeze({
|
|
43
|
+
model: model.provider_model_id,
|
|
44
|
+
model_key: model.model_key,
|
|
45
|
+
resolved: true,
|
|
46
|
+
reasoning_effort: preferences.reasoningEffort || 'auto',
|
|
47
|
+
service_tier: preferences.serviceTier || 'default',
|
|
48
|
+
active_input_tokens: active,
|
|
49
|
+
estimate_kind: options.estimateKind === 'provider' ? 'provider' : active == null ? null : 'local',
|
|
50
|
+
window_tokens: budget.windowTokens,
|
|
51
|
+
utilization_ratio: pressure?.utilizationRatio ?? null,
|
|
52
|
+
target_input_tokens: budget.targetInputTokens,
|
|
53
|
+
compact_at_tokens: budget.compactAtTokens,
|
|
54
|
+
intervene_at_tokens: budget.interveneAtTokens,
|
|
55
|
+
max_normal_input_tokens: budget.maxNormalInputTokens,
|
|
56
|
+
pricing_threshold_tokens: budget.pricingThresholdTokens,
|
|
57
|
+
tokens_until_pricing_threshold: pressure?.tokensUntilPricingThreshold ?? null,
|
|
58
|
+
pressure: pressure?.stage ?? 'unknown',
|
|
59
|
+
should_compact: pressure?.shouldCompact ?? false,
|
|
60
|
+
should_intervene: pressure?.shouldIntervene ?? false,
|
|
61
|
+
pricing_threshold_crossed: pressure?.pricingThresholdCrossed ?? false,
|
|
62
|
+
batch: model.batch,
|
|
63
|
+
pricing_source: model.pricing?.source || null,
|
|
64
|
+
pricing_as_of: model.pricing?.as_of || null,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function renderContextEconomics(report) {
|
|
69
|
+
if (!report.resolved) {
|
|
70
|
+
return [
|
|
71
|
+
'',
|
|
72
|
+
' Agent Sam · context',
|
|
73
|
+
` model ${report.model}`,
|
|
74
|
+
` reasoning ${report.reasoning_effort}`,
|
|
75
|
+
` processing ${report.service_tier}`,
|
|
76
|
+
'',
|
|
77
|
+
` ${report.reason}`,
|
|
78
|
+
'',
|
|
79
|
+
].join('\n');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const active = report.active_input_tokens == null
|
|
83
|
+
? 'unavailable · no provider/local usage snapshot yet'
|
|
84
|
+
: `${report.estimate_kind === 'provider' ? '' : '~'}${formatInteger(report.active_input_tokens)}${report.utilization_ratio == null ? '' : ` · ${percent(report.utilization_ratio)} of window`}`;
|
|
85
|
+
const remaining = report.tokens_until_pricing_threshold == null
|
|
86
|
+
? 'unavailable until active usage is known'
|
|
87
|
+
: `${report.tokens_until_pricing_threshold < 0 ? '-' : ''}${formatInteger(Math.abs(report.tokens_until_pricing_threshold))}`;
|
|
88
|
+
return [
|
|
89
|
+
'',
|
|
90
|
+
' Agent Sam · context',
|
|
91
|
+
` model ${report.model}`,
|
|
92
|
+
` reasoning ${report.reasoning_effort}`,
|
|
93
|
+
` processing ${report.service_tier}`,
|
|
94
|
+
'',
|
|
95
|
+
' Context',
|
|
96
|
+
` active ${active}`,
|
|
97
|
+
` window ${formatInteger(report.window_tokens)}`,
|
|
98
|
+
'',
|
|
99
|
+
' Working-set policy',
|
|
100
|
+
` target ${formatInteger(report.target_input_tokens)}`,
|
|
101
|
+
` compact at ${formatInteger(report.compact_at_tokens)}`,
|
|
102
|
+
` intervene at ${formatInteger(report.intervene_at_tokens)}`,
|
|
103
|
+
` max normal ${formatInteger(report.max_normal_input_tokens)}`,
|
|
104
|
+
'',
|
|
105
|
+
' Economics',
|
|
106
|
+
` price threshold ${formatInteger(report.pricing_threshold_tokens)}`,
|
|
107
|
+
` remaining ${remaining}`,
|
|
108
|
+
` pricing as-of ${report.pricing_as_of || 'unknown'}`,
|
|
109
|
+
'',
|
|
110
|
+
' Batch is a separate asynchronous execution lane; it is not an interactive service tier.',
|
|
111
|
+
' Use `/context repo` for repository/Git bridge context.',
|
|
112
|
+
'',
|
|
113
|
+
].join('\n');
|
|
114
|
+
}
|
package/src/commands/deploy.js
CHANGED
|
@@ -6,8 +6,9 @@ import path from 'node:path';
|
|
|
6
6
|
import readline from 'node:readline';
|
|
7
7
|
import { authenticateViaBrowser } from '../lib/auth.js';
|
|
8
8
|
import { getJson, streamScaffold } from '../lib/core-client.js';
|
|
9
|
-
import {
|
|
9
|
+
import { resolveAccountSdkKey } from '../lib/account-session.js';
|
|
10
10
|
import { getDefaultProfile, getDeployTarget, getLocalSchemaPath, getProjectName, getProjectPreset, readProjectConfig, setDeployTarget, writeProjectConfig } from '../lib/project-config.js';
|
|
11
|
+
import { isLocalStudioCheckout, runLocalStudioDeploy } from '../lib/deploy/local-studio.js';
|
|
11
12
|
|
|
12
13
|
function writeCloudflareAdapter(cwd, config, cf) {
|
|
13
14
|
const projectName = getProjectName(config, path.basename(cwd));
|
|
@@ -67,7 +68,7 @@ function ask(question) {
|
|
|
67
68
|
async function runCloudflareDeploy(cwd, config, accountId) {
|
|
68
69
|
console.log('\n Cloudflare deploy — browser sign-in + resource provisioning…\n');
|
|
69
70
|
|
|
70
|
-
let token =
|
|
71
|
+
let token = resolveAccountSdkKey({ env: process.env }).value;
|
|
71
72
|
if (!token) {
|
|
72
73
|
const session = await authenticateViaBrowser();
|
|
73
74
|
token = session.access_token;
|
|
@@ -122,10 +123,45 @@ async function runCloudflareDeploy(cwd, config, accountId) {
|
|
|
122
123
|
}
|
|
123
124
|
|
|
124
125
|
/**
|
|
125
|
-
* @param {{ cwd?: string, target?: string, accountId?: string }} [opts]
|
|
126
|
+
* @param {{ cwd?: string, target?: string, accountId?: string, dryRun?: boolean, plan?: boolean }} [opts]
|
|
126
127
|
*/
|
|
127
128
|
export async function runDeploy(opts = {}) {
|
|
128
129
|
const cwd = path.resolve(opts.cwd || process.cwd());
|
|
130
|
+
|
|
131
|
+
if (isLocalStudioCheckout(cwd)) {
|
|
132
|
+
const result = await runLocalStudioDeploy({
|
|
133
|
+
cwd,
|
|
134
|
+
dryRun: Boolean(opts.dryRun),
|
|
135
|
+
planOnly: Boolean(opts.plan),
|
|
136
|
+
execute: !opts.plan,
|
|
137
|
+
});
|
|
138
|
+
const payload = {
|
|
139
|
+
provider: result.plan.provider,
|
|
140
|
+
app: result.plan.app,
|
|
141
|
+
wranglerConfig: result.plan.wranglerConfig,
|
|
142
|
+
wranglerArgs: result.plan.wranglerArgs,
|
|
143
|
+
cwd: result.plan.cwd,
|
|
144
|
+
envLoaded: result.plan.envLoaded,
|
|
145
|
+
fingerprint: result.plan.fingerprint,
|
|
146
|
+
skip: result.plan.skip,
|
|
147
|
+
dryRun: Boolean(opts.dryRun),
|
|
148
|
+
plan: Boolean(opts.plan),
|
|
149
|
+
genericRootDeploy: false,
|
|
150
|
+
receipt: result.receipt,
|
|
151
|
+
};
|
|
152
|
+
console.log(JSON.stringify(payload, null, 2));
|
|
153
|
+
if (result.plan.skip) {
|
|
154
|
+
console.log('skip: unchanged deploy fingerprint');
|
|
155
|
+
} else if (opts.plan) {
|
|
156
|
+
console.log('plan only — no wrangler deploy');
|
|
157
|
+
} else if (opts.dryRun) {
|
|
158
|
+
console.log('wrangler dry-run complete');
|
|
159
|
+
} else {
|
|
160
|
+
console.log('local-studio deploy complete');
|
|
161
|
+
}
|
|
162
|
+
return result;
|
|
163
|
+
}
|
|
164
|
+
|
|
129
165
|
const config = readProjectConfig(cwd);
|
|
130
166
|
|
|
131
167
|
let target = opts.target || getDeployTarget(config) || 'cloudflare';
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { evaluateContextFixture, listContextEvalFixtures } from '../eval/index.js';
|
|
2
|
+
|
|
3
|
+
function parse(argv) {
|
|
4
|
+
const out = { subcommand: argv[0] || '', fixture: '', strategy: 'all', model: 'gpt-6-astra', json: false, list: false };
|
|
5
|
+
for (let i = 1; i < argv.length; i += 1) {
|
|
6
|
+
const arg = argv[i];
|
|
7
|
+
if (arg === '--fixture') out.fixture = argv[++i] || '';
|
|
8
|
+
else if (arg === '--strategy') out.strategy = argv[++i] || 'all';
|
|
9
|
+
else if (arg === '--model') out.model = argv[++i] || 'gpt-6-astra';
|
|
10
|
+
else if (arg === '--json') out.json = true;
|
|
11
|
+
else if (arg === '--list') out.list = true;
|
|
12
|
+
else if (arg === '--help' || arg === '-h') out.help = true;
|
|
13
|
+
else throw new Error(`unknown eval option: ${arg}`);
|
|
14
|
+
}
|
|
15
|
+
return out;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function render(report) {
|
|
19
|
+
const rows = [
|
|
20
|
+
'',
|
|
21
|
+
' AgentSam · Context Eval',
|
|
22
|
+
'',
|
|
23
|
+
` fixture ${report.fixture}`,
|
|
24
|
+
` model ${report.model}`,
|
|
25
|
+
` provider call no · deterministic/offline`,
|
|
26
|
+
'',
|
|
27
|
+
];
|
|
28
|
+
for (const row of report.strategies) {
|
|
29
|
+
rows.push(` ${row.strategy.toUpperCase()}`);
|
|
30
|
+
rows.push(` result ${row.result}`);
|
|
31
|
+
rows.push(` evidence ${row.required_found} / ${row.required_evidence} required · ${row.sources_selected} / ${row.sources_considered} selected`);
|
|
32
|
+
rows.push(` active context ~${row.active_context_tokens.toLocaleString('en-US')}`);
|
|
33
|
+
rows.push(` window ${row.window_tokens.toLocaleString('en-US')}`);
|
|
34
|
+
rows.push(` price threshold ${row.pricing_threshold_tokens.toLocaleString('en-US')}`);
|
|
35
|
+
rows.push(` remaining ${row.tokens_until_pricing_threshold.toLocaleString('en-US')}`);
|
|
36
|
+
rows.push(` tool schemas ${row.hydrated_tools} · ${row.tool_schema_chars.toLocaleString('en-US')} chars`);
|
|
37
|
+
rows.push(` compacted ${row.compacted_chars.toLocaleString('en-US')} chars`);
|
|
38
|
+
rows.push(` rehydrated ${row.rehydrated_refs.length ? row.rehydrated_refs.join(', ') : 'none'}`);
|
|
39
|
+
rows.push(` est input cost $${row.estimated_input_cost_usd.toFixed(4)}`);
|
|
40
|
+
rows.push('');
|
|
41
|
+
}
|
|
42
|
+
rows.push(` winner ${report.winner}`);
|
|
43
|
+
rows.push(` scoring ${report.scoring.join(' → ')}`);
|
|
44
|
+
rows.push('');
|
|
45
|
+
return rows.join('\n');
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export async function runEval(argv = [], options = {}) {
|
|
49
|
+
const args = parse(argv);
|
|
50
|
+
const write = options.write || (text => process.stdout.write(text));
|
|
51
|
+
if (args.help || args.subcommand !== 'context') {
|
|
52
|
+
write('agentsam eval context --fixture <name> [--strategy bounded|discovery|compact|all] [--model gpt-6-astra] [--json]\n');
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
if (args.list) {
|
|
56
|
+
const value = { fixtures: listContextEvalFixtures() };
|
|
57
|
+
write(args.json ? `${JSON.stringify(value)}\n` : `${value.fixtures.join('\n')}\n`);
|
|
58
|
+
return value;
|
|
59
|
+
}
|
|
60
|
+
const report = await evaluateContextFixture({ fixture: args.fixture || 'exact-symbol-callers', strategy: args.strategy, model: args.model });
|
|
61
|
+
write(args.json ? `${JSON.stringify(report)}\n` : render(report));
|
|
62
|
+
return report;
|
|
63
|
+
}
|
|
@@ -7,15 +7,12 @@ export async function runInteractive(options = {}) {
|
|
|
7
7
|
let identity = detectCliProject(options.cwd || process.cwd());
|
|
8
8
|
let preferences = readCliPreferences(identity.root);
|
|
9
9
|
|
|
10
|
-
if (!preferences) {
|
|
10
|
+
if (!preferences || preferences.trustedDirectory !== true) {
|
|
11
11
|
const configured = await configureCliPreferences({ cwd: identity.root, firstRun: true });
|
|
12
12
|
identity = configured.identity;
|
|
13
13
|
preferences = configured.preferences;
|
|
14
14
|
}
|
|
15
15
|
|
|
16
16
|
await runBootScene({ identity, preferences, animate: options.animate !== false });
|
|
17
|
-
await runShell([], {
|
|
18
|
-
cwd: identity.root,
|
|
19
|
-
intro: 'quiet',
|
|
20
|
-
});
|
|
17
|
+
await runShell([], { cwd: identity.root, intro: 'quiet' });
|
|
21
18
|
}
|
package/src/commands/models.js
CHANGED
|
@@ -1,33 +1,81 @@
|
|
|
1
1
|
import pc from 'picocolors';
|
|
2
2
|
import { probeOllama, resolveOllamaConfig } from './ollama.js';
|
|
3
|
+
import { listModelCatalog } from '../models/index.js';
|
|
4
|
+
import { resolveProviderCredential } from '../lib/provider-credentials.js';
|
|
3
5
|
|
|
4
6
|
const API_PROVIDERS = Object.freeze([
|
|
5
7
|
{ id: 'openai', label: 'OpenAI', credential: 'OPENAI_API_KEY' },
|
|
6
8
|
{ id: 'gemini', label: 'Gemini', credential: 'GEMINI_API_KEY' },
|
|
7
9
|
{ id: 'grok', label: 'Grok', credential: 'XAI_API_KEY' },
|
|
10
|
+
{ id: 'anthropic', label: 'Anthropic', credential: 'ANTHROPIC_API_KEY' },
|
|
8
11
|
]);
|
|
9
12
|
|
|
10
|
-
function clean(value) {
|
|
11
|
-
|
|
12
|
-
}
|
|
13
|
+
function clean(value) { return value == null ? '' : String(value).trim(); }
|
|
14
|
+
function configured(value) { return Boolean(clean(value)); }
|
|
13
15
|
|
|
14
|
-
function
|
|
15
|
-
|
|
16
|
+
async function discoverOpenAIModels(apiKey, fetchImpl) {
|
|
17
|
+
if (!apiKey) return { attempted: false, ok: false, models: [], error: null };
|
|
18
|
+
try {
|
|
19
|
+
const response = await fetchImpl('https://api.openai.com/v1/models', {
|
|
20
|
+
headers: { authorization: `Bearer ${apiKey}` },
|
|
21
|
+
signal: typeof AbortSignal?.timeout === 'function' ? AbortSignal.timeout(8_000) : undefined,
|
|
22
|
+
});
|
|
23
|
+
if (!response.ok) return { attempted: true, ok: false, models: [], error: `HTTP ${response.status}` };
|
|
24
|
+
const body = await response.json();
|
|
25
|
+
const models = Array.isArray(body?.data) ? body.data.map((row) => clean(row?.id)).filter(Boolean) : [];
|
|
26
|
+
return { attempted: true, ok: true, models, error: null };
|
|
27
|
+
} catch (error) {
|
|
28
|
+
return { attempted: true, ok: false, models: [], error: error?.message || String(error) };
|
|
29
|
+
}
|
|
16
30
|
}
|
|
17
31
|
|
|
18
32
|
export async function collectModelsStatus(options = {}) {
|
|
19
33
|
const env = options.env || process.env;
|
|
20
|
-
const
|
|
34
|
+
const ollamaFetchImpl = options.fetchImpl || fetch;
|
|
35
|
+
const providerFetchImpl = options.providerFetchImpl || fetch;
|
|
21
36
|
const ollamaConfig = resolveOllamaConfig({}, env);
|
|
22
|
-
const ollama = await probeOllama(ollamaConfig,
|
|
37
|
+
const ollama = await probeOllama(ollamaConfig, ollamaFetchImpl);
|
|
38
|
+
const credentials = new Map(API_PROVIDERS.map((provider) => [provider.id, resolveProviderCredential(provider.id, { env, home: options.home })]));
|
|
39
|
+
const providers = API_PROVIDERS.map((provider) => {
|
|
40
|
+
const credential = credentials.get(provider.id);
|
|
41
|
+
return {
|
|
42
|
+
...provider,
|
|
43
|
+
configured: credential?.configured === true,
|
|
44
|
+
source: credential?.source || null,
|
|
45
|
+
credentialError: credential?.error || null,
|
|
46
|
+
};
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
const openaiCredential = credentials.get('openai');
|
|
50
|
+
const shouldDiscover = options.discoverRemote !== false && Boolean(openaiCredential?.configured);
|
|
51
|
+
const openai = shouldDiscover
|
|
52
|
+
? await discoverOpenAIModels(clean(openaiCredential?.value), providerFetchImpl)
|
|
53
|
+
: { attempted: false, ok: false, models: [], error: null };
|
|
54
|
+
const availableIds = new Set(openai.models);
|
|
55
|
+
const catalogModels = listModelCatalog().map((record) => ({
|
|
56
|
+
model_key: record.model_key,
|
|
57
|
+
provider: record.provider,
|
|
58
|
+
provider_model_id: record.provider_model_id,
|
|
59
|
+
label: record.label,
|
|
60
|
+
availability: record.provider === 'openai' && openai.ok
|
|
61
|
+
? (availableIds.has(record.provider_model_id) ? 'available' : 'unavailable')
|
|
62
|
+
: 'unverified',
|
|
63
|
+
source: record.source,
|
|
64
|
+
}));
|
|
23
65
|
|
|
24
66
|
return {
|
|
25
|
-
schemaVersion: 'agentsam-model-inventory-
|
|
26
|
-
providers
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
67
|
+
schemaVersion: 'agentsam-model-inventory-v2',
|
|
68
|
+
providers,
|
|
69
|
+
discovery: {
|
|
70
|
+
openai: {
|
|
71
|
+
attempted: openai.attempted,
|
|
72
|
+
ok: openai.ok,
|
|
73
|
+
error: openai.error,
|
|
74
|
+
returnedModelCount: openai.models.length,
|
|
75
|
+
},
|
|
76
|
+
},
|
|
77
|
+
catalogModels,
|
|
78
|
+
availableModels: catalogModels.filter((row) => row.availability === 'available'),
|
|
31
79
|
local: {
|
|
32
80
|
provider: 'ollama',
|
|
33
81
|
configured: ollama.online,
|
|
@@ -41,67 +89,64 @@ export async function collectModelsStatus(options = {}) {
|
|
|
41
89
|
};
|
|
42
90
|
}
|
|
43
91
|
|
|
44
|
-
function statusMark(ok) {
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function writeLine(write, value = '') {
|
|
49
|
-
write(`${value}\n`);
|
|
50
|
-
}
|
|
92
|
+
function statusMark(ok) { return ok ? pc.green('●') : pc.dim('○'); }
|
|
93
|
+
function writeLine(write, value = '') { write(`${value}\n`); }
|
|
51
94
|
|
|
52
95
|
export function renderModelsStatus(status) {
|
|
53
96
|
const lines = [];
|
|
54
97
|
lines.push('');
|
|
55
98
|
lines.push(` ${pc.bold('Agent Sam · models')}`);
|
|
56
|
-
lines.push(` ${pc.dim('
|
|
99
|
+
lines.push(` ${pc.dim('Credential presence is local evidence; exact hosted-model availability is provider-verified when discovery succeeds.')}`);
|
|
57
100
|
lines.push('');
|
|
58
101
|
|
|
59
102
|
for (const provider of status.providers) {
|
|
60
|
-
const state = provider.configured ? pc.green('configured') : pc.dim('not configured');
|
|
61
|
-
const detail = provider.configured ?
|
|
103
|
+
const state = provider.configured ? pc.green('configured') : pc.dim(provider.credentialError ? 'blocked' : 'not configured');
|
|
104
|
+
const detail = provider.configured ? `credential available · ${provider.source || 'runtime'}` : provider.credentialError ? `${provider.credential} · ${provider.credentialError}` : provider.credential;
|
|
62
105
|
lines.push(` ${statusMark(provider.configured)} ${pc.cyan(provider.label.padEnd(10))} ${state.padEnd(20)} ${pc.dim(detail)}`);
|
|
63
106
|
}
|
|
64
107
|
|
|
108
|
+
const exact = status.availableModels || [];
|
|
109
|
+
if (exact.length) {
|
|
110
|
+
lines.push('');
|
|
111
|
+
lines.push(` ${pc.dim('provider-verified selectable models')}`);
|
|
112
|
+
for (const model of exact) lines.push(` ${pc.green('•')} ${model.provider_model_id}`);
|
|
113
|
+
} else if (status.discovery?.openai?.attempted) {
|
|
114
|
+
lines.push('');
|
|
115
|
+
lines.push(` ${pc.dim(`OpenAI discovery ${status.discovery.openai.ok ? 'completed; no catalog models matched' : `failed: ${status.discovery.openai.error || 'unknown error'}`}`)}`);
|
|
116
|
+
}
|
|
117
|
+
|
|
65
118
|
const local = status.local;
|
|
66
119
|
const localState = local.online ? pc.green('online') : pc.dim('offline');
|
|
67
120
|
lines.push(` ${statusMark(local.online)} ${pc.cyan('Ollama'.padEnd(10))} ${localState.padEnd(20)} ${pc.dim('local only')}`);
|
|
68
|
-
|
|
69
121
|
if (local.online) {
|
|
70
122
|
const names = local.models.map((row) => row.name).filter(Boolean);
|
|
71
123
|
lines.push('');
|
|
72
124
|
lines.push(` ${pc.dim('local models')}`);
|
|
73
|
-
if (names.length) {
|
|
74
|
-
|
|
75
|
-
} else {
|
|
76
|
-
lines.push(` ${pc.dim('no models reported')}`);
|
|
77
|
-
}
|
|
125
|
+
if (names.length) for (const name of names) lines.push(` ${pc.green('•')} ${name}`);
|
|
126
|
+
else lines.push(` ${pc.dim('no models reported')}`);
|
|
78
127
|
lines.push('');
|
|
79
128
|
lines.push(` ${pc.dim('chat default')} ${local.chatModel}`);
|
|
80
129
|
lines.push(` ${pc.dim('embed default')} ${local.embedModel}`);
|
|
81
130
|
}
|
|
82
|
-
|
|
83
131
|
lines.push('');
|
|
84
|
-
lines.push(` ${pc.dim('
|
|
85
|
-
lines.push(` ${pc.dim('this command only reports what this local CLI can prove is configured or available.')}`);
|
|
132
|
+
lines.push(` ${pc.dim('Use /model inside Agent Sam to choose an exact verified model, reasoning effort, and processing tier.')}`);
|
|
86
133
|
lines.push('');
|
|
87
134
|
return lines.join('\n');
|
|
88
135
|
}
|
|
89
136
|
|
|
90
137
|
export async function runModels(argv = [], options = {}) {
|
|
91
138
|
if (argv.some((arg) => arg === '--help' || arg === '-h')) {
|
|
92
|
-
const text = 'agentsam models [--json]\n\nShow
|
|
139
|
+
const text = 'agentsam models [--json] [--no-discover]\n\nShow configured providers, provider-verified known models when available, and local Ollama inventory.\n';
|
|
93
140
|
(options.write || process.stdout.write.bind(process.stdout))(text);
|
|
94
141
|
return;
|
|
95
142
|
}
|
|
96
|
-
const
|
|
143
|
+
const allowed = new Set(['--json', '--no-discover']);
|
|
144
|
+
const unknown = argv.filter((arg) => !allowed.has(arg));
|
|
97
145
|
if (unknown.length) throw new Error(`unknown models option: ${unknown[0]}`);
|
|
98
146
|
|
|
99
|
-
const status = await collectModelsStatus(options);
|
|
147
|
+
const status = await collectModelsStatus({ ...options, discoverRemote: !argv.includes('--no-discover') });
|
|
100
148
|
const write = options.write || ((text) => process.stdout.write(text));
|
|
101
|
-
if (argv.includes('--json'))
|
|
102
|
-
|
|
103
|
-
} else {
|
|
104
|
-
write(renderModelsStatus(status));
|
|
105
|
-
}
|
|
149
|
+
if (argv.includes('--json')) writeLine(write, JSON.stringify(status, null, 2));
|
|
150
|
+
else write(renderModelsStatus(status));
|
|
106
151
|
return status;
|
|
107
152
|
}
|