@netgreener/runtime 0.1.13 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/checkCli.d.ts +2 -0
- package/dist/cli/checkCli.js +121 -0
- package/dist/cli/helpText.js +1 -0
- package/dist/cli/netgreener.js +7 -0
- package/dist/externalApiMeter.d.ts +1 -0
- package/dist/externalApiMeter.js +45 -0
- package/dist/runContextEnrichment.js +44 -1
- package/dist/sdk/client.d.ts +25 -0
- package/dist/sdk/client.js +3 -0
- package/dist/sdk/index.d.ts +1 -0
- package/dist/types.d.ts +4 -0
- package/fixtures/external-api-gate/scenarios.v1.json +617 -0
- package/fixtures/release_identity_platforms.json +14 -0
- package/package.json +1 -1
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/** External usage budgets share the API evaluator with Python. */
|
|
2
|
+
import { loadRuntimeConfig } from '../config.js';
|
|
3
|
+
import { NetGreenerClient } from '../sdk/client.js';
|
|
4
|
+
export function formatCheckHelp() {
|
|
5
|
+
return `Usage: netgreener check --run-id <id> [options]
|
|
6
|
+
|
|
7
|
+
Check an uploaded run's observed token/API usage and estimated costs.
|
|
8
|
+
Options:
|
|
9
|
+
--max-total-tokens <n> Input + output token cap
|
|
10
|
+
--max-input-tokens <n> Input token cap
|
|
11
|
+
--max-output-tokens <n> Output token cap
|
|
12
|
+
--max-external-api-calls <n> Observed outbound call cap
|
|
13
|
+
--max-external-cost-usd <n> Estimated USD cap (configured server pricing)
|
|
14
|
+
--external-provider <key> Scope to provider or provider::model
|
|
15
|
+
--project-id <id> Verify the run belongs to this project
|
|
16
|
+
--run-id <id> Run ID (or NETGREENER_LAST_RUN_ID)
|
|
17
|
+
--output-json Print structured evaluation
|
|
18
|
+
--warn-only Report failures but exit 0
|
|
19
|
+
-h, --help Show help
|
|
20
|
+
|
|
21
|
+
Missing telemetry, incomplete required token coverage, or missing pricing fails.
|
|
22
|
+
Empty telemetry does not prove zero usage. OCR pricing is per call, not page.
|
|
23
|
+
Exit: 0 pass, 1 budget/data failure, 2 configuration/auth/API failure.
|
|
24
|
+
Python resource/baseline flags are not part of this Node command.
|
|
25
|
+
`;
|
|
26
|
+
}
|
|
27
|
+
export async function runCheckCli(argv, env = process.env, fetchImpl = globalThis.fetch.bind(globalThis)) {
|
|
28
|
+
try {
|
|
29
|
+
if (argv.includes('--help') || argv.includes('-h')) {
|
|
30
|
+
process.stdout.write(formatCheckHelp());
|
|
31
|
+
return 0;
|
|
32
|
+
}
|
|
33
|
+
const budgets = {};
|
|
34
|
+
const countFlags = new Map([
|
|
35
|
+
['--max-total-tokens', 'max_total_tokens'],
|
|
36
|
+
['--max-input-tokens', 'max_input_tokens'],
|
|
37
|
+
['--max-output-tokens', 'max_output_tokens'],
|
|
38
|
+
['--max-external-api-calls', 'max_external_api_calls'],
|
|
39
|
+
]);
|
|
40
|
+
let runId = Number(env.NETGREENER_LAST_RUN_ID || 0);
|
|
41
|
+
let projectId = Number(env.NETGREENER_PROJECT_ID || 0);
|
|
42
|
+
let outputJson = false;
|
|
43
|
+
let warnOnly = false;
|
|
44
|
+
for (let i = 0; i < argv.length; i++) {
|
|
45
|
+
const flag = argv[i];
|
|
46
|
+
if (flag === '--output-json') {
|
|
47
|
+
outputJson = true;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
if (flag === '--warn-only') {
|
|
51
|
+
warnOnly = true;
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (!countFlags.has(flag) && !['--max-external-cost-usd', '--external-provider', '--run-id', '--project-id'].includes(flag)) {
|
|
55
|
+
throw new Error(`Unknown option: ${flag}`);
|
|
56
|
+
}
|
|
57
|
+
const raw = argv[++i];
|
|
58
|
+
if (raw === undefined || !raw.trim())
|
|
59
|
+
throw new Error(`Missing value for ${flag}`);
|
|
60
|
+
if (flag === '--external-provider') {
|
|
61
|
+
budgets.external_provider = raw;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
const value = Number(raw);
|
|
65
|
+
if (!Number.isFinite(value) || value < 0 || (flag !== '--max-external-cost-usd' && !Number.isSafeInteger(value))) {
|
|
66
|
+
throw new Error(`Invalid nonnegative value for ${flag}`);
|
|
67
|
+
}
|
|
68
|
+
if (flag === '--run-id')
|
|
69
|
+
runId = value;
|
|
70
|
+
else if (flag === '--project-id')
|
|
71
|
+
projectId = value;
|
|
72
|
+
else if (flag === '--max-external-cost-usd')
|
|
73
|
+
budgets.max_external_cost_usd = value;
|
|
74
|
+
else
|
|
75
|
+
Object.assign(budgets, { [countFlags.get(flag)]: value });
|
|
76
|
+
}
|
|
77
|
+
if (!Object.keys(budgets).some(key => key.startsWith('max_')))
|
|
78
|
+
throw new Error('Set at least one external usage cap');
|
|
79
|
+
if (!Number.isSafeInteger(runId) || runId <= 0)
|
|
80
|
+
throw new Error('Specify --run-id or NETGREENER_LAST_RUN_ID');
|
|
81
|
+
if (!Number.isSafeInteger(projectId) || projectId < 0)
|
|
82
|
+
throw new Error('Invalid project ID');
|
|
83
|
+
const config = loadRuntimeConfig(env);
|
|
84
|
+
if (!config.token)
|
|
85
|
+
throw new Error('NETGREENER_TOKEN missing');
|
|
86
|
+
const client = new NetGreenerClient({
|
|
87
|
+
baseUrl: `${config.apiUrl}/api/v1`, token: config.token,
|
|
88
|
+
orgId: config.organizationId ?? undefined, fetchImpl,
|
|
89
|
+
});
|
|
90
|
+
if (config.organizationSlug)
|
|
91
|
+
await client.resolveOrgSlug(config.organizationSlug);
|
|
92
|
+
const result = await client.evaluateExternalApiGate(runId, budgets);
|
|
93
|
+
if (result.run_id !== runId || (projectId > 0 && result.project_id !== projectId)) {
|
|
94
|
+
throw new Error('Gate response does not match the requested run/project');
|
|
95
|
+
}
|
|
96
|
+
if (!Array.isArray(result.checks) || !result.checks.length || result.checks.some(c => !c || typeof c.passed !== 'boolean' || (c.passed && (typeof c.actual !== 'number' || !Number.isFinite(c.actual))))) {
|
|
97
|
+
throw new Error('Invalid external gate response');
|
|
98
|
+
}
|
|
99
|
+
if (Object.keys(budgets).filter(key => key.startsWith('max_')).some(key => !result.checks.some(c => c.name === key))) {
|
|
100
|
+
throw new Error('External gate response is missing requested checks');
|
|
101
|
+
}
|
|
102
|
+
const passed = result.checks.every(c => c.passed);
|
|
103
|
+
if (!passed && warnOnly)
|
|
104
|
+
result.warnings = [...(result.warnings || []), 'Gate would have failed (--warn-only)'];
|
|
105
|
+
const exitCode = passed || warnOnly ? 0 : 1;
|
|
106
|
+
if (outputJson) {
|
|
107
|
+
process.stdout.write(JSON.stringify({ ...result, passed: passed || warnOnly, exit_code: exitCode }) + '\n');
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
for (const c of result.checks)
|
|
111
|
+
process.stdout.write(`${c.passed ? 'PASS' : 'FAIL'} ${c.name}: ${c.actual ?? 'unknown'} / ${c.limit} ${c.detail}\n`);
|
|
112
|
+
for (const warning of result.warnings || [])
|
|
113
|
+
process.stdout.write(`Warning: ${warning}\n`);
|
|
114
|
+
}
|
|
115
|
+
return exitCode;
|
|
116
|
+
}
|
|
117
|
+
catch (error) {
|
|
118
|
+
process.stderr.write(`External gate error: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
119
|
+
return 2;
|
|
120
|
+
}
|
|
121
|
+
}
|
package/dist/cli/helpText.js
CHANGED
|
@@ -13,6 +13,7 @@ Commands:
|
|
|
13
13
|
runtime status [--json] Check local runtime configuration (no network)
|
|
14
14
|
analyze <projectDir> Local resource Analyze (manifest + findings)
|
|
15
15
|
run <script> [projectDir] Execute script with process metering (+ optional upload)
|
|
16
|
+
check --run-id <id> [options] Post-run token, API call, and estimated cost gates
|
|
16
17
|
optimize <projectDir> Local proposals; --llm for api_server optimize session
|
|
17
18
|
optimize <projectDir> --apply Safe local patches only (VerificationProvider)
|
|
18
19
|
optimize <projectDir> --llm Shared LLM /optimize/function (token + project-id)
|
package/dist/cli/netgreener.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* Does not flip EXPORT_MODE. Optimize never mutates the tree (see OPTIMIZE_STUB.md).
|
|
10
10
|
*/
|
|
11
|
+
import { formatCheckHelp, runCheckCli } from './checkCli.js';
|
|
11
12
|
import { resolve } from 'node:path';
|
|
12
13
|
import { pathToFileURL } from 'node:url';
|
|
13
14
|
import { runAnalyzeCli } from '../analyze/cli.js';
|
|
@@ -29,6 +30,10 @@ function printHelpFor(topic) {
|
|
|
29
30
|
process.stdout.write(formatRunHelp());
|
|
30
31
|
return 0;
|
|
31
32
|
}
|
|
33
|
+
if (key === 'check') {
|
|
34
|
+
process.stdout.write(formatCheckHelp());
|
|
35
|
+
return 0;
|
|
36
|
+
}
|
|
32
37
|
if (key === 'runtime') {
|
|
33
38
|
process.stdout.write(formatRuntimeHelp());
|
|
34
39
|
return 0;
|
|
@@ -61,6 +66,8 @@ export function runNetgreenerCli(argv) {
|
|
|
61
66
|
if (cmd === 'run') {
|
|
62
67
|
return runRunCli(rest);
|
|
63
68
|
}
|
|
69
|
+
if (cmd === 'check')
|
|
70
|
+
return runCheckCli(rest);
|
|
64
71
|
if (cmd === 'runtime') {
|
|
65
72
|
return runRuntimeCli(rest);
|
|
66
73
|
}
|
|
@@ -81,6 +81,7 @@ export declare class ExternalApiAggregator {
|
|
|
81
81
|
private providers;
|
|
82
82
|
private byServiceUnit;
|
|
83
83
|
private byTenant;
|
|
84
|
+
private byTenantServiceUnit;
|
|
84
85
|
private excludedHosts;
|
|
85
86
|
setExcludedHosts(hosts: Iterable<string>): void;
|
|
86
87
|
excludesHost(host: string): boolean;
|
package/dist/externalApiMeter.js
CHANGED
|
@@ -673,6 +673,7 @@ export class ExternalApiAggregator {
|
|
|
673
673
|
providers = new Map();
|
|
674
674
|
byServiceUnit = new Map();
|
|
675
675
|
byTenant = new Map();
|
|
676
|
+
byTenantServiceUnit = new Map();
|
|
676
677
|
excludedHosts = new Set();
|
|
677
678
|
setExcludedHosts(hosts) {
|
|
678
679
|
this.excludedHosts = new Set([...hosts].map((h) => h.toLowerCase().trim()).filter(Boolean));
|
|
@@ -819,6 +820,18 @@ export class ExternalApiAggregator {
|
|
|
819
820
|
outputTokens,
|
|
820
821
|
model,
|
|
821
822
|
});
|
|
823
|
+
let routes = this.byTenantServiceUnit.get(bucketId);
|
|
824
|
+
if (!routes) {
|
|
825
|
+
routes = new Map();
|
|
826
|
+
this.byTenantServiceUnit.set(bucketId, routes);
|
|
827
|
+
}
|
|
828
|
+
const unit = routes.has(serviceUnit) || routes.size < 32 ? serviceUnit : '__unknown__';
|
|
829
|
+
let routeBucket = routes.get(unit);
|
|
830
|
+
if (!routeBucket) {
|
|
831
|
+
routeBucket = new Map();
|
|
832
|
+
routes.set(unit, routeBucket);
|
|
833
|
+
}
|
|
834
|
+
this.recordProvider(routeBucket, { provider, host: opts.host, durationMs, error, inputTokens, outputTokens, model });
|
|
822
835
|
}
|
|
823
836
|
}
|
|
824
837
|
providerRow(p) {
|
|
@@ -900,6 +913,11 @@ export class ExternalApiAggregator {
|
|
|
900
913
|
calls: rows.reduce((s, r) => s + r.calls, 0),
|
|
901
914
|
errors: rows.reduce((s, r) => s + r.errors, 0),
|
|
902
915
|
providers: rows,
|
|
916
|
+
by_service_unit: Object.fromEntries([...(this.byTenantServiceUnit.get(tenant) ?? new Map())]
|
|
917
|
+
.map(([unit, providers]) => [unit, {
|
|
918
|
+
service_unit: unit,
|
|
919
|
+
providers: [...providers.values()].map(p => this.providerRow(p)),
|
|
920
|
+
}])),
|
|
903
921
|
};
|
|
904
922
|
}
|
|
905
923
|
const payload = {
|
|
@@ -917,10 +935,12 @@ export class ExternalApiAggregator {
|
|
|
917
935
|
snap.providers = this.providers;
|
|
918
936
|
snap.byServiceUnit = this.byServiceUnit;
|
|
919
937
|
snap.byTenant = this.byTenant;
|
|
938
|
+
snap.byTenantServiceUnit = this.byTenantServiceUnit;
|
|
920
939
|
snap.excludedHosts = new Set(this.excludedHosts);
|
|
921
940
|
this.providers = new Map();
|
|
922
941
|
this.byServiceUnit = new Map();
|
|
923
942
|
this.byTenant = new Map();
|
|
943
|
+
this.byTenantServiceUnit = new Map();
|
|
924
944
|
return snap;
|
|
925
945
|
}
|
|
926
946
|
static mergeProvider(target, source) {
|
|
@@ -1003,6 +1023,30 @@ export class ExternalApiAggregator {
|
|
|
1003
1023
|
ExternalApiAggregator.mergeProvider(dst, src);
|
|
1004
1024
|
}
|
|
1005
1025
|
}
|
|
1026
|
+
for (const [sourceTenant, routes] of snapshot.byTenantServiceUnit) {
|
|
1027
|
+
const tenant = tenantBucketId(sourceTenant, this.byTenant);
|
|
1028
|
+
let targetRoutes = this.byTenantServiceUnit.get(tenant);
|
|
1029
|
+
if (!targetRoutes) {
|
|
1030
|
+
targetRoutes = new Map();
|
|
1031
|
+
this.byTenantServiceUnit.set(tenant, targetRoutes);
|
|
1032
|
+
}
|
|
1033
|
+
for (const [sourceUnit, bucket] of routes) {
|
|
1034
|
+
const unit = targetRoutes.has(sourceUnit) || targetRoutes.size < 32 ? sourceUnit : '__unknown__';
|
|
1035
|
+
let targetBucket = targetRoutes.get(unit);
|
|
1036
|
+
if (!targetBucket) {
|
|
1037
|
+
targetBucket = new Map();
|
|
1038
|
+
targetRoutes.set(unit, targetBucket);
|
|
1039
|
+
}
|
|
1040
|
+
for (const [key, source] of bucket) {
|
|
1041
|
+
const target = targetBucket.get(key);
|
|
1042
|
+
if (target)
|
|
1043
|
+
ExternalApiAggregator.mergeProvider(target, source);
|
|
1044
|
+
else
|
|
1045
|
+
targetBucket.set(key, source);
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
}
|
|
1049
|
+
snapshot.byTenantServiceUnit = new Map();
|
|
1006
1050
|
snapshot.providers = new Map();
|
|
1007
1051
|
snapshot.byServiceUnit = new Map();
|
|
1008
1052
|
snapshot.byTenant = new Map();
|
|
@@ -1011,6 +1055,7 @@ export class ExternalApiAggregator {
|
|
|
1011
1055
|
this.providers.clear();
|
|
1012
1056
|
this.byServiceUnit.clear();
|
|
1013
1057
|
this.byTenant.clear();
|
|
1058
|
+
this.byTenantServiceUnit.clear();
|
|
1014
1059
|
}
|
|
1015
1060
|
}
|
|
1016
1061
|
const globalAgg = new ExternalApiAggregator();
|
|
@@ -39,7 +39,7 @@ function resolveCiProvider(env) {
|
|
|
39
39
|
return null;
|
|
40
40
|
}
|
|
41
41
|
function resolveReleaseTagFallback(env) {
|
|
42
|
-
|
|
42
|
+
const tag = firstEnv(env, [
|
|
43
43
|
'NETGREENER_RELEASE_TAG',
|
|
44
44
|
'RELEASE_TAG',
|
|
45
45
|
'IMAGE_TAG',
|
|
@@ -48,14 +48,52 @@ function resolveReleaseTagFallback(env) {
|
|
|
48
48
|
'CONTAINER_IMAGE_TAG',
|
|
49
49
|
'APP_VERSION',
|
|
50
50
|
'CI_COMMIT_TAG',
|
|
51
|
+
'RELEASE_RELEASENAME',
|
|
51
52
|
'BUILD_BUILDNUMBER',
|
|
52
53
|
]);
|
|
54
|
+
if (tag)
|
|
55
|
+
return tag;
|
|
56
|
+
const image = firstEnv(env, [
|
|
57
|
+
'NETGREENER_K8S_IMAGE', 'K8S_CONTAINER_IMAGE', 'CONTAINER_IMAGE', 'DOCKER_IMAGE', 'IMAGE',
|
|
58
|
+
]);
|
|
59
|
+
if (image) {
|
|
60
|
+
const digestAt = image.toLowerCase().lastIndexOf('@sha256:');
|
|
61
|
+
if (digestAt >= 0) {
|
|
62
|
+
const digest = image.slice(digestAt + 1).trim();
|
|
63
|
+
const hex = digest.slice(7).trim();
|
|
64
|
+
return hex.length >= 12 ? `sha256:${hex.slice(0, 12)}` : digest;
|
|
65
|
+
}
|
|
66
|
+
const name = image.split('/').at(-1) || '';
|
|
67
|
+
if (name.includes(':'))
|
|
68
|
+
return name.slice(name.lastIndexOf(':') + 1).trim() || null;
|
|
69
|
+
}
|
|
70
|
+
const digest = firstEnv(env, ['NETGREENER_K8S_IMAGE_DIGEST', 'K8S_IMAGE_DIGEST', 'IMAGE_DIGEST']);
|
|
71
|
+
if (digest?.toLowerCase().startsWith('sha256:') && digest.slice(7).length >= 12) {
|
|
72
|
+
return `sha256:${digest.slice(7, 19)}`;
|
|
73
|
+
}
|
|
74
|
+
if (digest)
|
|
75
|
+
return digest;
|
|
76
|
+
const vercel = firstEnv(env, ['VERCEL_DEPLOYMENT_ID', 'VERCEL_URL']);
|
|
77
|
+
if (vercel)
|
|
78
|
+
return `vercel:${vercel}`;
|
|
79
|
+
const azure = firstEnv(env, ['RELEASE_RELEASEID', 'RELEASE_DEPLOYMENTID']);
|
|
80
|
+
return azure ? `azure:${azure}` : null;
|
|
53
81
|
}
|
|
54
82
|
/**
|
|
55
83
|
* Attach release identity + CI flags onto an existing run_context object.
|
|
56
84
|
* Never overwrites an already-set ``release_tag`` / ``git_sha`` / ``deploy_id``.
|
|
57
85
|
*/
|
|
58
86
|
export function enrichRunContextForAnomaly(ctx, env = process.env) {
|
|
87
|
+
if (!(typeof ctx.deploy_environment === 'string' && ctx.deploy_environment.trim())) {
|
|
88
|
+
const explicit = firstEnv(env, ['NETGREENER_DEPLOY_ENVIRONMENT', 'DEPLOY_ENV', 'NETGREENER_ENV']);
|
|
89
|
+
const tier = explicit || firstEnv(env, ['VERCEL_ENV']);
|
|
90
|
+
const normalized = tier?.toLowerCase();
|
|
91
|
+
const mapped = normalized === 'production' ? 'prod'
|
|
92
|
+
: normalized === 'preview' ? 'staging'
|
|
93
|
+
: normalized === 'development' ? 'dev' : normalized;
|
|
94
|
+
if (mapped)
|
|
95
|
+
ctx.deploy_environment = mapped;
|
|
96
|
+
}
|
|
59
97
|
if (!(typeof ctx.release_tag === 'string' && ctx.release_tag.trim())) {
|
|
60
98
|
const tag = resolveReleaseTagFallback(env);
|
|
61
99
|
if (tag)
|
|
@@ -67,6 +105,7 @@ export function enrichRunContextForAnomaly(ctx, env = process.env) {
|
|
|
67
105
|
'GIT_SHA',
|
|
68
106
|
'GITHUB_SHA',
|
|
69
107
|
'BUILD_SOURCEVERSION',
|
|
108
|
+
'VERCEL_GIT_COMMIT_SHA',
|
|
70
109
|
]);
|
|
71
110
|
if (sha)
|
|
72
111
|
ctx.git_sha = sha.slice(0, 64);
|
|
@@ -75,6 +114,10 @@ export function enrichRunContextForAnomaly(ctx, env = process.env) {
|
|
|
75
114
|
const deployId = firstEnv(env, [
|
|
76
115
|
'NETGREENER_DEPLOY_ID',
|
|
77
116
|
'DEPLOY_ID',
|
|
117
|
+
'VERCEL_DEPLOYMENT_ID',
|
|
118
|
+
'VERCEL_URL',
|
|
119
|
+
'RELEASE_DEPLOYMENTID',
|
|
120
|
+
'RELEASE_RELEASEID',
|
|
78
121
|
'BUILD_BUILDID',
|
|
79
122
|
]);
|
|
80
123
|
if (deployId)
|
package/dist/sdk/client.d.ts
CHANGED
|
@@ -23,6 +23,30 @@ type ProjectRow = Record<string, unknown> & {
|
|
|
23
23
|
directory?: string;
|
|
24
24
|
git_url?: string | null;
|
|
25
25
|
};
|
|
26
|
+
export type ExternalApiGateBudgets = {
|
|
27
|
+
max_total_tokens?: number;
|
|
28
|
+
max_input_tokens?: number;
|
|
29
|
+
max_output_tokens?: number;
|
|
30
|
+
max_external_api_calls?: number;
|
|
31
|
+
max_external_cost_usd?: number;
|
|
32
|
+
external_provider?: string;
|
|
33
|
+
};
|
|
34
|
+
export type ExternalApiGateEvaluation = {
|
|
35
|
+
gate_version: number;
|
|
36
|
+
passed: boolean;
|
|
37
|
+
exit_code: number;
|
|
38
|
+
run_id: number;
|
|
39
|
+
project_id: number;
|
|
40
|
+
current: Record<string, unknown>;
|
|
41
|
+
checks: Array<{
|
|
42
|
+
name: string;
|
|
43
|
+
limit: number | null;
|
|
44
|
+
actual: number | null;
|
|
45
|
+
passed: boolean;
|
|
46
|
+
detail: string;
|
|
47
|
+
}>;
|
|
48
|
+
warnings: string[];
|
|
49
|
+
};
|
|
26
50
|
export declare class NetGreenerClient {
|
|
27
51
|
baseUrl: string;
|
|
28
52
|
private token;
|
|
@@ -48,6 +72,7 @@ export declare class NetGreenerClient {
|
|
|
48
72
|
login(email: string, password: string): Promise<Record<string, unknown>>;
|
|
49
73
|
getMe(): Promise<Record<string, unknown>>;
|
|
50
74
|
postRunSession(payload: Record<string, unknown>): Promise<Record<string, unknown>>;
|
|
75
|
+
evaluateExternalApiGate(runId: number, budgets: ExternalApiGateBudgets): Promise<ExternalApiGateEvaluation>;
|
|
51
76
|
getRunSession(runId: number): Promise<Record<string, unknown>>;
|
|
52
77
|
getRunOutcome(runId: number): Promise<Record<string, unknown>>;
|
|
53
78
|
attachRunOutcome(runId: number, payload: Record<string, unknown>): Promise<Record<string, unknown>>;
|
package/dist/sdk/client.js
CHANGED
|
@@ -123,6 +123,9 @@ export class NetGreenerClient {
|
|
|
123
123
|
async postRunSession(payload) {
|
|
124
124
|
return this.request('POST', '/runsessions/', payload);
|
|
125
125
|
}
|
|
126
|
+
async evaluateExternalApiGate(runId, budgets) {
|
|
127
|
+
return this.request('POST', `/runsessions/${runId}/external-gate`, budgets);
|
|
128
|
+
}
|
|
126
129
|
async getRunSession(runId) {
|
|
127
130
|
return this.request('GET', `/runsessions/${runId}`);
|
|
128
131
|
}
|
package/dist/sdk/index.d.ts
CHANGED
package/dist/types.d.ts
CHANGED
|
@@ -104,6 +104,10 @@ export type ExternalApiTenantBucket = {
|
|
|
104
104
|
calls: number;
|
|
105
105
|
errors: number;
|
|
106
106
|
providers: ExternalApiProvider[];
|
|
107
|
+
by_service_unit?: Record<string, {
|
|
108
|
+
service_unit: string;
|
|
109
|
+
providers: ExternalApiProvider[];
|
|
110
|
+
}>;
|
|
107
111
|
};
|
|
108
112
|
export type ExternalApiV0 = {
|
|
109
113
|
collector: string;
|
|
@@ -0,0 +1,617 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 1,
|
|
3
|
+
"scenarios": [
|
|
4
|
+
{
|
|
5
|
+
"name": "token_boundary",
|
|
6
|
+
"providers": [
|
|
7
|
+
{
|
|
8
|
+
"provider_key": "openai",
|
|
9
|
+
"calls": 2,
|
|
10
|
+
"input_tokens": 1000,
|
|
11
|
+
"output_tokens": 500,
|
|
12
|
+
"calls_with_usage": 2
|
|
13
|
+
}
|
|
14
|
+
],
|
|
15
|
+
"budgets": {
|
|
16
|
+
"max_total_tokens": 1500
|
|
17
|
+
},
|
|
18
|
+
"passed": true,
|
|
19
|
+
"catalog": []
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"name": "token_exceeded",
|
|
23
|
+
"providers": [
|
|
24
|
+
{
|
|
25
|
+
"provider_key": "openai",
|
|
26
|
+
"calls": 2,
|
|
27
|
+
"input_tokens": 1000,
|
|
28
|
+
"output_tokens": 500,
|
|
29
|
+
"calls_with_usage": 2
|
|
30
|
+
}
|
|
31
|
+
],
|
|
32
|
+
"budgets": {
|
|
33
|
+
"max_total_tokens": 1499
|
|
34
|
+
},
|
|
35
|
+
"passed": false,
|
|
36
|
+
"catalog": []
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"name": "input_exceeded",
|
|
40
|
+
"providers": [
|
|
41
|
+
{
|
|
42
|
+
"provider_key": "openai",
|
|
43
|
+
"calls": 2,
|
|
44
|
+
"input_tokens": 1000,
|
|
45
|
+
"output_tokens": 500,
|
|
46
|
+
"calls_with_usage": 2
|
|
47
|
+
}
|
|
48
|
+
],
|
|
49
|
+
"budgets": {
|
|
50
|
+
"max_input_tokens": 999
|
|
51
|
+
},
|
|
52
|
+
"passed": false,
|
|
53
|
+
"catalog": []
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"name": "output_boundary",
|
|
57
|
+
"providers": [
|
|
58
|
+
{
|
|
59
|
+
"provider_key": "openai",
|
|
60
|
+
"calls": 2,
|
|
61
|
+
"input_tokens": 1000,
|
|
62
|
+
"output_tokens": 500,
|
|
63
|
+
"calls_with_usage": 2
|
|
64
|
+
}
|
|
65
|
+
],
|
|
66
|
+
"budgets": {
|
|
67
|
+
"max_output_tokens": 500
|
|
68
|
+
},
|
|
69
|
+
"passed": true,
|
|
70
|
+
"catalog": []
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"name": "calls_boundary",
|
|
74
|
+
"providers": [
|
|
75
|
+
{
|
|
76
|
+
"provider_key": "openai",
|
|
77
|
+
"calls": 2,
|
|
78
|
+
"input_tokens": 1000,
|
|
79
|
+
"output_tokens": 500,
|
|
80
|
+
"calls_with_usage": 2
|
|
81
|
+
}
|
|
82
|
+
],
|
|
83
|
+
"budgets": {
|
|
84
|
+
"max_external_api_calls": 2
|
|
85
|
+
},
|
|
86
|
+
"passed": true,
|
|
87
|
+
"catalog": []
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
"name": "zero_call_budget",
|
|
91
|
+
"providers": [
|
|
92
|
+
{
|
|
93
|
+
"provider_key": "openai",
|
|
94
|
+
"calls": 2,
|
|
95
|
+
"input_tokens": 1000,
|
|
96
|
+
"output_tokens": 500,
|
|
97
|
+
"calls_with_usage": 2
|
|
98
|
+
}
|
|
99
|
+
],
|
|
100
|
+
"budgets": {
|
|
101
|
+
"max_external_api_calls": 0
|
|
102
|
+
},
|
|
103
|
+
"passed": false,
|
|
104
|
+
"catalog": []
|
|
105
|
+
},
|
|
106
|
+
{
|
|
107
|
+
"name": "cost_boundary",
|
|
108
|
+
"providers": [
|
|
109
|
+
{
|
|
110
|
+
"provider_key": "openai",
|
|
111
|
+
"calls": 2,
|
|
112
|
+
"input_tokens": 1000,
|
|
113
|
+
"output_tokens": 500,
|
|
114
|
+
"calls_with_usage": 2
|
|
115
|
+
}
|
|
116
|
+
],
|
|
117
|
+
"budgets": {
|
|
118
|
+
"max_external_cost_usd": 0.02
|
|
119
|
+
},
|
|
120
|
+
"passed": true,
|
|
121
|
+
"catalog": [
|
|
122
|
+
{
|
|
123
|
+
"provider_key": "openai",
|
|
124
|
+
"category": "llm",
|
|
125
|
+
"cost_per_call_usd": 0.01,
|
|
126
|
+
"cost_per_1k_input_tokens_usd": 0.01,
|
|
127
|
+
"cost_per_1k_output_tokens_usd": 0.02
|
|
128
|
+
}
|
|
129
|
+
]
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
"name": "cost_exceeded",
|
|
133
|
+
"providers": [
|
|
134
|
+
{
|
|
135
|
+
"provider_key": "openai",
|
|
136
|
+
"calls": 2,
|
|
137
|
+
"input_tokens": 1000,
|
|
138
|
+
"output_tokens": 500,
|
|
139
|
+
"calls_with_usage": 2
|
|
140
|
+
}
|
|
141
|
+
],
|
|
142
|
+
"budgets": {
|
|
143
|
+
"max_external_cost_usd": 0.019
|
|
144
|
+
},
|
|
145
|
+
"passed": false,
|
|
146
|
+
"catalog": [
|
|
147
|
+
{
|
|
148
|
+
"provider_key": "openai",
|
|
149
|
+
"category": "llm",
|
|
150
|
+
"cost_per_call_usd": 0.01,
|
|
151
|
+
"cost_per_1k_input_tokens_usd": 0.01,
|
|
152
|
+
"cost_per_1k_output_tokens_usd": 0.02
|
|
153
|
+
}
|
|
154
|
+
]
|
|
155
|
+
},
|
|
156
|
+
{
|
|
157
|
+
"name": "missing_telemetry",
|
|
158
|
+
"providers": null,
|
|
159
|
+
"budgets": {
|
|
160
|
+
"max_total_tokens": 9999
|
|
161
|
+
},
|
|
162
|
+
"passed": false,
|
|
163
|
+
"catalog": []
|
|
164
|
+
},
|
|
165
|
+
{
|
|
166
|
+
"name": "empty_telemetry",
|
|
167
|
+
"providers": [],
|
|
168
|
+
"budgets": {
|
|
169
|
+
"max_external_api_calls": 5
|
|
170
|
+
},
|
|
171
|
+
"passed": false,
|
|
172
|
+
"catalog": []
|
|
173
|
+
},
|
|
174
|
+
{
|
|
175
|
+
"name": "partial_token_coverage",
|
|
176
|
+
"providers": [
|
|
177
|
+
{
|
|
178
|
+
"provider_key": "openai",
|
|
179
|
+
"calls": 2,
|
|
180
|
+
"input_tokens": 1000,
|
|
181
|
+
"output_tokens": 500,
|
|
182
|
+
"calls_with_usage": 1
|
|
183
|
+
}
|
|
184
|
+
],
|
|
185
|
+
"budgets": {
|
|
186
|
+
"max_total_tokens": 9999
|
|
187
|
+
},
|
|
188
|
+
"passed": false,
|
|
189
|
+
"catalog": []
|
|
190
|
+
},
|
|
191
|
+
{
|
|
192
|
+
"name": "legacy_coverage_unknown",
|
|
193
|
+
"providers": [
|
|
194
|
+
{
|
|
195
|
+
"provider_key": "openai",
|
|
196
|
+
"calls": 2,
|
|
197
|
+
"input_tokens": 1000,
|
|
198
|
+
"output_tokens": 500
|
|
199
|
+
}
|
|
200
|
+
],
|
|
201
|
+
"budgets": {
|
|
202
|
+
"max_total_tokens": 9999
|
|
203
|
+
},
|
|
204
|
+
"passed": false,
|
|
205
|
+
"catalog": []
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
"name": "calls_do_not_require_tokens",
|
|
209
|
+
"providers": [
|
|
210
|
+
{
|
|
211
|
+
"provider_key": "openai",
|
|
212
|
+
"calls": 2
|
|
213
|
+
}
|
|
214
|
+
],
|
|
215
|
+
"budgets": {
|
|
216
|
+
"max_external_api_calls": 2
|
|
217
|
+
},
|
|
218
|
+
"passed": true,
|
|
219
|
+
"catalog": []
|
|
220
|
+
},
|
|
221
|
+
{
|
|
222
|
+
"name": "missing_usage_cost",
|
|
223
|
+
"providers": [
|
|
224
|
+
{
|
|
225
|
+
"provider_key": "openai",
|
|
226
|
+
"calls": 2
|
|
227
|
+
}
|
|
228
|
+
],
|
|
229
|
+
"budgets": {
|
|
230
|
+
"max_external_cost_usd": 999
|
|
231
|
+
},
|
|
232
|
+
"passed": false,
|
|
233
|
+
"catalog": [
|
|
234
|
+
{
|
|
235
|
+
"provider_key": "openai",
|
|
236
|
+
"category": "llm",
|
|
237
|
+
"cost_per_call_usd": 0.01,
|
|
238
|
+
"cost_per_1k_input_tokens_usd": 0.01,
|
|
239
|
+
"cost_per_1k_output_tokens_usd": 0.02
|
|
240
|
+
}
|
|
241
|
+
]
|
|
242
|
+
},
|
|
243
|
+
{
|
|
244
|
+
"name": "unpriced_vendor",
|
|
245
|
+
"providers": [
|
|
246
|
+
{
|
|
247
|
+
"provider_key": "new.example",
|
|
248
|
+
"calls": 1
|
|
249
|
+
}
|
|
250
|
+
],
|
|
251
|
+
"budgets": {
|
|
252
|
+
"max_external_cost_usd": 999
|
|
253
|
+
},
|
|
254
|
+
"passed": false,
|
|
255
|
+
"catalog": []
|
|
256
|
+
},
|
|
257
|
+
{
|
|
258
|
+
"name": "ocr_calls",
|
|
259
|
+
"providers": [
|
|
260
|
+
{
|
|
261
|
+
"provider_key": "aws_textract",
|
|
262
|
+
"calls": 3
|
|
263
|
+
}
|
|
264
|
+
],
|
|
265
|
+
"budgets": {
|
|
266
|
+
"max_external_api_calls": 3,
|
|
267
|
+
"external_provider": "aws_textract"
|
|
268
|
+
},
|
|
269
|
+
"passed": true,
|
|
270
|
+
"catalog": []
|
|
271
|
+
},
|
|
272
|
+
{
|
|
273
|
+
"name": "ocr_cost",
|
|
274
|
+
"providers": [
|
|
275
|
+
{
|
|
276
|
+
"provider_key": "aws_textract",
|
|
277
|
+
"calls": 3
|
|
278
|
+
}
|
|
279
|
+
],
|
|
280
|
+
"budgets": {
|
|
281
|
+
"max_external_cost_usd": 0.0045
|
|
282
|
+
},
|
|
283
|
+
"passed": true,
|
|
284
|
+
"catalog": []
|
|
285
|
+
},
|
|
286
|
+
{
|
|
287
|
+
"name": "ocr_does_not_need_tokens",
|
|
288
|
+
"providers": [
|
|
289
|
+
{
|
|
290
|
+
"provider_key": "aws_textract",
|
|
291
|
+
"calls": 3
|
|
292
|
+
},
|
|
293
|
+
{
|
|
294
|
+
"provider_key": "openai",
|
|
295
|
+
"calls": 2,
|
|
296
|
+
"input_tokens": 1000,
|
|
297
|
+
"output_tokens": 500,
|
|
298
|
+
"calls_with_usage": 2
|
|
299
|
+
}
|
|
300
|
+
],
|
|
301
|
+
"budgets": {
|
|
302
|
+
"max_total_tokens": 1500
|
|
303
|
+
},
|
|
304
|
+
"passed": true,
|
|
305
|
+
"catalog": []
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
"name": "filter_provider",
|
|
309
|
+
"providers": [
|
|
310
|
+
{
|
|
311
|
+
"provider_key": "aws_textract",
|
|
312
|
+
"calls": 3
|
|
313
|
+
},
|
|
314
|
+
{
|
|
315
|
+
"provider_key": "openai",
|
|
316
|
+
"calls": 2,
|
|
317
|
+
"input_tokens": 1000,
|
|
318
|
+
"output_tokens": 500,
|
|
319
|
+
"calls_with_usage": 2
|
|
320
|
+
}
|
|
321
|
+
],
|
|
322
|
+
"budgets": {
|
|
323
|
+
"max_external_api_calls": 2,
|
|
324
|
+
"external_provider": "openai"
|
|
325
|
+
},
|
|
326
|
+
"passed": true,
|
|
327
|
+
"catalog": []
|
|
328
|
+
},
|
|
329
|
+
{
|
|
330
|
+
"name": "missing_provider",
|
|
331
|
+
"providers": [
|
|
332
|
+
{
|
|
333
|
+
"provider_key": "openai",
|
|
334
|
+
"calls": 2,
|
|
335
|
+
"input_tokens": 1000,
|
|
336
|
+
"output_tokens": 500,
|
|
337
|
+
"calls_with_usage": 2
|
|
338
|
+
}
|
|
339
|
+
],
|
|
340
|
+
"budgets": {
|
|
341
|
+
"max_external_api_calls": 99,
|
|
342
|
+
"external_provider": "aws_textract"
|
|
343
|
+
},
|
|
344
|
+
"passed": false,
|
|
345
|
+
"catalog": []
|
|
346
|
+
},
|
|
347
|
+
{
|
|
348
|
+
"name": "model_not_double_counted",
|
|
349
|
+
"providers": [
|
|
350
|
+
{
|
|
351
|
+
"provider_key": "openai",
|
|
352
|
+
"calls": 2,
|
|
353
|
+
"input_tokens": 1000,
|
|
354
|
+
"output_tokens": 500,
|
|
355
|
+
"calls_with_usage": 2,
|
|
356
|
+
"by_model": [
|
|
357
|
+
{
|
|
358
|
+
"model": "test-model",
|
|
359
|
+
"calls": 2,
|
|
360
|
+
"input_tokens": 1000,
|
|
361
|
+
"output_tokens": 500,
|
|
362
|
+
"calls_with_usage": 2
|
|
363
|
+
}
|
|
364
|
+
]
|
|
365
|
+
}
|
|
366
|
+
],
|
|
367
|
+
"budgets": {
|
|
368
|
+
"max_total_tokens": 1500,
|
|
369
|
+
"max_external_api_calls": 2,
|
|
370
|
+
"max_external_cost_usd": 0.02
|
|
371
|
+
},
|
|
372
|
+
"passed": true,
|
|
373
|
+
"catalog": [
|
|
374
|
+
{
|
|
375
|
+
"provider_key": "openai",
|
|
376
|
+
"category": "llm",
|
|
377
|
+
"cost_per_call_usd": 0.01,
|
|
378
|
+
"cost_per_1k_input_tokens_usd": 0.01,
|
|
379
|
+
"cost_per_1k_output_tokens_usd": 0.02
|
|
380
|
+
}
|
|
381
|
+
]
|
|
382
|
+
},
|
|
383
|
+
{
|
|
384
|
+
"name": "model_override",
|
|
385
|
+
"providers": [
|
|
386
|
+
{
|
|
387
|
+
"provider_key": "openai",
|
|
388
|
+
"calls": 2,
|
|
389
|
+
"input_tokens": 1000,
|
|
390
|
+
"output_tokens": 500,
|
|
391
|
+
"calls_with_usage": 2,
|
|
392
|
+
"by_model": [
|
|
393
|
+
{
|
|
394
|
+
"model": "test-model",
|
|
395
|
+
"calls": 2,
|
|
396
|
+
"input_tokens": 1000,
|
|
397
|
+
"output_tokens": 500,
|
|
398
|
+
"calls_with_usage": 2
|
|
399
|
+
}
|
|
400
|
+
]
|
|
401
|
+
}
|
|
402
|
+
],
|
|
403
|
+
"budgets": {
|
|
404
|
+
"max_external_cost_usd": 0.002
|
|
405
|
+
},
|
|
406
|
+
"passed": true,
|
|
407
|
+
"catalog": [
|
|
408
|
+
{
|
|
409
|
+
"provider_key": "openai",
|
|
410
|
+
"category": "llm",
|
|
411
|
+
"cost_per_call_usd": 0.01,
|
|
412
|
+
"cost_per_1k_input_tokens_usd": 0.01,
|
|
413
|
+
"cost_per_1k_output_tokens_usd": 0.02
|
|
414
|
+
},
|
|
415
|
+
{
|
|
416
|
+
"provider_key": "openai::test-model",
|
|
417
|
+
"category": "llm",
|
|
418
|
+
"cost_per_call_usd": 0.01,
|
|
419
|
+
"cost_per_1k_input_tokens_usd": 0.001,
|
|
420
|
+
"cost_per_1k_output_tokens_usd": 0.002
|
|
421
|
+
}
|
|
422
|
+
]
|
|
423
|
+
},
|
|
424
|
+
{
|
|
425
|
+
"name": "unrounded_small_cost",
|
|
426
|
+
"providers": [
|
|
427
|
+
{
|
|
428
|
+
"provider_key": "aws_textract",
|
|
429
|
+
"calls": 1
|
|
430
|
+
}
|
|
431
|
+
],
|
|
432
|
+
"budgets": {
|
|
433
|
+
"max_external_cost_usd": 0
|
|
434
|
+
},
|
|
435
|
+
"passed": false,
|
|
436
|
+
"catalog": [
|
|
437
|
+
{
|
|
438
|
+
"provider_key": "aws_textract",
|
|
439
|
+
"category": "document_ai",
|
|
440
|
+
"cost_per_call_usd": 1e-06
|
|
441
|
+
}
|
|
442
|
+
]
|
|
443
|
+
},
|
|
444
|
+
{
|
|
445
|
+
"name": "invalid_counter",
|
|
446
|
+
"providers": [
|
|
447
|
+
{
|
|
448
|
+
"provider_key": "openai",
|
|
449
|
+
"calls": -1,
|
|
450
|
+
"input_tokens": 1000,
|
|
451
|
+
"output_tokens": 500,
|
|
452
|
+
"calls_with_usage": 2
|
|
453
|
+
}
|
|
454
|
+
],
|
|
455
|
+
"budgets": {
|
|
456
|
+
"max_external_api_calls": 99
|
|
457
|
+
},
|
|
458
|
+
"passed": false,
|
|
459
|
+
"catalog": []
|
|
460
|
+
},
|
|
461
|
+
{
|
|
462
|
+
"name": "invalid_counter_type",
|
|
463
|
+
"providers": [
|
|
464
|
+
{
|
|
465
|
+
"provider_key": "openai",
|
|
466
|
+
"calls": "bad",
|
|
467
|
+
"input_tokens": 1000,
|
|
468
|
+
"output_tokens": 500,
|
|
469
|
+
"calls_with_usage": 2
|
|
470
|
+
}
|
|
471
|
+
],
|
|
472
|
+
"budgets": {
|
|
473
|
+
"max_external_api_calls": 99
|
|
474
|
+
},
|
|
475
|
+
"passed": false,
|
|
476
|
+
"catalog": []
|
|
477
|
+
},
|
|
478
|
+
{
|
|
479
|
+
"name": "interrupted",
|
|
480
|
+
"providers": [
|
|
481
|
+
{
|
|
482
|
+
"provider_key": "openai",
|
|
483
|
+
"calls": 2,
|
|
484
|
+
"input_tokens": 1000,
|
|
485
|
+
"output_tokens": 500,
|
|
486
|
+
"calls_with_usage": 2
|
|
487
|
+
}
|
|
488
|
+
],
|
|
489
|
+
"budgets": {
|
|
490
|
+
"max_total_tokens": 9999
|
|
491
|
+
},
|
|
492
|
+
"passed": false,
|
|
493
|
+
"catalog": [],
|
|
494
|
+
"interrupted": true
|
|
495
|
+
},
|
|
496
|
+
{
|
|
497
|
+
"name": "model_residual_pricing",
|
|
498
|
+
"providers": [
|
|
499
|
+
{
|
|
500
|
+
"provider_key": "openai",
|
|
501
|
+
"calls": 2,
|
|
502
|
+
"input_tokens": 1000,
|
|
503
|
+
"output_tokens": 500,
|
|
504
|
+
"calls_with_usage": 2,
|
|
505
|
+
"by_model": [
|
|
506
|
+
{
|
|
507
|
+
"model": "test-model",
|
|
508
|
+
"calls": 1,
|
|
509
|
+
"input_tokens": 300,
|
|
510
|
+
"output_tokens": 200,
|
|
511
|
+
"calls_with_usage": 1
|
|
512
|
+
}
|
|
513
|
+
]
|
|
514
|
+
}
|
|
515
|
+
],
|
|
516
|
+
"budgets": {
|
|
517
|
+
"max_external_cost_usd": 0.0137,
|
|
518
|
+
"max_external_api_calls": 2
|
|
519
|
+
},
|
|
520
|
+
"passed": true,
|
|
521
|
+
"catalog": [
|
|
522
|
+
{
|
|
523
|
+
"provider_key": "openai",
|
|
524
|
+
"category": "llm",
|
|
525
|
+
"cost_per_call_usd": 0.01,
|
|
526
|
+
"cost_per_1k_input_tokens_usd": 0.01,
|
|
527
|
+
"cost_per_1k_output_tokens_usd": 0.02
|
|
528
|
+
},
|
|
529
|
+
{
|
|
530
|
+
"provider_key": "openai::test-model",
|
|
531
|
+
"category": "llm",
|
|
532
|
+
"cost_per_call_usd": 0.01,
|
|
533
|
+
"cost_per_1k_input_tokens_usd": 0.001,
|
|
534
|
+
"cost_per_1k_output_tokens_usd": 0.002
|
|
535
|
+
}
|
|
536
|
+
]
|
|
537
|
+
},
|
|
538
|
+
{
|
|
539
|
+
"name": "filter_model",
|
|
540
|
+
"providers": [
|
|
541
|
+
{
|
|
542
|
+
"provider_key": "openai",
|
|
543
|
+
"calls": 2,
|
|
544
|
+
"input_tokens": 1000,
|
|
545
|
+
"output_tokens": 500,
|
|
546
|
+
"calls_with_usage": 2,
|
|
547
|
+
"by_model": [
|
|
548
|
+
{
|
|
549
|
+
"model": "test-model",
|
|
550
|
+
"calls": 1,
|
|
551
|
+
"input_tokens": 300,
|
|
552
|
+
"output_tokens": 200,
|
|
553
|
+
"calls_with_usage": 1
|
|
554
|
+
}
|
|
555
|
+
]
|
|
556
|
+
}
|
|
557
|
+
],
|
|
558
|
+
"budgets": {
|
|
559
|
+
"external_provider": "openai::test-model",
|
|
560
|
+
"max_total_tokens": 500,
|
|
561
|
+
"max_external_api_calls": 1
|
|
562
|
+
},
|
|
563
|
+
"passed": true,
|
|
564
|
+
"catalog": []
|
|
565
|
+
},
|
|
566
|
+
{
|
|
567
|
+
"name": "zero_price_is_unknown",
|
|
568
|
+
"providers": [
|
|
569
|
+
{
|
|
570
|
+
"provider_key": "openai",
|
|
571
|
+
"calls": 2,
|
|
572
|
+
"input_tokens": 1000,
|
|
573
|
+
"output_tokens": 500,
|
|
574
|
+
"calls_with_usage": 2
|
|
575
|
+
}
|
|
576
|
+
],
|
|
577
|
+
"budgets": {
|
|
578
|
+
"max_external_cost_usd": 10
|
|
579
|
+
},
|
|
580
|
+
"passed": false,
|
|
581
|
+
"catalog": [
|
|
582
|
+
{
|
|
583
|
+
"provider_key": "openai",
|
|
584
|
+
"category": "llm",
|
|
585
|
+
"cost_per_call_usd": 0,
|
|
586
|
+
"cost_per_1k_input_tokens_usd": 0,
|
|
587
|
+
"cost_per_1k_output_tokens_usd": 0
|
|
588
|
+
}
|
|
589
|
+
]
|
|
590
|
+
},
|
|
591
|
+
{
|
|
592
|
+
"name": "invalid_model_coverage",
|
|
593
|
+
"providers": [
|
|
594
|
+
{
|
|
595
|
+
"provider_key": "openai",
|
|
596
|
+
"calls": 2,
|
|
597
|
+
"input_tokens": 1000,
|
|
598
|
+
"output_tokens": 500,
|
|
599
|
+
"calls_with_usage": 2,
|
|
600
|
+
"by_model": [
|
|
601
|
+
{
|
|
602
|
+
"model": "test-model",
|
|
603
|
+
"calls": 2,
|
|
604
|
+
"input_tokens": 1000,
|
|
605
|
+
"output_tokens": 500
|
|
606
|
+
}
|
|
607
|
+
]
|
|
608
|
+
}
|
|
609
|
+
],
|
|
610
|
+
"budgets": {
|
|
611
|
+
"max_total_tokens": 9999
|
|
612
|
+
},
|
|
613
|
+
"passed": false,
|
|
614
|
+
"catalog": []
|
|
615
|
+
}
|
|
616
|
+
]
|
|
617
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
[
|
|
2
|
+
{"name":"Vercel production deployment", "env":{"VERCEL_ENV":"production","VERCEL_DEPLOYMENT_ID":"dpl_one","VERCEL_URL":"app-one.vercel.app","VERCEL_GIT_COMMIT_SHA":"same-commit"}, "expected":{"release_tag":"vercel:dpl_one","deploy_id":"dpl_one","git_sha":"same-commit","deploy_environment":"prod"}},
|
|
3
|
+
{"name":"Vercel redeploy of the same commit", "env":{"VERCEL_ENV":"production","VERCEL_DEPLOYMENT_ID":"dpl_two","VERCEL_GIT_COMMIT_SHA":"same-commit"}, "expected":{"release_tag":"vercel:dpl_two","deploy_id":"dpl_two","git_sha":"same-commit","deploy_environment":"prod"}},
|
|
4
|
+
{"name":"Vercel preview without Git", "env":{"VERCEL_ENV":"preview","VERCEL_URL":"app-preview-123.vercel.app"}, "expected":{"release_tag":"vercel:app-preview-123.vercel.app","deploy_id":"app-preview-123.vercel.app","git_sha":null,"deploy_environment":"staging"}},
|
|
5
|
+
{"name":"Vercel development", "env":{"VERCEL_ENV":"development"}, "expected":{"release_tag":null,"deploy_id":null,"git_sha":null,"deploy_environment":"dev"}},
|
|
6
|
+
{"name":"Stable project and branch URLs are not releases", "env":{"VERCEL_PROJECT_PRODUCTION_URL":"app.example.com","VERCEL_BRANCH_URL":"app-main.vercel.app"}, "expected":{"release_tag":null,"deploy_id":null,"git_sha":null}},
|
|
7
|
+
{"name":"Explicit runtime settings win", "env":{"NETGREENER_RELEASE_TAG":"v4","NETGREENER_GIT_SHA":"override-sha","NETGREENER_DEPLOY_ID":"override-id","NETGREENER_DEPLOY_ENVIRONMENT":"staging","VERCEL_ENV":"production","VERCEL_DEPLOYMENT_ID":"dpl_ignored","VERCEL_GIT_COMMIT_SHA":"ignored"}, "expected":{"release_tag":"v4","deploy_id":"override-id","git_sha":"override-sha","deploy_environment":"staging"}},
|
|
8
|
+
{"name":"Existing context wins", "context":{"release_tag":"existing","git_sha":"saved-sha","deploy_id":"saved-id","deploy_environment":"staging"}, "env":{"VERCEL_ENV":"production","VERCEL_DEPLOYMENT_ID":"dpl_ignored","VERCEL_GIT_COMMIT_SHA":"ignored"}, "expected":{"release_tag":"existing","deploy_id":"saved-id","git_sha":"saved-sha","deploy_environment":"staging"}},
|
|
9
|
+
{"name":"Azure named release", "env":{"RELEASE_RELEASENAME":"Release-42","RELEASE_RELEASEID":"42","RELEASE_DEPLOYMENTID":"99","BUILD_BUILDNUMBER":"artifact-build","BUILD_SOURCEVERSION":"azure-sha","NETGREENER_ENV":"prod"}, "expected":{"release_tag":"Release-42","deploy_id":"99","git_sha":"azure-sha","deploy_environment":"prod"}},
|
|
10
|
+
{"name":"Azure release ID without Git", "env":{"RELEASE_RELEASEID":"43","RELEASE_DEPLOYMENTID":"100","DEPLOY_ENV":"staging"}, "expected":{"release_tag":"azure:43","deploy_id":"100","git_sha":null,"deploy_environment":"staging"}},
|
|
11
|
+
{"name":"Azure deployment ID only", "env":{"RELEASE_DEPLOYMENTID":"101"}, "expected":{"release_tag":"azure:101","deploy_id":"101","git_sha":null}},
|
|
12
|
+
{"name":"Azure build metadata remains supported", "env":{"BUILD_BUILDNUMBER":"2026.10.24.1","BUILD_BUILDID":"77","BUILD_SOURCEVERSION":"build-sha"}, "expected":{"release_tag":"2026.10.24.1","deploy_id":"77","git_sha":"build-sha"}},
|
|
13
|
+
{"name":"Blank platform values do not create identities", "env":{"VERCEL_DEPLOYMENT_ID":" ","VERCEL_URL":" ","RELEASE_RELEASEID":" "}, "expected":{"release_tag":null,"deploy_id":null,"git_sha":null}}
|
|
14
|
+
]
|
package/package.json
CHANGED