@langwatch/mcp-server 1.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.python-version +1 -0
- package/CHANGELOG.md +110 -0
- package/CONTRIBUTING.md +1 -1
- package/README.md +86 -4
- package/dist/{archive-project-MK3673A7.js → archive-project-W3IKI7UL.js} +5 -5
- package/dist/archive-run-plan-474OKHHU.js +16 -0
- package/dist/{archive-scenario-KATSXRIN.js → archive-scenario-V5UNYK26.js} +5 -5
- package/dist/archive-test-suite-O7FLDMXK.js +16 -0
- package/dist/{assign-prompt-tag-E3C5W6PI.js → assign-prompt-tag-Q46WRB6P.js} +4 -4
- package/dist/chunk-56OGTS24.js +44 -0
- package/dist/chunk-6DJDEXET.js +63 -0
- package/dist/chunk-7E4FZIYJ.js +15 -0
- package/dist/{chunk-J45SVWPR.js → chunk-AN4OUA4D.js} +2 -2
- package/dist/chunk-CKNAH5TG.js +74 -0
- package/dist/{chunk-CE6QMZOP.js → chunk-EYXMLE5X.js} +86 -11
- package/dist/{chunk-XY4STOBE.js → chunk-G4ZAZTZW.js} +2 -2
- package/dist/{chunk-5UWY24RP.js → chunk-GELJ2MQR.js} +12 -3
- package/dist/chunk-HV4DWKBI.js +46 -0
- package/dist/chunk-JR6DJZOU.js +49 -0
- package/dist/{chunk-DAIQKS27.js → chunk-JTQJRNFJ.js} +2 -2
- package/dist/{chunk-WWMDHZZV.js → chunk-K37ZNPXK.js} +2 -2
- package/dist/{chunk-EU22L52K.js → chunk-OJLYYJWI.js} +2 -2
- package/dist/{chunk-Y3HDZ6OW.js → chunk-QAMP3CGR.js} +2 -2
- package/dist/{chunk-VYPSKRPN.js → chunk-QRO35SOM.js} +347 -146
- package/dist/{chunk-O3LWVN67.js → chunk-RVAE2MEA.js} +2 -2
- package/dist/{chunk-UXXA4D6K.js → chunk-S4GVX36W.js} +2 -2
- package/dist/{chunk-BJ7B4R26.js → chunk-SAS5OB7F.js} +18 -2
- package/dist/{chunk-RYRB2YVJ.js → chunk-SR6QPRIE.js} +2 -2
- package/dist/chunk-UF6XIGYR.js +58 -0
- package/dist/{chunk-T7WJVS2N.js → chunk-UFOYFYI7.js} +2 -2
- package/dist/chunk-UJPXFA26.js +267 -0
- package/dist/{chunk-KOO5733S.js → chunk-WT4PO4A3.js} +8 -2
- package/dist/config.js +2 -2
- package/dist/{create-agent-7W57J7MT.js → create-agent-JFJ2K6X3.js} +5 -5
- package/dist/{create-annotation-W2XELBCQ.js → create-annotation-EUXXQ6NS.js} +5 -5
- package/dist/{create-api-key-MKRXKFJM.js → create-api-key-BGBB76JX.js} +5 -5
- package/dist/{create-dashboard-QGL7IODC.js → create-dashboard-Z76AIPLS.js} +5 -5
- package/dist/{create-dataset-3PH7A4HG.js → create-dataset-FD5VAXU4.js} +5 -5
- package/dist/{create-dataset-records-4DZS3VRL.js → create-dataset-records-WQUXXOHI.js} +5 -5
- package/dist/{create-evaluator-YWH6NC4M.js → create-evaluator-6NV33Z52.js} +5 -5
- package/dist/create-mcp-server.js +9 -4
- package/dist/{create-project-BOBQ32C5.js → create-project-ZQ7TFIO2.js} +5 -5
- package/dist/{create-prompt-FLCFGITE.js → create-prompt-BS6QRCUU.js} +4 -4
- package/dist/{create-prompt-tag-EMPNIKEA.js → create-prompt-tag-FCQ5ELSQ.js} +4 -4
- package/dist/{create-scenario-BSBIHUL6.js → create-scenario-DRPTPXU3.js} +12 -5
- package/dist/create-test-suite-ZIZCCFJ4.js +37 -0
- package/dist/{delete-agent-7MSYNY5E.js → delete-agent-SA4JMUZF.js} +5 -5
- package/dist/{delete-annotation-ZT5IGN24.js → delete-annotation-X3KKK7QY.js} +5 -5
- package/dist/{delete-dashboard-BTIY5S3L.js → delete-dashboard-LYMRPF27.js} +5 -5
- package/dist/{delete-dataset-U45HQHPR.js → delete-dataset-W6EZ44HY.js} +5 -5
- package/dist/{delete-dataset-records-R43IMAWB.js → delete-dataset-records-7CNOTALB.js} +5 -5
- package/dist/{delete-evaluator-U6JMQ6C7.js → delete-evaluator-RKYC4TNB.js} +5 -5
- package/dist/{delete-prompt-tag-4EV4TDHB.js → delete-prompt-tag-QB3OCZUV.js} +4 -4
- package/dist/{delete-workflow-5QYALXEL.js → delete-workflow-NG2VXYYM.js} +5 -5
- package/dist/{discover-evaluator-schema-MRGHVNTT.js → discover-evaluator-schema-ESSHQGPY.js} +58 -321
- package/dist/{discover-scenario-schema-UPBTCWPG.js → discover-scenario-schema-W7KBXA4J.js} +54 -2
- package/dist/{discover-schema-ICGXPHUW.js → discover-schema-L6QDPON3.js} +2 -2
- package/dist/get-agent-7THF7HCU.js +66 -0
- package/dist/{get-analytics-PVERWOA5.js → get-analytics-R6AZD3AW.js} +4 -4
- package/dist/{get-dashboard-ZZPC3WMD.js → get-dashboard-P4WWVHFU.js} +5 -5
- package/dist/{get-dataset-25TB2LQH.js → get-dataset-AKTYSQDI.js} +5 -5
- package/dist/{get-evaluator-3FTRQKEC.js → get-evaluator-RZ44UIN6.js} +5 -5
- package/dist/{get-project-5OAXYC57.js → get-project-QSQEAB33.js} +5 -5
- package/dist/get-prompt-GVKXF5NL.js +100 -0
- package/dist/get-run-plan-NY7J5GRE.js +61 -0
- package/dist/{get-scenario-DSSTR6FI.js → get-scenario-Q3LLQE4T.js} +9 -5
- package/dist/{get-simulation-run-RQPMS2MC.js → get-simulation-run-WINEYDMK.js} +14 -5
- package/dist/get-test-suite-4HKA7OKS.js +49 -0
- package/dist/{get-trace-O5VT3NAC.js → get-trace-RCUV7N4O.js} +4 -4
- package/dist/{get-workflow-3EZDOEIF.js → get-workflow-EEOC7Y45.js} +5 -5
- package/dist/http-security-EU3CHTY7.js +26 -0
- package/dist/{http-server-2ACJ3CXW.js → http-server-WCDUARTH.js} +1038 -693
- package/dist/index.js +6437 -4020
- package/dist/{langwatch-api-annotations-E65MI3WZ.js → langwatch-api-annotations-6DHVZWUK.js} +5 -5
- package/dist/{langwatch-api-dashboards-QCDECRCY.js → langwatch-api-dashboards-2KJL2M42.js} +5 -5
- package/dist/{langwatch-api-monitors-UIO2KTDV.js → langwatch-api-monitors-6ISFJ7WX.js} +4 -4
- package/dist/{langwatch-api-secrets-4T2EKSJ7.js → langwatch-api-secrets-OEBD6GBU.js} +4 -4
- package/dist/{langwatch-api-triggers-MPOEKPUY.js → langwatch-api-triggers-5E6DDGX6.js} +4 -4
- package/dist/langwatch-api-workflows-6UJL5U4Z.js +16 -0
- package/dist/list-agents-AMVDQPQL.js +43 -0
- package/dist/{list-annotations-CGX5BT7E.js → list-annotations-6PR5NLIC.js} +5 -5
- package/dist/{list-api-keys-JLWJPFRX.js → list-api-keys-GVPNBIHQ.js} +5 -5
- package/dist/{list-dashboards-RNIWX3TX.js → list-dashboards-Z3BZKDGF.js} +5 -5
- package/dist/{list-dataset-records-B5MHZR66.js → list-dataset-records-O67HZ35Y.js} +5 -5
- package/dist/{list-datasets-BVCAMQCP.js → list-datasets-WXZL6UMP.js} +5 -5
- package/dist/{list-evaluators-65FBTL2U.js → list-evaluators-PAX72WCN.js} +5 -5
- package/dist/{list-model-providers-GRK3I3VV.js → list-model-providers-VVLZMLCX.js} +5 -5
- package/dist/{list-projects-JLMBFMW3.js → list-projects-AQVI6SUK.js} +5 -5
- package/dist/{list-prompt-tags-UHNWDGQI.js → list-prompt-tags-WZFBJOHG.js} +4 -4
- package/dist/{list-prompts-7254B3U7.js → list-prompts-BQOHRHYT.js} +4 -4
- package/dist/list-run-plans-UBAU2DGH.js +46 -0
- package/dist/{list-scenarios-46YNICKM.js → list-scenarios-MS4IUIVM.js} +10 -7
- package/dist/{list-simulation-runs-LGNEFNPN.js → list-simulation-runs-D6YF4ZOU.js} +13 -7
- package/dist/list-test-suites-L7R2O6DR.js +35 -0
- package/dist/{list-workflows-MLQDM6CP.js → list-workflows-6MV4XJHF.js} +5 -5
- package/dist/{rename-prompt-tag-RHQMNEQL.js → rename-prompt-tag-K5LQ4CY6.js} +4 -4
- package/dist/rename-test-suite-NTYHAO5X.js +21 -0
- package/dist/report-issue-RHZZ33IS.js +885 -0
- package/dist/rerun-run-plan-MUDWJBE4.js +22 -0
- package/dist/{revoke-api-key-3Y35EMBT.js → revoke-api-key-6LAZ3QZZ.js} +5 -5
- package/dist/run-agent-CAGWGTYT.js +72 -0
- package/dist/run-plan-T4ZYOZGA.js +39 -0
- package/dist/run-test-suite-L4MZ7G4D.js +26 -0
- package/dist/{run-workflow-5LVBZYLS.js → run-workflow-DFIS23AP.js} +5 -5
- package/dist/{search-traces-U4HLOCZ2.js → search-traces-4XTJYXIK.js} +4 -4
- package/dist/{set-model-provider-V3M5CDIY.js → set-model-provider-XRNMFOVR.js} +5 -5
- package/dist/{update-agent-47RZIHS7.js → update-agent-DUWNYBNA.js} +5 -5
- package/dist/{update-dataset-G6QJJGS5.js → update-dataset-YSOMDCOB.js} +5 -5
- package/dist/{update-dataset-record-RMWLSWCS.js → update-dataset-record-DEA2ITYR.js} +5 -5
- package/dist/{update-evaluator-SFRRI5Q3.js → update-evaluator-NH7Q5GWV.js} +5 -5
- package/dist/{update-project-PI5DWBVK.js → update-project-Q26FBSYH.js} +5 -5
- package/dist/update-prompt-ZQQ2C3KU.js +143 -0
- package/dist/{update-scenario-G3IU2UIB.js → update-scenario-WUX34JAR.js} +12 -5
- package/package.json +8 -9
- package/pyproject.toml +5 -5
- package/src/__tests__/agent-tools.unit.test.ts +257 -0
- package/src/__tests__/all-tools.integration.test.ts +321 -75
- package/src/__tests__/config.unit.test.ts +33 -0
- package/src/__tests__/docker.integration.test.ts +91 -7
- package/src/__tests__/documentation-fetch-security.integration.test.ts +8 -2
- package/src/__tests__/feature-map-tool-drift.unit.test.ts +93 -0
- package/src/__tests__/http-security-origin.unit.test.ts +97 -0
- package/src/__tests__/http-security-sessions.unit.test.ts +209 -0
- package/src/__tests__/http-security-verifier.unit.test.ts +248 -0
- package/src/__tests__/http-server-allocation.integration.test.ts +279 -0
- package/src/__tests__/http-server-origin.integration.test.ts +169 -0
- package/src/__tests__/http-server-session-auth.integration.test.ts +341 -0
- package/src/__tests__/http-transport.integration.test.ts +85 -87
- package/src/__tests__/integration.integration.test.ts +7 -11
- package/src/__tests__/prompt-response-types.unit.test.ts +51 -0
- package/src/__tests__/prompts.integration.test.ts +736 -0
- package/src/__tests__/run-plan-tools.unit.test.ts +565 -0
- package/src/__tests__/scenario-tools.unit.test.ts +19 -0
- package/src/__tests__/simulation-run-tools.unit.test.ts +116 -0
- package/src/__tests__/support/http-server-harness.ts +138 -0
- package/src/__tests__/test-suite-tools.unit.test.ts +615 -0
- package/src/__tests__/{tools.unit.test.ts → tools.integration.test.ts} +31 -59
- package/src/config.ts +17 -2
- package/src/create-mcp-server.ts +334 -55
- package/src/http-security.ts +492 -0
- package/src/http-server.ts +536 -152
- package/src/index.ts +49 -2
- package/src/langwatch-api-agents.ts +196 -12
- package/src/langwatch-api-run-plans.ts +157 -0
- package/src/langwatch-api-scenarios.ts +9 -0
- package/src/langwatch-api-simulation-runs.ts +5 -0
- package/src/langwatch-api-test-suites.ts +135 -0
- package/src/langwatch-api.ts +62 -12
- package/src/schemas/index.ts +5 -0
- package/src/schemas/run-plan.ts +92 -0
- package/src/schemas/suite-fields.ts +149 -0
- package/src/tools/archive-run-plan.ts +12 -0
- package/src/tools/archive-test-suite.ts +12 -0
- package/src/tools/create-scenario.ts +8 -0
- package/src/tools/create-test-suite.ts +43 -0
- package/src/tools/discover-evaluator-schema.ts +2 -2
- package/src/tools/discover-scenario-schema.ts +54 -0
- package/src/tools/format-run-plan.ts +68 -0
- package/src/tools/format-scenario.ts +14 -0
- package/src/tools/format-suite-details.ts +96 -0
- package/src/tools/get-agent.ts +45 -3
- package/src/tools/get-prompt.ts +108 -24
- package/src/tools/get-run-plan.ts +62 -0
- package/src/tools/get-scenario.ts +3 -0
- package/src/tools/get-simulation-run.ts +9 -0
- package/src/tools/get-test-suite.ts +49 -0
- package/src/tools/list-agents.ts +20 -3
- package/src/tools/list-run-plans.ts +47 -0
- package/src/tools/list-scenarios.ts +12 -2
- package/src/tools/list-simulation-runs.ts +6 -2
- package/src/tools/list-test-suites.ts +35 -0
- package/src/tools/rename-test-suite.ts +18 -0
- package/src/tools/rerun-run-plan.ts +21 -0
- package/src/tools/run-agent.ts +54 -6
- package/src/tools/run-plan.ts +49 -0
- package/src/tools/run-test-suite.ts +30 -0
- package/src/tools/test-agent.ts +18 -0
- package/src/tools/update-prompt.ts +195 -10
- package/src/tools/update-scenario.ts +8 -0
- package/src/tools/update-test-suite.ts +45 -0
- package/tsconfig.json +7 -2
- package/uv.lock +418 -298
- package/dist/archive-suite-C3KX6IIZ.js +0 -16
- package/dist/chunk-WDH5P5RF.js +0 -48
- package/dist/create-suite-TWF6BBR4.js +0 -39
- package/dist/get-agent-7QJVPH72.js +0 -29
- package/dist/get-prompt-I753LZKZ.js +0 -46
- package/dist/get-suite-GTOCY5PL.js +0 -45
- package/dist/langwatch-api-workflows-5HRZ7HR4.js +0 -16
- package/dist/list-agents-TIJTP566.js +0 -33
- package/dist/list-suites-5VK4HXEK.js +0 -43
- package/dist/report-issue-DXJ6TPTC.js +0 -412
- package/dist/run-agent-RBSIFKRT.js +0 -37
- package/dist/run-suite-K3DNQ3SE.js +0 -30
- package/dist/update-prompt-RQFTEO7Y.js +0 -25
- package/dist/update-suite-TRGGLNR7.js +0 -40
- package/pnpm-workspace.yaml +0 -34
- package/src/langwatch-api-suites.ts +0 -118
- package/src/tools/archive-suite.ts +0 -12
- package/src/tools/create-suite.ts +0 -42
- package/src/tools/get-suite.ts +0 -47
- package/src/tools/list-suites.ts +0 -43
- package/src/tools/run-suite.ts +0 -28
- package/src/tools/update-suite.ts +0 -44
package/dist/{discover-evaluator-schema-MRGHVNTT.js → discover-evaluator-schema-ESSHQGPY.js}
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import "./chunk-
|
|
1
|
+
import "./chunk-SAS5OB7F.js";
|
|
2
2
|
|
|
3
|
-
//
|
|
3
|
+
// ../../services/langevals/ts-integration/evaluators.generated.ts
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
var evaluatorTypesSchema = z.string();
|
|
6
6
|
var moneySchema = z.object({
|
|
@@ -18,7 +18,8 @@ var evaluationResultSchema = z.object({
|
|
|
18
18
|
});
|
|
19
19
|
var evaluationResultSkippedSchema = z.object({
|
|
20
20
|
status: z.literal("skipped"),
|
|
21
|
-
details: z.string().optional()
|
|
21
|
+
details: z.string().optional(),
|
|
22
|
+
cost: moneySchema.optional()
|
|
22
23
|
});
|
|
23
24
|
var evaluationResultErrorSchema = z.object({
|
|
24
25
|
status: z.literal("error"),
|
|
@@ -379,69 +380,6 @@ var evaluatorsSchema = z.object({
|
|
|
379
380
|
).default(0.25)
|
|
380
381
|
})
|
|
381
382
|
}),
|
|
382
|
-
"legacy/ragas_answer_correctness": z.object({
|
|
383
|
-
settings: z.object({
|
|
384
|
-
model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
|
|
385
|
-
embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
|
|
386
|
-
max_tokens: z.number().describe(
|
|
387
|
-
"The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
|
|
388
|
-
).default(2048)
|
|
389
|
-
})
|
|
390
|
-
}),
|
|
391
|
-
"legacy/ragas_answer_relevancy": z.object({
|
|
392
|
-
settings: z.object({
|
|
393
|
-
model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
|
|
394
|
-
embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
|
|
395
|
-
max_tokens: z.number().describe(
|
|
396
|
-
"The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
|
|
397
|
-
).default(2048)
|
|
398
|
-
})
|
|
399
|
-
}),
|
|
400
|
-
"legacy/ragas_context_precision": z.object({
|
|
401
|
-
settings: z.object({
|
|
402
|
-
model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
|
|
403
|
-
embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
|
|
404
|
-
max_tokens: z.number().describe(
|
|
405
|
-
"The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
|
|
406
|
-
).default(2048)
|
|
407
|
-
})
|
|
408
|
-
}),
|
|
409
|
-
"legacy/ragas_context_recall": z.object({
|
|
410
|
-
settings: z.object({
|
|
411
|
-
model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
|
|
412
|
-
embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
|
|
413
|
-
max_tokens: z.number().describe(
|
|
414
|
-
"The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
|
|
415
|
-
).default(2048)
|
|
416
|
-
})
|
|
417
|
-
}),
|
|
418
|
-
"legacy/ragas_context_relevancy": z.object({
|
|
419
|
-
settings: z.object({
|
|
420
|
-
model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
|
|
421
|
-
embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
|
|
422
|
-
max_tokens: z.number().describe(
|
|
423
|
-
"The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
|
|
424
|
-
).default(2048)
|
|
425
|
-
})
|
|
426
|
-
}),
|
|
427
|
-
"legacy/ragas_context_utilization": z.object({
|
|
428
|
-
settings: z.object({
|
|
429
|
-
model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
|
|
430
|
-
embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
|
|
431
|
-
max_tokens: z.number().describe(
|
|
432
|
-
"The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
|
|
433
|
-
).default(2048)
|
|
434
|
-
})
|
|
435
|
-
}),
|
|
436
|
-
"legacy/ragas_faithfulness": z.object({
|
|
437
|
-
settings: z.object({
|
|
438
|
-
model: z.string().describe("The model to use for evaluation.").default("openai/gpt-5-mini"),
|
|
439
|
-
embeddings_model: z.string().describe("The model to use for embeddings.").default("openai/text-embedding-ada-002"),
|
|
440
|
-
max_tokens: z.number().describe(
|
|
441
|
-
"The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped."
|
|
442
|
-
).default(2048)
|
|
443
|
-
})
|
|
444
|
-
}),
|
|
445
383
|
"langevals/basic": z.object({
|
|
446
384
|
settings: z.object({
|
|
447
385
|
rules: z.array(
|
|
@@ -580,6 +518,12 @@ var evaluatorsSchema = z.object({
|
|
|
580
518
|
include_metrics: z.array(z.union([z.literal("cost"), z.literal("duration")])).describe("Per-candidate metrics to inject into the judge prompt").default([])
|
|
581
519
|
})
|
|
582
520
|
}),
|
|
521
|
+
"langevals/query_resolution": z.object({
|
|
522
|
+
settings: z.object({
|
|
523
|
+
model: z.string().describe("The model to use for evaluation").default("openai/gpt-5-mini"),
|
|
524
|
+
max_tokens: z.number().describe("Max tokens allowed for evaluation").default(128e3)
|
|
525
|
+
})
|
|
526
|
+
}),
|
|
583
527
|
"langevals/select_best_compare": z.object({
|
|
584
528
|
settings: z.object({
|
|
585
529
|
model: z.string().describe("The model to use for evaluation").default("openai/gpt-5-mini"),
|
|
@@ -609,13 +553,13 @@ slot label. Use "tie" only when no candidate is clearly better.
|
|
|
609
553
|
allow_tie: z.boolean().describe(
|
|
610
554
|
"Allow the judge to return 'tie' when candidates are equivalent"
|
|
611
555
|
).default(true),
|
|
612
|
-
include_metrics: z.array(z.union([z.literal("cost"), z.literal("duration")])).describe("Per-candidate metrics to inject into the judge prompt").default([])
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
556
|
+
include_metrics: z.array(z.union([z.literal("cost"), z.literal("duration")])).describe("Per-candidate metrics to inject into the judge prompt").default([]),
|
|
557
|
+
temperature: z.number().describe(
|
|
558
|
+
"Sampling temperature for the judge call. Lower is more deterministic."
|
|
559
|
+
).default(0),
|
|
560
|
+
swap_and_reconcile: z.boolean().describe(
|
|
561
|
+
"Check each row twice, the second time with the candidate order reversed, and record no result for that row when the two disagree \u2014 rather than a tie, which would claim the candidates are equally good. Doubles judge-call cost per row (literature: swapping candidate order alone flips 10-30% of verdicts on contested rows)."
|
|
562
|
+
).default(true)
|
|
619
563
|
})
|
|
620
564
|
}),
|
|
621
565
|
"langevals/sentiment": z.object({
|
|
@@ -1237,223 +1181,6 @@ or if it's in a specific expected language.
|
|
|
1237
1181
|
}
|
|
1238
1182
|
}
|
|
1239
1183
|
},
|
|
1240
|
-
"legacy/ragas_answer_correctness": {
|
|
1241
|
-
name: `Ragas Answer Correctness`,
|
|
1242
|
-
description: `
|
|
1243
|
-
Computes with an LLM a weighted combination of factual as well as semantic similarity between the generated answer and the expected output.
|
|
1244
|
-
`,
|
|
1245
|
-
category: "rag",
|
|
1246
|
-
docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/answer_correctness.html",
|
|
1247
|
-
isGuardrail: false,
|
|
1248
|
-
requiredFields: ["output", "expected_output"],
|
|
1249
|
-
optionalFields: ["input"],
|
|
1250
|
-
settings: {
|
|
1251
|
-
model: {
|
|
1252
|
-
description: "The model to use for evaluation.",
|
|
1253
|
-
default: "openai/gpt-5-mini"
|
|
1254
|
-
},
|
|
1255
|
-
embeddings_model: {
|
|
1256
|
-
description: "The model to use for embeddings.",
|
|
1257
|
-
default: "openai/text-embedding-ada-002"
|
|
1258
|
-
},
|
|
1259
|
-
max_tokens: {
|
|
1260
|
-
description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
|
|
1261
|
-
default: 2048
|
|
1262
|
-
}
|
|
1263
|
-
},
|
|
1264
|
-
envVars: [],
|
|
1265
|
-
result: {
|
|
1266
|
-
score: {
|
|
1267
|
-
description: "A score between 0.0 and 1.0 indicating the correctness of the answer."
|
|
1268
|
-
}
|
|
1269
|
-
}
|
|
1270
|
-
},
|
|
1271
|
-
"legacy/ragas_answer_relevancy": {
|
|
1272
|
-
name: `Ragas Answer Relevancy`,
|
|
1273
|
-
description: `
|
|
1274
|
-
Evaluates how pertinent the generated answer is to the given prompt. Higher scores indicate better relevancy.
|
|
1275
|
-
`,
|
|
1276
|
-
category: "rag",
|
|
1277
|
-
docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/answer_relevance.html",
|
|
1278
|
-
isGuardrail: false,
|
|
1279
|
-
requiredFields: ["input", "output"],
|
|
1280
|
-
optionalFields: [],
|
|
1281
|
-
settings: {
|
|
1282
|
-
model: {
|
|
1283
|
-
description: "The model to use for evaluation.",
|
|
1284
|
-
default: "openai/gpt-5-mini"
|
|
1285
|
-
},
|
|
1286
|
-
embeddings_model: {
|
|
1287
|
-
description: "The model to use for embeddings.",
|
|
1288
|
-
default: "openai/text-embedding-ada-002"
|
|
1289
|
-
},
|
|
1290
|
-
max_tokens: {
|
|
1291
|
-
description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
|
|
1292
|
-
default: 2048
|
|
1293
|
-
}
|
|
1294
|
-
},
|
|
1295
|
-
envVars: [],
|
|
1296
|
-
result: {
|
|
1297
|
-
score: {
|
|
1298
|
-
description: "A score between 0.0 and 1.0 indicating the relevance of the answer."
|
|
1299
|
-
}
|
|
1300
|
-
}
|
|
1301
|
-
},
|
|
1302
|
-
"legacy/ragas_context_precision": {
|
|
1303
|
-
name: `Ragas Context Precision`,
|
|
1304
|
-
description: `
|
|
1305
|
-
This metric evaluates whether all of the ground-truth relevant items present in the contexts are ranked higher or not. Higher scores indicate better precision.
|
|
1306
|
-
`,
|
|
1307
|
-
category: "rag",
|
|
1308
|
-
docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_precision.html",
|
|
1309
|
-
isGuardrail: false,
|
|
1310
|
-
requiredFields: ["input", "contexts", "expected_output"],
|
|
1311
|
-
optionalFields: [],
|
|
1312
|
-
settings: {
|
|
1313
|
-
model: {
|
|
1314
|
-
description: "The model to use for evaluation.",
|
|
1315
|
-
default: "openai/gpt-5-mini"
|
|
1316
|
-
},
|
|
1317
|
-
embeddings_model: {
|
|
1318
|
-
description: "The model to use for embeddings.",
|
|
1319
|
-
default: "openai/text-embedding-ada-002"
|
|
1320
|
-
},
|
|
1321
|
-
max_tokens: {
|
|
1322
|
-
description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
|
|
1323
|
-
default: 2048
|
|
1324
|
-
}
|
|
1325
|
-
},
|
|
1326
|
-
envVars: [],
|
|
1327
|
-
result: {
|
|
1328
|
-
score: {
|
|
1329
|
-
description: "A score between 0.0 and 1.0 indicating the precision of the context."
|
|
1330
|
-
}
|
|
1331
|
-
}
|
|
1332
|
-
},
|
|
1333
|
-
"legacy/ragas_context_recall": {
|
|
1334
|
-
name: `Ragas Context Recall`,
|
|
1335
|
-
description: `
|
|
1336
|
-
This evaluator measures the extent to which the retrieved context aligns with the annotated answer, treated as the ground truth. Higher values indicate better performance.
|
|
1337
|
-
`,
|
|
1338
|
-
category: "rag",
|
|
1339
|
-
docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_recall.html",
|
|
1340
|
-
isGuardrail: false,
|
|
1341
|
-
requiredFields: ["input", "contexts", "expected_output"],
|
|
1342
|
-
optionalFields: [],
|
|
1343
|
-
settings: {
|
|
1344
|
-
model: {
|
|
1345
|
-
description: "The model to use for evaluation.",
|
|
1346
|
-
default: "openai/gpt-5-mini"
|
|
1347
|
-
},
|
|
1348
|
-
embeddings_model: {
|
|
1349
|
-
description: "The model to use for embeddings.",
|
|
1350
|
-
default: "openai/text-embedding-ada-002"
|
|
1351
|
-
},
|
|
1352
|
-
max_tokens: {
|
|
1353
|
-
description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
|
|
1354
|
-
default: 2048
|
|
1355
|
-
}
|
|
1356
|
-
},
|
|
1357
|
-
envVars: [],
|
|
1358
|
-
result: {
|
|
1359
|
-
score: {
|
|
1360
|
-
description: "A score between 0.0 and 1.0 indicating the recall of the context."
|
|
1361
|
-
}
|
|
1362
|
-
}
|
|
1363
|
-
},
|
|
1364
|
-
"legacy/ragas_context_relevancy": {
|
|
1365
|
-
name: `Ragas Context Relevancy`,
|
|
1366
|
-
description: `
|
|
1367
|
-
This metric gauges the relevancy of the retrieved context, calculated based on both the question and contexts. The values fall within the range of (0, 1), with higher values indicating better relevancy.
|
|
1368
|
-
`,
|
|
1369
|
-
category: "rag",
|
|
1370
|
-
docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_relevancy.html",
|
|
1371
|
-
isGuardrail: false,
|
|
1372
|
-
requiredFields: ["output", "contexts"],
|
|
1373
|
-
optionalFields: [],
|
|
1374
|
-
settings: {
|
|
1375
|
-
model: {
|
|
1376
|
-
description: "The model to use for evaluation.",
|
|
1377
|
-
default: "openai/gpt-5-mini"
|
|
1378
|
-
},
|
|
1379
|
-
embeddings_model: {
|
|
1380
|
-
description: "The model to use for embeddings.",
|
|
1381
|
-
default: "openai/text-embedding-ada-002"
|
|
1382
|
-
},
|
|
1383
|
-
max_tokens: {
|
|
1384
|
-
description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
|
|
1385
|
-
default: 2048
|
|
1386
|
-
}
|
|
1387
|
-
},
|
|
1388
|
-
envVars: [],
|
|
1389
|
-
result: {
|
|
1390
|
-
score: {
|
|
1391
|
-
description: "A score between 0.0 and 1.0 indicating the relevancy of the context."
|
|
1392
|
-
}
|
|
1393
|
-
}
|
|
1394
|
-
},
|
|
1395
|
-
"legacy/ragas_context_utilization": {
|
|
1396
|
-
name: `Ragas Context Utilization`,
|
|
1397
|
-
description: `
|
|
1398
|
-
This metric evaluates whether all of the output relevant items present in the contexts are ranked higher or not. Higher scores indicate better utilization.
|
|
1399
|
-
`,
|
|
1400
|
-
category: "rag",
|
|
1401
|
-
docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/context_precision.html",
|
|
1402
|
-
isGuardrail: false,
|
|
1403
|
-
requiredFields: ["input", "output", "contexts"],
|
|
1404
|
-
optionalFields: [],
|
|
1405
|
-
settings: {
|
|
1406
|
-
model: {
|
|
1407
|
-
description: "The model to use for evaluation.",
|
|
1408
|
-
default: "openai/gpt-5-mini"
|
|
1409
|
-
},
|
|
1410
|
-
embeddings_model: {
|
|
1411
|
-
description: "The model to use for embeddings.",
|
|
1412
|
-
default: "openai/text-embedding-ada-002"
|
|
1413
|
-
},
|
|
1414
|
-
max_tokens: {
|
|
1415
|
-
description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
|
|
1416
|
-
default: 2048
|
|
1417
|
-
}
|
|
1418
|
-
},
|
|
1419
|
-
envVars: [],
|
|
1420
|
-
result: {
|
|
1421
|
-
score: {
|
|
1422
|
-
description: "A score between 0.0 and 1.0 indicating the utilization of the context."
|
|
1423
|
-
}
|
|
1424
|
-
}
|
|
1425
|
-
},
|
|
1426
|
-
"legacy/ragas_faithfulness": {
|
|
1427
|
-
name: `Ragas Faithfulness`,
|
|
1428
|
-
description: `
|
|
1429
|
-
This evaluator assesses the extent to which the generated answer is consistent with the provided context. Higher scores indicate better faithfulness to the context, useful for detecting hallucinations.
|
|
1430
|
-
`,
|
|
1431
|
-
category: "rag",
|
|
1432
|
-
docsUrl: "https://docs.ragas.io/en/latest/concepts/metrics/faithfulness.html",
|
|
1433
|
-
isGuardrail: false,
|
|
1434
|
-
requiredFields: ["output", "contexts"],
|
|
1435
|
-
optionalFields: [],
|
|
1436
|
-
settings: {
|
|
1437
|
-
model: {
|
|
1438
|
-
description: "The model to use for evaluation.",
|
|
1439
|
-
default: "openai/gpt-5-mini"
|
|
1440
|
-
},
|
|
1441
|
-
embeddings_model: {
|
|
1442
|
-
description: "The model to use for embeddings.",
|
|
1443
|
-
default: "openai/text-embedding-ada-002"
|
|
1444
|
-
},
|
|
1445
|
-
max_tokens: {
|
|
1446
|
-
description: "The maximum number of tokens allowed for evaluation, a too high number can be costly. Entries above this amount will be skipped.",
|
|
1447
|
-
default: 2048
|
|
1448
|
-
}
|
|
1449
|
-
},
|
|
1450
|
-
envVars: [],
|
|
1451
|
-
result: {
|
|
1452
|
-
score: {
|
|
1453
|
-
description: "A score between 0.0 and 1.0 indicating the faithfulness of the answer."
|
|
1454
|
-
}
|
|
1455
|
-
}
|
|
1456
|
-
},
|
|
1457
1184
|
"langevals/basic": {
|
|
1458
1185
|
name: `Custom Basic Evaluator`,
|
|
1459
1186
|
description: `
|
|
@@ -1597,8 +1324,8 @@ A simple evaluator that checks if the output matches the expected_output exactly
|
|
|
1597
1324
|
category: "quality",
|
|
1598
1325
|
docsUrl: "",
|
|
1599
1326
|
isGuardrail: false,
|
|
1600
|
-
requiredFields: [],
|
|
1601
|
-
optionalFields: [
|
|
1327
|
+
requiredFields: ["output", "expected_output"],
|
|
1328
|
+
optionalFields: [],
|
|
1602
1329
|
settings: {
|
|
1603
1330
|
case_sensitive: {
|
|
1604
1331
|
description: "True if the comparison should be case-sensitive, False otherwise",
|
|
@@ -1628,8 +1355,8 @@ Uses an LLM to check if the generated output answers a question correctly the sa
|
|
|
1628
1355
|
category: "quality",
|
|
1629
1356
|
docsUrl: "",
|
|
1630
1357
|
isGuardrail: false,
|
|
1631
|
-
requiredFields: [],
|
|
1632
|
-
optionalFields: ["input"
|
|
1358
|
+
requiredFields: ["output", "expected_output"],
|
|
1359
|
+
optionalFields: ["input"],
|
|
1633
1360
|
settings: {
|
|
1634
1361
|
model: {
|
|
1635
1362
|
description: "The model to use for evaluation",
|
|
@@ -1866,19 +1593,44 @@ better one. Superseded by Comparison, which judges two or more candidates.
|
|
|
1866
1593
|
}
|
|
1867
1594
|
}
|
|
1868
1595
|
},
|
|
1596
|
+
"langevals/query_resolution": {
|
|
1597
|
+
name: `Query Resolution`,
|
|
1598
|
+
description: `
|
|
1599
|
+
This evaluator checks if all the user queries in the conversation were resolved. Useful to detect when the bot doesn't know how to answer or can't help the user.
|
|
1600
|
+
`,
|
|
1601
|
+
category: "quality",
|
|
1602
|
+
docsUrl: "",
|
|
1603
|
+
isGuardrail: false,
|
|
1604
|
+
requiredFields: ["conversation"],
|
|
1605
|
+
optionalFields: [],
|
|
1606
|
+
settings: {
|
|
1607
|
+
model: {
|
|
1608
|
+
description: "The model to use for evaluation",
|
|
1609
|
+
default: "openai/gpt-5-mini"
|
|
1610
|
+
},
|
|
1611
|
+
max_tokens: {
|
|
1612
|
+
description: "Max tokens allowed for evaluation",
|
|
1613
|
+
default: 128e3
|
|
1614
|
+
}
|
|
1615
|
+
},
|
|
1616
|
+
envVars: [],
|
|
1617
|
+
result: {}
|
|
1618
|
+
},
|
|
1869
1619
|
"langevals/select_best_compare": {
|
|
1870
1620
|
name: `Comparison`,
|
|
1871
1621
|
description: `
|
|
1872
1622
|
Compare two or more candidate outputs and pick the best one, optionally
|
|
1873
1623
|
against a reference answer. The judge sees every candidate at once and
|
|
1874
1624
|
explains why the winner is better. Candidate order is shuffled so that a
|
|
1875
|
-
candidate's position never sways the verdict
|
|
1625
|
+
candidate's position never sways the verdict, and by default each row is
|
|
1626
|
+
checked twice \u2014 once more with the order reversed \u2014 recording no result
|
|
1627
|
+
for that row when the two checks disagree, rather than guessing.
|
|
1876
1628
|
`,
|
|
1877
1629
|
category: "quality",
|
|
1878
1630
|
docsUrl: "",
|
|
1879
1631
|
isGuardrail: false,
|
|
1880
|
-
requiredFields: [
|
|
1881
|
-
optionalFields: ["input", "golden", "row_index"],
|
|
1632
|
+
requiredFields: [],
|
|
1633
|
+
optionalFields: ["input", "golden", "candidates", "row_index"],
|
|
1882
1634
|
settings: {
|
|
1883
1635
|
model: {
|
|
1884
1636
|
description: "The model to use for evaluation",
|
|
@@ -1918,6 +1670,14 @@ slot label. Use "tie" only when no candidate is clearly better.
|
|
|
1918
1670
|
include_metrics: {
|
|
1919
1671
|
description: "Per-candidate metrics to inject into the judge prompt",
|
|
1920
1672
|
default: []
|
|
1673
|
+
},
|
|
1674
|
+
temperature: {
|
|
1675
|
+
description: "Sampling temperature for the judge call. Lower is more deterministic.",
|
|
1676
|
+
default: 0
|
|
1677
|
+
},
|
|
1678
|
+
swap_and_reconcile: {
|
|
1679
|
+
description: "Check each row twice, the second time with the candidate order reversed, and record no result for that row when the two disagree \u2014 rather than a tie, which would claim the candidates are equally good. Doubles judge-call cost per row (literature: swapping candidate order alone flips 10-30% of verdicts on contested rows).",
|
|
1680
|
+
default: true
|
|
1921
1681
|
}
|
|
1922
1682
|
},
|
|
1923
1683
|
envVars: [],
|
|
@@ -1930,29 +1690,6 @@ slot label. Use "tie" only when no candidate is clearly better.
|
|
|
1930
1690
|
}
|
|
1931
1691
|
}
|
|
1932
1692
|
},
|
|
1933
|
-
"langevals/query_resolution": {
|
|
1934
|
-
name: `Query Resolution`,
|
|
1935
|
-
description: `
|
|
1936
|
-
This evaluator checks if all the user queries in the conversation were resolved. Useful to detect when the bot doesn't know how to answer or can't help the user.
|
|
1937
|
-
`,
|
|
1938
|
-
category: "quality",
|
|
1939
|
-
docsUrl: "",
|
|
1940
|
-
isGuardrail: false,
|
|
1941
|
-
requiredFields: ["conversation"],
|
|
1942
|
-
optionalFields: [],
|
|
1943
|
-
settings: {
|
|
1944
|
-
model: {
|
|
1945
|
-
description: "The model to use for evaluation",
|
|
1946
|
-
default: "openai/gpt-5-mini"
|
|
1947
|
-
},
|
|
1948
|
-
max_tokens: {
|
|
1949
|
-
description: "Max tokens allowed for evaluation",
|
|
1950
|
-
default: 128e3
|
|
1951
|
-
}
|
|
1952
|
-
},
|
|
1953
|
-
envVars: [],
|
|
1954
|
-
result: {}
|
|
1955
|
-
},
|
|
1956
1693
|
"langevals/sentiment": {
|
|
1957
1694
|
name: `Sentiment Evaluator`,
|
|
1958
1695
|
description: `
|
|
@@ -2172,4 +1909,4 @@ function extractFirstLine(description) {
|
|
|
2172
1909
|
export {
|
|
2173
1910
|
formatEvaluatorSchema
|
|
2174
1911
|
};
|
|
2175
|
-
//# sourceMappingURL=discover-evaluator-schema-
|
|
1912
|
+
//# sourceMappingURL=discover-evaluator-schema-ESSHQGPY.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import "./chunk-
|
|
1
|
+
import "./chunk-SAS5OB7F.js";
|
|
2
2
|
|
|
3
3
|
// src/tools/discover-scenario-schema.ts
|
|
4
4
|
function formatScenarioSchema() {
|
|
@@ -17,6 +17,58 @@ function formatScenarioSchema() {
|
|
|
17
17
|
lines.push(
|
|
18
18
|
'- **labels** (array of strings): Tags for organizing scenarios (e.g., "auth", "happy-path", "edge-case")'
|
|
19
19
|
);
|
|
20
|
+
lines.push(
|
|
21
|
+
"- **testSuiteId** (string): The test suite the scenario is filed in. Every project has a Default suite."
|
|
22
|
+
);
|
|
23
|
+
lines.push(
|
|
24
|
+
`- **fields** (object): The scenario's value for each field its test suite declares, by identifier (e.g. { "golden_sql": "SELECT ..." }). A text field takes a string, a number field a number, a boolean field true or false. A field the suite does not declare is refused. A blank value skips the evaluators that read it.`
|
|
25
|
+
);
|
|
26
|
+
lines.push("\n## Test Suite Fields\n");
|
|
27
|
+
lines.push(
|
|
28
|
+
"A test suite reads like a dataset: it declares typed fields beyond the situation and the criteria, and every scenario filed in it carries one value per field. Declare them with `platform_create_test_suite` or `platform_update_test_suite`:"
|
|
29
|
+
);
|
|
30
|
+
lines.push("```json");
|
|
31
|
+
lines.push('{ "fields": [{ "identifier": "golden_sql", "type": "text" }, { "identifier": "row_limit", "type": "number" }] }');
|
|
32
|
+
lines.push("```");
|
|
33
|
+
lines.push(
|
|
34
|
+
"- **identifier**: lowercase letters, digits and underscores, starting with a letter. situation, criteria, name, input and output are reserved."
|
|
35
|
+
);
|
|
36
|
+
lines.push("- **type**: text, number or boolean.");
|
|
37
|
+
lines.push(
|
|
38
|
+
"- On update, the list replaces the one the suite holds. A field an attached evaluator still reads cannot be removed."
|
|
39
|
+
);
|
|
40
|
+
lines.push("\n## Evaluator Attachments\n");
|
|
41
|
+
lines.push(
|
|
42
|
+
"A saved evaluator attaches to a test suite (and, for checks that read only the conversation or the trace, to a run plan). After every scenario run the platform runs it and stores one result per evaluator on the run. A **required** pass/fail evaluator that fails fails the scenario; a score-only evaluator reports beside the verdict and never gates."
|
|
43
|
+
);
|
|
44
|
+
lines.push("```json");
|
|
45
|
+
lines.push('{ "evaluators": [{');
|
|
46
|
+
lines.push(' "evaluatorId": "<id from platform_list_evaluators>",');
|
|
47
|
+
lines.push(' "required": true,');
|
|
48
|
+
lines.push(' "mappings": {');
|
|
49
|
+
lines.push(' "output": { "type": "source", "sourceId": "trace", "path": ["tool_calls", "run_sql", "input"] },');
|
|
50
|
+
lines.push(' "expected_output": { "type": "source", "sourceId": "scenario", "path": ["fields", "golden_sql"] }');
|
|
51
|
+
lines.push(" }");
|
|
52
|
+
lines.push("}] }");
|
|
53
|
+
lines.push("```");
|
|
54
|
+
lines.push(
|
|
55
|
+
"Each mapping names where one evaluator input reads its value. Read the evaluator's inputs from `platform_get_evaluator`; every required input needs a mapping before a run. The paths:"
|
|
56
|
+
);
|
|
57
|
+
lines.push(
|
|
58
|
+
'- **conversation**: `["first_user_message"]`, `["last_agent_message"]`, `["transcript"]` (role: content lines) or `["messages"]` (JSON)'
|
|
59
|
+
);
|
|
60
|
+
lines.push(
|
|
61
|
+
'- **scenario**: `["situation"]`, `["criteria"]` or `["fields", "<identifier>"]` (a field the suite declares; not on a run plan)'
|
|
62
|
+
);
|
|
63
|
+
lines.push(
|
|
64
|
+
'- **trace**: `["contexts"]` (the retrieved contexts of the run\'s traces) or `["tool_calls", "<toolName>", "input" | "output"]` (the last matching tool call)'
|
|
65
|
+
);
|
|
66
|
+
lines.push(
|
|
67
|
+
'- a literal: `{ "type": "value", "value": "..." }`'
|
|
68
|
+
);
|
|
69
|
+
lines.push(
|
|
70
|
+
"A scenario whose field is blank skips the evaluators that read it, with a reason. A trace with no such tool call or no contexts fails them, with a reason. The results come back under `results.evaluations` on `platform_get_simulation_run`."
|
|
71
|
+
);
|
|
20
72
|
lines.push("\n## Writing a Good Situation\n");
|
|
21
73
|
lines.push(
|
|
22
74
|
"The situation drives the user simulator. Include these elements:"
|
|
@@ -64,4 +116,4 @@ function formatScenarioSchema() {
|
|
|
64
116
|
export {
|
|
65
117
|
formatScenarioSchema
|
|
66
118
|
};
|
|
67
|
-
//# sourceMappingURL=discover-scenario-schema-
|
|
119
|
+
//# sourceMappingURL=discover-scenario-schema-W7KBXA4J.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import "./chunk-
|
|
1
|
+
import "./chunk-SAS5OB7F.js";
|
|
2
2
|
|
|
3
3
|
// src/schemas/filter-fields.ts
|
|
4
4
|
var filterFields = [
|
|
@@ -445,4 +445,4 @@ function formatGroups() {
|
|
|
445
445
|
export {
|
|
446
446
|
formatSchema
|
|
447
447
|
};
|
|
448
|
-
//# sourceMappingURL=discover-schema-
|
|
448
|
+
//# sourceMappingURL=discover-schema-L6QDPON3.js.map
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import {
|
|
2
|
+
getAgent
|
|
3
|
+
} from "./chunk-EYXMLE5X.js";
|
|
4
|
+
import "./chunk-GELJ2MQR.js";
|
|
5
|
+
import "./chunk-WT4PO4A3.js";
|
|
6
|
+
import "./chunk-SAS5OB7F.js";
|
|
7
|
+
|
|
8
|
+
// src/tools/get-agent.ts
|
|
9
|
+
var describeParameter = ({
|
|
10
|
+
name,
|
|
11
|
+
type,
|
|
12
|
+
options,
|
|
13
|
+
default: defaultValue,
|
|
14
|
+
description,
|
|
15
|
+
required
|
|
16
|
+
}) => {
|
|
17
|
+
const parts = [type];
|
|
18
|
+
if (options?.length) parts.push(`one of ${options.join(", ")}`);
|
|
19
|
+
if (defaultValue !== void 0) parts.push(`default ${JSON.stringify(defaultValue)}`);
|
|
20
|
+
if (required) parts.push("required");
|
|
21
|
+
const suffix = description ? `: ${description}` : "";
|
|
22
|
+
return `- **${name}** (${parts.join(", ")})${suffix}`;
|
|
23
|
+
};
|
|
24
|
+
async function handleGetAgent({ id }) {
|
|
25
|
+
const agent = await getAgent(id);
|
|
26
|
+
const lines = [];
|
|
27
|
+
lines.push(`# ${agent.name}
|
|
28
|
+
`);
|
|
29
|
+
lines.push(`**ID**: ${agent.id}`);
|
|
30
|
+
lines.push(`**Type**: ${agent.type}`);
|
|
31
|
+
if (agent.environment) lines.push(`**Environment**: ${agent.environment}`);
|
|
32
|
+
if (agent.status) lines.push(`**Status**: ${agent.status}`);
|
|
33
|
+
if (agent.owner?.name) lines.push(`**Owner**: ${agent.owner.name}`);
|
|
34
|
+
else if (agent.hostLabel) lines.push(`**Host**: ${agent.hostLabel}`);
|
|
35
|
+
if (agent.selectable === false) {
|
|
36
|
+
lines.push("**Run**: only its owner can run this agent");
|
|
37
|
+
}
|
|
38
|
+
if (agent.lastSeenAt) lines.push(`**Last seen**: ${agent.lastSeenAt}`);
|
|
39
|
+
lines.push(`**Created**: ${agent.createdAt}`);
|
|
40
|
+
lines.push(`**Updated**: ${agent.updatedAt}`);
|
|
41
|
+
if (agent.parameters && agent.parameters.length > 0) {
|
|
42
|
+
lines.push("\n## Parameters\n");
|
|
43
|
+
for (const parameter of agent.parameters) lines.push(describeParameter(parameter));
|
|
44
|
+
}
|
|
45
|
+
if (agent.instances && agent.instances.length > 0) {
|
|
46
|
+
lines.push(`
|
|
47
|
+
## Instances (${agent.instances.length})
|
|
48
|
+
`);
|
|
49
|
+
for (const instance of agent.instances) {
|
|
50
|
+
const label = instance.label ? ` (${instance.label})` : "";
|
|
51
|
+
lines.push(`- ${instance.hostname || instance.id}${label}, connected ${instance.connectedAt}`);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
if (agent.config && Object.keys(agent.config).length > 0) {
|
|
55
|
+
lines.push("\n## Config\n");
|
|
56
|
+
lines.push("```json");
|
|
57
|
+
lines.push(JSON.stringify(agent.config, null, 2));
|
|
58
|
+
lines.push("```");
|
|
59
|
+
}
|
|
60
|
+
return lines.join("\n");
|
|
61
|
+
}
|
|
62
|
+
export {
|
|
63
|
+
describeParameter,
|
|
64
|
+
handleGetAgent
|
|
65
|
+
};
|
|
66
|
+
//# sourceMappingURL=get-agent-7THF7HCU.js.map
|
|
@@ -3,9 +3,9 @@ import {
|
|
|
3
3
|
} from "./chunk-ZXKLPC2E.js";
|
|
4
4
|
import {
|
|
5
5
|
getAnalyticsTimeseries
|
|
6
|
-
} from "./chunk-
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
6
|
+
} from "./chunk-GELJ2MQR.js";
|
|
7
|
+
import "./chunk-WT4PO4A3.js";
|
|
8
|
+
import "./chunk-SAS5OB7F.js";
|
|
9
9
|
|
|
10
10
|
// src/tools/get-analytics.ts
|
|
11
11
|
function getGroupedData(bucket, groupBy) {
|
|
@@ -70,4 +70,4 @@ async function handleGetAnalytics(params) {
|
|
|
70
70
|
export {
|
|
71
71
|
handleGetAnalytics
|
|
72
72
|
};
|
|
73
|
-
//# sourceMappingURL=get-analytics-
|
|
73
|
+
//# sourceMappingURL=get-analytics-R6AZD3AW.js.map
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
getDashboard
|
|
3
|
-
} from "./chunk-
|
|
4
|
-
import "./chunk-
|
|
5
|
-
import "./chunk-
|
|
6
|
-
import "./chunk-
|
|
3
|
+
} from "./chunk-S4GVX36W.js";
|
|
4
|
+
import "./chunk-GELJ2MQR.js";
|
|
5
|
+
import "./chunk-WT4PO4A3.js";
|
|
6
|
+
import "./chunk-SAS5OB7F.js";
|
|
7
7
|
|
|
8
8
|
// src/tools/get-dashboard.ts
|
|
9
9
|
async function handleGetDashboard(params) {
|
|
@@ -27,4 +27,4 @@ async function handleGetDashboard(params) {
|
|
|
27
27
|
export {
|
|
28
28
|
handleGetDashboard
|
|
29
29
|
};
|
|
30
|
-
//# sourceMappingURL=get-dashboard-
|
|
30
|
+
//# sourceMappingURL=get-dashboard-P4WWVHFU.js.map
|
|
@@ -3,10 +3,10 @@ import {
|
|
|
3
3
|
} from "./chunk-NUFPTHKU.js";
|
|
4
4
|
import {
|
|
5
5
|
getDataset
|
|
6
|
-
} from "./chunk-
|
|
7
|
-
import "./chunk-
|
|
8
|
-
import "./chunk-
|
|
9
|
-
import "./chunk-
|
|
6
|
+
} from "./chunk-AN4OUA4D.js";
|
|
7
|
+
import "./chunk-GELJ2MQR.js";
|
|
8
|
+
import "./chunk-WT4PO4A3.js";
|
|
9
|
+
import "./chunk-SAS5OB7F.js";
|
|
10
10
|
|
|
11
11
|
// src/tools/get-dataset.ts
|
|
12
12
|
function formatDatasetResponse(dataset) {
|
|
@@ -46,4 +46,4 @@ export {
|
|
|
46
46
|
formatDatasetResponse,
|
|
47
47
|
handleGetDataset
|
|
48
48
|
};
|
|
49
|
-
//# sourceMappingURL=get-dataset-
|
|
49
|
+
//# sourceMappingURL=get-dataset-AKTYSQDI.js.map
|