@bryan-thompson/inspector-assessment-client 1.10.1 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-Bi_akC9k.js → OAuthCallback-DA2koy6X.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-DupZK8qN.js → OAuthDebugCallback-Bx60PQTT.js} +1 -1
- package/dist/assets/{index-BQQRMUvH.js → index-kJ0jPd4m.js} +1218 -384
- package/dist/index.html +1 -1
- package/lib/lib/assessmentTypes.d.ts +68 -1
- package/lib/lib/assessmentTypes.d.ts.map +1 -1
- package/lib/lib/moduleScoring.d.ts +23 -0
- package/lib/lib/moduleScoring.d.ts.map +1 -0
- package/lib/lib/moduleScoring.js +53 -0
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +14 -37
- package/lib/services/assessment/TestDataGenerator.d.ts +22 -0
- package/lib/services/assessment/TestDataGenerator.d.ts.map +1 -1
- package/lib/services/assessment/TestDataGenerator.js +78 -0
- package/lib/services/assessment/modules/FunctionalityAssessor.d.ts +15 -0
- package/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.js +137 -6
- package/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/SecurityAssessor.js +26 -0
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +4 -0
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +81 -1
- package/package.json +1 -1
|
@@ -16320,7 +16320,7 @@ object({
|
|
|
16320
16320
|
token_type_hint: string().optional()
|
|
16321
16321
|
}).strip();
|
|
16322
16322
|
const name = "@bryan-thompson/inspector-assessment-client";
|
|
16323
|
-
const version$1 = "1.
|
|
16323
|
+
const version$1 = "1.11.1";
|
|
16324
16324
|
const packageJson = {
|
|
16325
16325
|
name,
|
|
16326
16326
|
version: version$1
|
|
@@ -48863,7 +48863,7 @@ const useTheme = () => {
|
|
|
48863
48863
|
[theme, setThemeWithSideEffect]
|
|
48864
48864
|
);
|
|
48865
48865
|
};
|
|
48866
|
-
const version = "1.
|
|
48866
|
+
const version = "1.11.1";
|
|
48867
48867
|
var [createTooltipContext] = createContextScope("Tooltip", [
|
|
48868
48868
|
createPopperScope
|
|
48869
48869
|
]);
|
|
@@ -53608,7 +53608,1065 @@ class ResponseValidator {
|
|
|
53608
53608
|
return totalWeight > 0 ? weightedSum / totalWeight * 100 : 0;
|
|
53609
53609
|
}
|
|
53610
53610
|
}
|
|
53611
|
+
var ToolCategory = /* @__PURE__ */ ((ToolCategory2) => {
|
|
53612
|
+
ToolCategory2["CALCULATOR"] = "calculator";
|
|
53613
|
+
ToolCategory2["SYSTEM_EXEC"] = "system_exec";
|
|
53614
|
+
ToolCategory2["DATA_ACCESS"] = "data_access";
|
|
53615
|
+
ToolCategory2["TOOL_OVERRIDE"] = "tool_override";
|
|
53616
|
+
ToolCategory2["CONFIG_MODIFIER"] = "config_modifier";
|
|
53617
|
+
ToolCategory2["URL_FETCHER"] = "fetcher";
|
|
53618
|
+
ToolCategory2["UNICODE_PROCESSOR"] = "unicode";
|
|
53619
|
+
ToolCategory2["JSON_PARSER"] = "parser";
|
|
53620
|
+
ToolCategory2["PACKAGE_INSTALLER"] = "installer";
|
|
53621
|
+
ToolCategory2["RUG_PULL"] = "rug_pull";
|
|
53622
|
+
ToolCategory2["SAFE_STORAGE"] = "safe_storage";
|
|
53623
|
+
ToolCategory2["API_WRAPPER"] = "api_wrapper";
|
|
53624
|
+
ToolCategory2["SEARCH_RETRIEVAL"] = "search_retrieval";
|
|
53625
|
+
ToolCategory2["CRUD_CREATION"] = "crud_creation";
|
|
53626
|
+
ToolCategory2["READ_ONLY_INFO"] = "read_only_info";
|
|
53627
|
+
ToolCategory2["GENERIC"] = "generic";
|
|
53628
|
+
return ToolCategory2;
|
|
53629
|
+
})(ToolCategory || {});
|
|
53630
|
+
class ToolClassifier {
|
|
53631
|
+
/**
|
|
53632
|
+
* Classify a tool into one or more categories
|
|
53633
|
+
* Returns multiple categories if tool matches multiple patterns
|
|
53634
|
+
*/
|
|
53635
|
+
classify(toolName, description2) {
|
|
53636
|
+
const categories = [];
|
|
53637
|
+
const confidenceScores = [];
|
|
53638
|
+
const reasons = [];
|
|
53639
|
+
const toolText = `${toolName} ${description2 || ""}`.toLowerCase();
|
|
53640
|
+
if (this.matchesPattern(toolText, [
|
|
53641
|
+
/calculator/i,
|
|
53642
|
+
/compute/i,
|
|
53643
|
+
/math/i,
|
|
53644
|
+
/calc/i,
|
|
53645
|
+
/eval/i,
|
|
53646
|
+
/arithmetic/i,
|
|
53647
|
+
/expression/i
|
|
53648
|
+
])) {
|
|
53649
|
+
categories.push(
|
|
53650
|
+
"calculator"
|
|
53651
|
+
/* CALCULATOR */
|
|
53652
|
+
);
|
|
53653
|
+
confidenceScores.push(90);
|
|
53654
|
+
reasons.push("Calculator pattern detected (arithmetic execution risk)");
|
|
53655
|
+
}
|
|
53656
|
+
if (this.matchesPattern(toolText, [
|
|
53657
|
+
/system.*exec/i,
|
|
53658
|
+
/exec.*tool/i,
|
|
53659
|
+
/command/i,
|
|
53660
|
+
/shell/i,
|
|
53661
|
+
/\brun\b/i,
|
|
53662
|
+
/execute/i,
|
|
53663
|
+
/process/i
|
|
53664
|
+
])) {
|
|
53665
|
+
categories.push(
|
|
53666
|
+
"system_exec"
|
|
53667
|
+
/* SYSTEM_EXEC */
|
|
53668
|
+
);
|
|
53669
|
+
confidenceScores.push(95);
|
|
53670
|
+
reasons.push(
|
|
53671
|
+
"System execution pattern detected (command injection risk)"
|
|
53672
|
+
);
|
|
53673
|
+
}
|
|
53674
|
+
if (this.matchesPattern(toolText, [
|
|
53675
|
+
/leak/i,
|
|
53676
|
+
/\bdata\b/i,
|
|
53677
|
+
/show/i,
|
|
53678
|
+
/\bget\b/i,
|
|
53679
|
+
/\blist\b/i,
|
|
53680
|
+
/display/i,
|
|
53681
|
+
/\benv/i,
|
|
53682
|
+
/secret/i,
|
|
53683
|
+
/\bkey\b/i,
|
|
53684
|
+
/credential/i,
|
|
53685
|
+
/exfiltrat/i
|
|
53686
|
+
])) {
|
|
53687
|
+
categories.push(
|
|
53688
|
+
"data_access"
|
|
53689
|
+
/* DATA_ACCESS */
|
|
53690
|
+
);
|
|
53691
|
+
confidenceScores.push(85);
|
|
53692
|
+
reasons.push("Data access pattern detected (data exfiltration risk)");
|
|
53693
|
+
}
|
|
53694
|
+
if (this.matchesPattern(toolText, [
|
|
53695
|
+
/override/i,
|
|
53696
|
+
/shadow/i,
|
|
53697
|
+
/poison/i,
|
|
53698
|
+
/create.*tool/i,
|
|
53699
|
+
/register.*tool/i,
|
|
53700
|
+
/define.*tool/i,
|
|
53701
|
+
/tool.*creator/i,
|
|
53702
|
+
/add.*tool/i
|
|
53703
|
+
])) {
|
|
53704
|
+
categories.push(
|
|
53705
|
+
"tool_override"
|
|
53706
|
+
/* TOOL_OVERRIDE */
|
|
53707
|
+
);
|
|
53708
|
+
confidenceScores.push(92);
|
|
53709
|
+
reasons.push("Tool override pattern detected (shadowing/poisoning risk)");
|
|
53710
|
+
}
|
|
53711
|
+
if (this.matchesPattern(toolText, [
|
|
53712
|
+
/config/i,
|
|
53713
|
+
/setting/i,
|
|
53714
|
+
/modifier/i,
|
|
53715
|
+
/\badmin\b/i,
|
|
53716
|
+
/privilege/i,
|
|
53717
|
+
/permission/i,
|
|
53718
|
+
/configure/i,
|
|
53719
|
+
/drift/i
|
|
53720
|
+
])) {
|
|
53721
|
+
categories.push(
|
|
53722
|
+
"config_modifier"
|
|
53723
|
+
/* CONFIG_MODIFIER */
|
|
53724
|
+
);
|
|
53725
|
+
confidenceScores.push(88);
|
|
53726
|
+
reasons.push(
|
|
53727
|
+
"Config modification pattern detected (configuration drift risk)"
|
|
53728
|
+
);
|
|
53729
|
+
}
|
|
53730
|
+
if (this.matchesPattern(toolText, [
|
|
53731
|
+
/fetch/i,
|
|
53732
|
+
/\burl\b/i,
|
|
53733
|
+
/http/i,
|
|
53734
|
+
/download/i,
|
|
53735
|
+
/load/i,
|
|
53736
|
+
/retrieve/i,
|
|
53737
|
+
/\bget\b.*url/i,
|
|
53738
|
+
/external/i
|
|
53739
|
+
])) {
|
|
53740
|
+
categories.push(
|
|
53741
|
+
"fetcher"
|
|
53742
|
+
/* URL_FETCHER */
|
|
53743
|
+
);
|
|
53744
|
+
confidenceScores.push(87);
|
|
53745
|
+
reasons.push(
|
|
53746
|
+
"URL fetcher pattern detected (indirect prompt injection risk)"
|
|
53747
|
+
);
|
|
53748
|
+
}
|
|
53749
|
+
if (this.matchesPattern(toolText, [
|
|
53750
|
+
/unicode/i,
|
|
53751
|
+
/encode/i,
|
|
53752
|
+
/decode/i,
|
|
53753
|
+
/charset/i,
|
|
53754
|
+
/utf/i,
|
|
53755
|
+
/hex/i,
|
|
53756
|
+
/escape/i
|
|
53757
|
+
])) {
|
|
53758
|
+
categories.push(
|
|
53759
|
+
"unicode"
|
|
53760
|
+
/* UNICODE_PROCESSOR */
|
|
53761
|
+
);
|
|
53762
|
+
confidenceScores.push(75);
|
|
53763
|
+
reasons.push("Unicode processor pattern detected (bypass encoding risk)");
|
|
53764
|
+
}
|
|
53765
|
+
if (this.matchesPattern(toolText, [
|
|
53766
|
+
/parser/i,
|
|
53767
|
+
/parse/i,
|
|
53768
|
+
/json/i,
|
|
53769
|
+
/xml/i,
|
|
53770
|
+
/yaml/i,
|
|
53771
|
+
/nested/i,
|
|
53772
|
+
/deserialize/i,
|
|
53773
|
+
/unmarshal/i
|
|
53774
|
+
])) {
|
|
53775
|
+
categories.push(
|
|
53776
|
+
"parser"
|
|
53777
|
+
/* JSON_PARSER */
|
|
53778
|
+
);
|
|
53779
|
+
confidenceScores.push(78);
|
|
53780
|
+
reasons.push(
|
|
53781
|
+
"JSON/nested parser pattern detected (nested injection risk)"
|
|
53782
|
+
);
|
|
53783
|
+
}
|
|
53784
|
+
if (this.matchesPattern(toolText, [
|
|
53785
|
+
/install/i,
|
|
53786
|
+
/package/i,
|
|
53787
|
+
/\bnpm\b/i,
|
|
53788
|
+
/\bpip\b/i,
|
|
53789
|
+
/dependency/i,
|
|
53790
|
+
/module/i,
|
|
53791
|
+
/library/i,
|
|
53792
|
+
/\bgem\b/i
|
|
53793
|
+
])) {
|
|
53794
|
+
categories.push(
|
|
53795
|
+
"installer"
|
|
53796
|
+
/* PACKAGE_INSTALLER */
|
|
53797
|
+
);
|
|
53798
|
+
confidenceScores.push(70);
|
|
53799
|
+
reasons.push("Package installer pattern detected (typosquatting risk)");
|
|
53800
|
+
}
|
|
53801
|
+
if (this.matchesPattern(toolText, [
|
|
53802
|
+
/rug.*pull/i,
|
|
53803
|
+
/trust/i,
|
|
53804
|
+
/behavior.*change/i,
|
|
53805
|
+
/malicious.*after/i,
|
|
53806
|
+
/invocation.*count/i
|
|
53807
|
+
])) {
|
|
53808
|
+
categories.push(
|
|
53809
|
+
"rug_pull"
|
|
53810
|
+
/* RUG_PULL */
|
|
53811
|
+
);
|
|
53812
|
+
confidenceScores.push(80);
|
|
53813
|
+
reasons.push("Rug pull pattern detected (behavioral change risk)");
|
|
53814
|
+
}
|
|
53815
|
+
if (this.matchesPattern(toolText, [
|
|
53816
|
+
/firecrawl/i,
|
|
53817
|
+
/\bscrape\b/i,
|
|
53818
|
+
/\bcrawl\b/i,
|
|
53819
|
+
/web.*scraping/i,
|
|
53820
|
+
/api.*wrapper/i,
|
|
53821
|
+
/http.*client/i,
|
|
53822
|
+
/web.*client/i,
|
|
53823
|
+
/rest.*client/i,
|
|
53824
|
+
/graphql.*client/i,
|
|
53825
|
+
/fetch.*web.*content/i
|
|
53826
|
+
])) {
|
|
53827
|
+
categories.push(
|
|
53828
|
+
"api_wrapper"
|
|
53829
|
+
/* API_WRAPPER */
|
|
53830
|
+
);
|
|
53831
|
+
confidenceScores.push(95);
|
|
53832
|
+
reasons.push(
|
|
53833
|
+
"API wrapper pattern detected (safe data passing, not code execution)"
|
|
53834
|
+
);
|
|
53835
|
+
}
|
|
53836
|
+
if (this.matchesPattern(toolText, [
|
|
53837
|
+
/\bsearch\b/i,
|
|
53838
|
+
/\bfind\b/i,
|
|
53839
|
+
/\blookup\b/i,
|
|
53840
|
+
/\bquery\b/i,
|
|
53841
|
+
/retrieve/i,
|
|
53842
|
+
/\blist\b/i,
|
|
53843
|
+
/get.*users/i,
|
|
53844
|
+
/get.*pages/i,
|
|
53845
|
+
/get.*database/i
|
|
53846
|
+
])) {
|
|
53847
|
+
categories.push(
|
|
53848
|
+
"search_retrieval"
|
|
53849
|
+
/* SEARCH_RETRIEVAL */
|
|
53850
|
+
);
|
|
53851
|
+
confidenceScores.push(93);
|
|
53852
|
+
reasons.push(
|
|
53853
|
+
"Search/retrieval pattern detected (returns data, not code execution)"
|
|
53854
|
+
);
|
|
53855
|
+
}
|
|
53856
|
+
if (this.matchesPattern(toolText, [
|
|
53857
|
+
/\bcreate\b/i,
|
|
53858
|
+
/\badd\b/i,
|
|
53859
|
+
/\binsert\b/i,
|
|
53860
|
+
/\bupdate\b/i,
|
|
53861
|
+
/\bmodify\b/i,
|
|
53862
|
+
/\bdelete\b/i,
|
|
53863
|
+
/\bduplicate\b/i,
|
|
53864
|
+
/\bmove\b/i,
|
|
53865
|
+
/\bappend\b/i
|
|
53866
|
+
])) {
|
|
53867
|
+
categories.push(
|
|
53868
|
+
"crud_creation"
|
|
53869
|
+
/* CRUD_CREATION */
|
|
53870
|
+
);
|
|
53871
|
+
confidenceScores.push(92);
|
|
53872
|
+
reasons.push(
|
|
53873
|
+
"CRUD operation pattern detected (data manipulation, not code execution)"
|
|
53874
|
+
);
|
|
53875
|
+
}
|
|
53876
|
+
if (this.matchesPattern(toolText, [
|
|
53877
|
+
/get.*self/i,
|
|
53878
|
+
/get.*teams/i,
|
|
53879
|
+
/get.*info/i,
|
|
53880
|
+
/get.*status/i,
|
|
53881
|
+
/\bwhoami\b/i,
|
|
53882
|
+
/get.*workspace/i,
|
|
53883
|
+
/get.*user/i,
|
|
53884
|
+
/current.*user/i
|
|
53885
|
+
])) {
|
|
53886
|
+
categories.push(
|
|
53887
|
+
"read_only_info"
|
|
53888
|
+
/* READ_ONLY_INFO */
|
|
53889
|
+
);
|
|
53890
|
+
confidenceScores.push(94);
|
|
53891
|
+
reasons.push(
|
|
53892
|
+
"Read-only info pattern detected (intended data exposure, not vulnerability)"
|
|
53893
|
+
);
|
|
53894
|
+
}
|
|
53895
|
+
if (this.matchesPattern(toolText, [
|
|
53896
|
+
/safe.*storage/i,
|
|
53897
|
+
/safe.*search/i,
|
|
53898
|
+
/safe.*list/i,
|
|
53899
|
+
/safe.*info/i,
|
|
53900
|
+
/safe.*echo/i,
|
|
53901
|
+
/safe.*validate/i,
|
|
53902
|
+
/safe.*tool/i
|
|
53903
|
+
])) {
|
|
53904
|
+
categories.push(
|
|
53905
|
+
"safe_storage"
|
|
53906
|
+
/* SAFE_STORAGE */
|
|
53907
|
+
);
|
|
53908
|
+
confidenceScores.push(99);
|
|
53909
|
+
reasons.push(
|
|
53910
|
+
"Safe tool pattern detected (control group - should be safe)"
|
|
53911
|
+
);
|
|
53912
|
+
}
|
|
53913
|
+
if (categories.length === 0) {
|
|
53914
|
+
categories.push(
|
|
53915
|
+
"generic"
|
|
53916
|
+
/* GENERIC */
|
|
53917
|
+
);
|
|
53918
|
+
confidenceScores.push(50);
|
|
53919
|
+
reasons.push("No specific pattern match, using generic tests");
|
|
53920
|
+
}
|
|
53921
|
+
const avgConfidence = confidenceScores.reduce((a, b) => a + b, 0) / confidenceScores.length;
|
|
53922
|
+
return {
|
|
53923
|
+
toolName,
|
|
53924
|
+
categories,
|
|
53925
|
+
confidence: Math.round(avgConfidence),
|
|
53926
|
+
reasoning: reasons.join("; ")
|
|
53927
|
+
};
|
|
53928
|
+
}
|
|
53929
|
+
/**
|
|
53930
|
+
* Check if text matches any of the provided patterns
|
|
53931
|
+
*/
|
|
53932
|
+
matchesPattern(text, patterns) {
|
|
53933
|
+
return patterns.some((pattern2) => pattern2.test(text));
|
|
53934
|
+
}
|
|
53935
|
+
/**
|
|
53936
|
+
* Get all tool categories (for testing/debugging)
|
|
53937
|
+
*/
|
|
53938
|
+
static getAllCategories() {
|
|
53939
|
+
return Object.values(ToolCategory);
|
|
53940
|
+
}
|
|
53941
|
+
/**
|
|
53942
|
+
* Get risk level for a category
|
|
53943
|
+
*/
|
|
53944
|
+
static getRiskLevel(category) {
|
|
53945
|
+
const highRiskCategories = [
|
|
53946
|
+
"calculator",
|
|
53947
|
+
"system_exec",
|
|
53948
|
+
"data_access",
|
|
53949
|
+
"tool_override",
|
|
53950
|
+
"config_modifier",
|
|
53951
|
+
"fetcher"
|
|
53952
|
+
/* URL_FETCHER */
|
|
53953
|
+
];
|
|
53954
|
+
const mediumRiskCategories = [
|
|
53955
|
+
"unicode",
|
|
53956
|
+
"parser",
|
|
53957
|
+
"installer",
|
|
53958
|
+
"rug_pull"
|
|
53959
|
+
/* RUG_PULL */
|
|
53960
|
+
];
|
|
53961
|
+
const lowRiskCategories = [
|
|
53962
|
+
"api_wrapper",
|
|
53963
|
+
"search_retrieval",
|
|
53964
|
+
"crud_creation",
|
|
53965
|
+
"read_only_info",
|
|
53966
|
+
"safe_storage",
|
|
53967
|
+
"generic"
|
|
53968
|
+
/* GENERIC */
|
|
53969
|
+
];
|
|
53970
|
+
if (highRiskCategories.includes(category)) return "HIGH";
|
|
53971
|
+
if (mediumRiskCategories.includes(category)) return "MEDIUM";
|
|
53972
|
+
if (lowRiskCategories.includes(category)) return "LOW";
|
|
53973
|
+
return "LOW";
|
|
53974
|
+
}
|
|
53975
|
+
/**
|
|
53976
|
+
* Classify multiple tools at once
|
|
53977
|
+
*/
|
|
53978
|
+
classifyBatch(tools) {
|
|
53979
|
+
return tools.map((tool) => this.classify(tool.name, tool.description));
|
|
53980
|
+
}
|
|
53981
|
+
}
|
|
53982
|
+
class TestDataGenerator {
|
|
53983
|
+
// Optional Claude Code bridge for intelligent test generation
|
|
53984
|
+
static claudeBridge = null;
|
|
53985
|
+
/**
|
|
53986
|
+
* Set the Claude Code bridge for intelligent test generation
|
|
53987
|
+
* Call this once during initialization if Claude integration is enabled
|
|
53988
|
+
*/
|
|
53989
|
+
static setClaudeBridge(bridge) {
|
|
53990
|
+
this.claudeBridge = bridge;
|
|
53991
|
+
}
|
|
53992
|
+
/**
|
|
53993
|
+
* Check if Claude Code integration is available and enabled
|
|
53994
|
+
*/
|
|
53995
|
+
static isClaudeEnabled() {
|
|
53996
|
+
return this.claudeBridge !== null && this.claudeBridge.isFeatureEnabled("intelligentTestGeneration");
|
|
53997
|
+
}
|
|
53998
|
+
// Realistic data pools for different types - using values that are more likely to exist
|
|
53999
|
+
static REALISTIC_DATA = {
|
|
54000
|
+
urls: [
|
|
54001
|
+
"https://www.google.com",
|
|
54002
|
+
// Public, always accessible
|
|
54003
|
+
"https://api.github.com/users/octocat",
|
|
54004
|
+
// Public API endpoint that exists
|
|
54005
|
+
"https://jsonplaceholder.typicode.com/posts/1",
|
|
54006
|
+
// Test API that always works
|
|
54007
|
+
"https://httpbin.org/get",
|
|
54008
|
+
// HTTP testing service
|
|
54009
|
+
"https://example.com",
|
|
54010
|
+
// RFC 2606 reserved domain for examples
|
|
54011
|
+
"https://www.wikipedia.org",
|
|
54012
|
+
// Public, stable site
|
|
54013
|
+
"https://api.openweathermap.org/data/2.5/weather?q=London"
|
|
54014
|
+
// Public API
|
|
54015
|
+
],
|
|
54016
|
+
emails: [
|
|
54017
|
+
"admin@example.com",
|
|
54018
|
+
// Common admin email
|
|
54019
|
+
"support@example.com",
|
|
54020
|
+
// Common support email
|
|
54021
|
+
"info@example.com",
|
|
54022
|
+
// Common info email
|
|
54023
|
+
"test@test.com",
|
|
54024
|
+
// Generic test email
|
|
54025
|
+
"user@domain.com",
|
|
54026
|
+
// Generic user email
|
|
54027
|
+
"noreply@example.com",
|
|
54028
|
+
// Common no-reply format
|
|
54029
|
+
"hello@world.com"
|
|
54030
|
+
// Simple, memorable
|
|
54031
|
+
],
|
|
54032
|
+
names: [
|
|
54033
|
+
"Default",
|
|
54034
|
+
// Common default name
|
|
54035
|
+
"Admin",
|
|
54036
|
+
// Common admin user
|
|
54037
|
+
"Test User",
|
|
54038
|
+
// Clear test user
|
|
54039
|
+
"Sample Item",
|
|
54040
|
+
// Generic sample
|
|
54041
|
+
"Example Project",
|
|
54042
|
+
// Clear example
|
|
54043
|
+
"Demo Application",
|
|
54044
|
+
// Common demo name
|
|
54045
|
+
"Main"
|
|
54046
|
+
// Common main/primary name
|
|
54047
|
+
],
|
|
54048
|
+
ids: [
|
|
54049
|
+
"1",
|
|
54050
|
+
// Simple numeric ID that often exists
|
|
54051
|
+
"123",
|
|
54052
|
+
// Common test ID
|
|
54053
|
+
"550e8400-e29b-41d4-a716-446655440000",
|
|
54054
|
+
// Valid UUID v4 (replaces "test")
|
|
54055
|
+
"default",
|
|
54056
|
+
// Common default ID
|
|
54057
|
+
"main",
|
|
54058
|
+
// Common main ID
|
|
54059
|
+
"264051cd-48ab-80ff-864e-d1aa9bc41429",
|
|
54060
|
+
// Valid UUID from realistic data
|
|
54061
|
+
"00000000-0000-0000-0000-000000000000",
|
|
54062
|
+
// Nil UUID (often used as placeholder)
|
|
54063
|
+
"admin",
|
|
54064
|
+
// Common admin ID
|
|
54065
|
+
"user1"
|
|
54066
|
+
// Common user ID pattern
|
|
54067
|
+
],
|
|
54068
|
+
paths: [
|
|
54069
|
+
"/tmp/test.txt",
|
|
54070
|
+
// Common temp file path (usually writable)
|
|
54071
|
+
"/home",
|
|
54072
|
+
// Common home directory
|
|
54073
|
+
"./README.md",
|
|
54074
|
+
// Often exists in projects
|
|
54075
|
+
"./package.json",
|
|
54076
|
+
// Common in Node projects
|
|
54077
|
+
"./src",
|
|
54078
|
+
// Common source directory
|
|
54079
|
+
"./test",
|
|
54080
|
+
// Common test directory
|
|
54081
|
+
"./config",
|
|
54082
|
+
// Common config directory
|
|
54083
|
+
"/var/log",
|
|
54084
|
+
// Common log directory (readable)
|
|
54085
|
+
"/etc"
|
|
54086
|
+
// Common config directory (readable)
|
|
54087
|
+
],
|
|
54088
|
+
queries: [
|
|
54089
|
+
"test",
|
|
54090
|
+
// Simple search term
|
|
54091
|
+
"hello",
|
|
54092
|
+
// Common greeting
|
|
54093
|
+
"*",
|
|
54094
|
+
// Wildcard that matches everything
|
|
54095
|
+
"name",
|
|
54096
|
+
// Common field name
|
|
54097
|
+
"id:1",
|
|
54098
|
+
// Common ID search
|
|
54099
|
+
"status:active",
|
|
54100
|
+
// Common status filter
|
|
54101
|
+
"type:user",
|
|
54102
|
+
// Common type filter
|
|
54103
|
+
"limit:10",
|
|
54104
|
+
// Common pagination
|
|
54105
|
+
'{"match_all": {}}'
|
|
54106
|
+
// Elasticsearch match all
|
|
54107
|
+
],
|
|
54108
|
+
numbers: [0, 1, 10, 100, 1e3, 5, 50, 200, 404, 500],
|
|
54109
|
+
booleans: [true, false],
|
|
54110
|
+
jsonObjects: [
|
|
54111
|
+
{ message: "Hello World" },
|
|
54112
|
+
// Simple message object
|
|
54113
|
+
{ status: "ok", code: 200 },
|
|
54114
|
+
// Common status response
|
|
54115
|
+
{ data: [], total: 0 },
|
|
54116
|
+
// Empty result set
|
|
54117
|
+
{ id: 1, name: "Test" },
|
|
54118
|
+
// Simple entity
|
|
54119
|
+
{ success: true },
|
|
54120
|
+
// Common success response
|
|
54121
|
+
{ error: false },
|
|
54122
|
+
// Common no-error response
|
|
54123
|
+
{ results: [] },
|
|
54124
|
+
// Common empty results
|
|
54125
|
+
{}
|
|
54126
|
+
// Empty object (often valid)
|
|
54127
|
+
],
|
|
54128
|
+
arrays: [
|
|
54129
|
+
[],
|
|
54130
|
+
// Empty array (often valid)
|
|
54131
|
+
[1],
|
|
54132
|
+
// Single item
|
|
54133
|
+
["a", "b", "c"],
|
|
54134
|
+
// Simple string array
|
|
54135
|
+
[1, 2, 3],
|
|
54136
|
+
// Simple number array
|
|
54137
|
+
[{ id: 1 }, { id: 2 }],
|
|
54138
|
+
// Simple object array
|
|
54139
|
+
["test"],
|
|
54140
|
+
// Single test item
|
|
54141
|
+
[true, false]
|
|
54142
|
+
// Boolean array
|
|
54143
|
+
],
|
|
54144
|
+
timestamps: [
|
|
54145
|
+
(/* @__PURE__ */ new Date()).toISOString(),
|
|
54146
|
+
// Current time (always valid)
|
|
54147
|
+
new Date(Date.now() - 864e5).toISOString(),
|
|
54148
|
+
// Yesterday
|
|
54149
|
+
new Date(Date.now() + 864e5).toISOString(),
|
|
54150
|
+
// Tomorrow
|
|
54151
|
+
"2024-01-01T00:00:00Z",
|
|
54152
|
+
// New Year 2024
|
|
54153
|
+
"2023-12-31T23:59:59Z",
|
|
54154
|
+
// End of 2023
|
|
54155
|
+
(/* @__PURE__ */ new Date(0)).toISOString(),
|
|
54156
|
+
// Unix epoch
|
|
54157
|
+
"2024-06-15T12:00:00Z"
|
|
54158
|
+
// Midday mid-year
|
|
54159
|
+
]
|
|
54160
|
+
};
|
|
54161
|
+
/**
|
|
54162
|
+
* Generate multiple test scenarios for a tool
|
|
54163
|
+
* Uses Claude Code if available for intelligent generation,
|
|
54164
|
+
* otherwise falls back to schema-based generation.
|
|
54165
|
+
*/
|
|
54166
|
+
static generateTestScenarios(tool) {
|
|
54167
|
+
const scenarios = [];
|
|
54168
|
+
scenarios.push(this.generateHappyPathScenario(tool));
|
|
54169
|
+
const edgeCases = this.generateEdgeCaseScenarios(tool);
|
|
54170
|
+
scenarios.push(...edgeCases);
|
|
54171
|
+
const boundaryScenarios = this.generateBoundaryScenarios(tool);
|
|
54172
|
+
scenarios.push(...boundaryScenarios);
|
|
54173
|
+
scenarios.push(this.generateErrorScenario(tool));
|
|
54174
|
+
return scenarios;
|
|
54175
|
+
}
|
|
54176
|
+
/**
|
|
54177
|
+
* Generate test scenarios with optional Claude enhancement
|
|
54178
|
+
* This async version tries Claude first if enabled, then falls back to schema-based.
|
|
54179
|
+
*/
|
|
54180
|
+
static async generateTestScenariosAsync(tool) {
|
|
54181
|
+
if (this.isClaudeEnabled() && this.claudeBridge) {
|
|
54182
|
+
try {
|
|
54183
|
+
const claudeParams = await this.claudeBridge.generateTestParameters(tool);
|
|
54184
|
+
if (claudeParams && claudeParams.length > 0) {
|
|
54185
|
+
console.log(
|
|
54186
|
+
`[TestDataGenerator] Using Claude-generated params for ${tool.name}`
|
|
54187
|
+
);
|
|
54188
|
+
const claudeScenarios = claudeParams.map(
|
|
54189
|
+
(params, index2) => ({
|
|
54190
|
+
name: this.getClaudeScenarioName(index2),
|
|
54191
|
+
description: `Claude-generated test case ${index2 + 1} for ${tool.name}`,
|
|
54192
|
+
params,
|
|
54193
|
+
expectedBehavior: "Should execute successfully with valid response",
|
|
54194
|
+
category: this.getClaudeScenarioCategory(index2),
|
|
54195
|
+
source: "claude-generated"
|
|
54196
|
+
})
|
|
54197
|
+
);
|
|
54198
|
+
claudeScenarios.push({
|
|
54199
|
+
...this.generateErrorScenario(tool),
|
|
54200
|
+
source: "schema-based"
|
|
54201
|
+
});
|
|
54202
|
+
return claudeScenarios;
|
|
54203
|
+
}
|
|
54204
|
+
} catch (error) {
|
|
54205
|
+
console.warn(
|
|
54206
|
+
`[TestDataGenerator] Claude generation failed for ${tool.name}, falling back to schema-based:`,
|
|
54207
|
+
error
|
|
54208
|
+
);
|
|
54209
|
+
}
|
|
54210
|
+
}
|
|
54211
|
+
return this.generateTestScenarios(tool).map((scenario) => ({
|
|
54212
|
+
...scenario,
|
|
54213
|
+
source: "schema-based"
|
|
54214
|
+
}));
|
|
54215
|
+
}
|
|
54216
|
+
/**
|
|
54217
|
+
* Get scenario name based on index for Claude-generated scenarios
|
|
54218
|
+
*/
|
|
54219
|
+
static getClaudeScenarioName(index2) {
|
|
54220
|
+
const names2 = [
|
|
54221
|
+
"Happy Path - Typical Usage",
|
|
54222
|
+
"Edge Case - Boundary Values",
|
|
54223
|
+
"Minimal Input - Required Fields Only",
|
|
54224
|
+
"Comprehensive - All Fields Populated",
|
|
54225
|
+
"Variant - Alternative Valid Input"
|
|
54226
|
+
];
|
|
54227
|
+
return names2[index2] || `Test Case ${index2 + 1}`;
|
|
54228
|
+
}
|
|
54229
|
+
/**
|
|
54230
|
+
* Get scenario category based on index for Claude-generated scenarios
|
|
54231
|
+
*/
|
|
54232
|
+
static getClaudeScenarioCategory(index2) {
|
|
54233
|
+
const categories = [
|
|
54234
|
+
"happy_path",
|
|
54235
|
+
"edge_case",
|
|
54236
|
+
"boundary",
|
|
54237
|
+
"happy_path",
|
|
54238
|
+
"edge_case"
|
|
54239
|
+
];
|
|
54240
|
+
return categories[index2] || "happy_path";
|
|
54241
|
+
}
|
|
54242
|
+
/**
|
|
54243
|
+
* Generate a happy path scenario with realistic data
|
|
54244
|
+
*/
|
|
54245
|
+
static generateHappyPathScenario(tool) {
|
|
54246
|
+
const params = this.generateRealisticParams(tool, "typical");
|
|
54247
|
+
return {
|
|
54248
|
+
name: "Happy Path - Typical Usage",
|
|
54249
|
+
description: `Test ${tool.name} with typical, valid inputs`,
|
|
54250
|
+
params,
|
|
54251
|
+
expectedBehavior: "Should execute successfully and return valid response",
|
|
54252
|
+
category: "happy_path"
|
|
54253
|
+
};
|
|
54254
|
+
}
|
|
54255
|
+
/**
|
|
54256
|
+
* Generate edge case scenarios
|
|
54257
|
+
*/
|
|
54258
|
+
static generateEdgeCaseScenarios(tool) {
|
|
54259
|
+
const scenarios = [];
|
|
54260
|
+
const emptyParams = this.generateRealisticParams(tool, "empty");
|
|
54261
|
+
if (Object.keys(emptyParams).length > 0) {
|
|
54262
|
+
scenarios.push({
|
|
54263
|
+
name: "Edge Case - Empty Values",
|
|
54264
|
+
description: "Test with empty but valid values",
|
|
54265
|
+
params: emptyParams,
|
|
54266
|
+
expectedBehavior: "Should handle empty values gracefully",
|
|
54267
|
+
category: "edge_case"
|
|
54268
|
+
});
|
|
54269
|
+
}
|
|
54270
|
+
const maxParams = this.generateRealisticParams(tool, "maximum");
|
|
54271
|
+
scenarios.push({
|
|
54272
|
+
name: "Edge Case - Maximum Values",
|
|
54273
|
+
description: "Test with maximum/large values",
|
|
54274
|
+
params: maxParams,
|
|
54275
|
+
expectedBehavior: "Should handle large inputs without issues",
|
|
54276
|
+
category: "edge_case"
|
|
54277
|
+
});
|
|
54278
|
+
if (this.hasStringInputs(tool)) {
|
|
54279
|
+
const specialParams = this.generateRealisticParams(tool, "special");
|
|
54280
|
+
scenarios.push({
|
|
54281
|
+
name: "Edge Case - Special Characters",
|
|
54282
|
+
description: "Test with special characters and unicode",
|
|
54283
|
+
params: specialParams,
|
|
54284
|
+
expectedBehavior: "Should properly handle special characters",
|
|
54285
|
+
category: "edge_case"
|
|
54286
|
+
});
|
|
54287
|
+
}
|
|
54288
|
+
return scenarios;
|
|
54289
|
+
}
|
|
54290
|
+
/**
|
|
54291
|
+
* Generate boundary value scenarios
|
|
54292
|
+
*/
|
|
54293
|
+
static generateBoundaryScenarios(tool) {
|
|
54294
|
+
const scenarios = [];
|
|
54295
|
+
if (!tool.inputSchema || tool.inputSchema.type !== "object") {
|
|
54296
|
+
return scenarios;
|
|
54297
|
+
}
|
|
54298
|
+
const properties2 = tool.inputSchema.properties || {};
|
|
54299
|
+
let hasBoundaries = false;
|
|
54300
|
+
for (const [_key, schema] of Object.entries(properties2)) {
|
|
54301
|
+
const schemaObj = schema;
|
|
54302
|
+
if (schemaObj.minimum !== void 0 || schemaObj.maximum !== void 0 || schemaObj.minLength !== void 0 || schemaObj.maxLength !== void 0) {
|
|
54303
|
+
hasBoundaries = true;
|
|
54304
|
+
break;
|
|
54305
|
+
}
|
|
54306
|
+
}
|
|
54307
|
+
if (!hasBoundaries) {
|
|
54308
|
+
return scenarios;
|
|
54309
|
+
}
|
|
54310
|
+
for (const [key, schema] of Object.entries(properties2)) {
|
|
54311
|
+
const schemaObj = schema;
|
|
54312
|
+
if (schemaObj.type === "number" || schemaObj.type === "integer") {
|
|
54313
|
+
if (schemaObj.minimum !== void 0) {
|
|
54314
|
+
const params = this.generateRealisticParams(tool, "typical");
|
|
54315
|
+
params[key] = schemaObj.minimum;
|
|
54316
|
+
scenarios.push({
|
|
54317
|
+
name: `Boundary - ${key} at minimum`,
|
|
54318
|
+
description: `Test ${key} at its minimum value`,
|
|
54319
|
+
params,
|
|
54320
|
+
expectedBehavior: "Should accept minimum value",
|
|
54321
|
+
category: "boundary"
|
|
54322
|
+
});
|
|
54323
|
+
}
|
|
54324
|
+
if (schemaObj.maximum !== void 0) {
|
|
54325
|
+
const params = this.generateRealisticParams(tool, "typical");
|
|
54326
|
+
params[key] = schemaObj.maximum;
|
|
54327
|
+
scenarios.push({
|
|
54328
|
+
name: `Boundary - ${key} at maximum`,
|
|
54329
|
+
description: `Test ${key} at its maximum value`,
|
|
54330
|
+
params,
|
|
54331
|
+
expectedBehavior: "Should accept maximum value",
|
|
54332
|
+
category: "boundary"
|
|
54333
|
+
});
|
|
54334
|
+
}
|
|
54335
|
+
}
|
|
54336
|
+
if (schemaObj.type === "string") {
|
|
54337
|
+
if (schemaObj.minLength !== void 0) {
|
|
54338
|
+
const params = this.generateRealisticParams(tool, "typical");
|
|
54339
|
+
params[key] = "a".repeat(schemaObj.minLength);
|
|
54340
|
+
scenarios.push({
|
|
54341
|
+
name: `Boundary - ${key} at min length`,
|
|
54342
|
+
description: `Test ${key} at minimum length`,
|
|
54343
|
+
params,
|
|
54344
|
+
expectedBehavior: "Should accept minimum length string",
|
|
54345
|
+
category: "boundary"
|
|
54346
|
+
});
|
|
54347
|
+
}
|
|
54348
|
+
if (schemaObj.maxLength !== void 0) {
|
|
54349
|
+
const params = this.generateRealisticParams(tool, "typical");
|
|
54350
|
+
params[key] = "a".repeat(schemaObj.maxLength);
|
|
54351
|
+
scenarios.push({
|
|
54352
|
+
name: `Boundary - ${key} at max length`,
|
|
54353
|
+
description: `Test ${key} at maximum length`,
|
|
54354
|
+
params,
|
|
54355
|
+
expectedBehavior: "Should accept maximum length string",
|
|
54356
|
+
category: "boundary"
|
|
54357
|
+
});
|
|
54358
|
+
}
|
|
54359
|
+
}
|
|
54360
|
+
}
|
|
54361
|
+
return scenarios;
|
|
54362
|
+
}
|
|
54363
|
+
/**
|
|
54364
|
+
* Generate an error scenario
|
|
54365
|
+
*/
|
|
54366
|
+
static generateErrorScenario(tool) {
|
|
54367
|
+
const params = {};
|
|
54368
|
+
if (tool.inputSchema && tool.inputSchema.type === "object" && tool.inputSchema.properties) {
|
|
54369
|
+
for (const [key, schema] of Object.entries(tool.inputSchema.properties)) {
|
|
54370
|
+
const schemaObj = schema;
|
|
54371
|
+
switch (schemaObj.type) {
|
|
54372
|
+
case "string":
|
|
54373
|
+
params[key] = 123;
|
|
54374
|
+
break;
|
|
54375
|
+
case "number":
|
|
54376
|
+
case "integer":
|
|
54377
|
+
params[key] = "not_a_number";
|
|
54378
|
+
break;
|
|
54379
|
+
case "boolean":
|
|
54380
|
+
params[key] = "not_a_boolean";
|
|
54381
|
+
break;
|
|
54382
|
+
case "array":
|
|
54383
|
+
params[key] = "not_an_array";
|
|
54384
|
+
break;
|
|
54385
|
+
case "object":
|
|
54386
|
+
params[key] = "not_an_object";
|
|
54387
|
+
break;
|
|
54388
|
+
default:
|
|
54389
|
+
params[key] = null;
|
|
54390
|
+
}
|
|
54391
|
+
break;
|
|
54392
|
+
}
|
|
54393
|
+
}
|
|
54394
|
+
return {
|
|
54395
|
+
name: "Error Case - Invalid Type",
|
|
54396
|
+
description: "Test error handling with invalid parameter types",
|
|
54397
|
+
params,
|
|
54398
|
+
expectedBehavior: "Should return clear error about invalid parameter type",
|
|
54399
|
+
category: "error_case"
|
|
54400
|
+
};
|
|
54401
|
+
}
|
|
54402
|
+
/**
|
|
54403
|
+
* Generate realistic parameters based on schema and variant
|
|
54404
|
+
*/
|
|
54405
|
+
static generateRealisticParams(tool, variant) {
|
|
54406
|
+
const params = {};
|
|
54407
|
+
if (!tool.inputSchema || tool.inputSchema.type !== "object") {
|
|
54408
|
+
return params;
|
|
54409
|
+
}
|
|
54410
|
+
const properties2 = tool.inputSchema.properties || {};
|
|
54411
|
+
for (const [key, schema] of Object.entries(properties2)) {
|
|
54412
|
+
params[key] = this.generateRealisticValue(key, schema, variant);
|
|
54413
|
+
}
|
|
54414
|
+
return params;
|
|
54415
|
+
}
|
|
54416
|
+
/**
|
|
54417
|
+
* Generate a realistic value based on field name and schema
|
|
54418
|
+
*/
|
|
54419
|
+
static generateRealisticValue(fieldName, schema, variant) {
|
|
54420
|
+
const lowerFieldName = fieldName.toLowerCase();
|
|
54421
|
+
switch (schema.type) {
|
|
54422
|
+
case "string":
|
|
54423
|
+
if (schema.enum && schema.enum.length > 0) {
|
|
54424
|
+
return variant === "typical" ? schema.enum[0] : schema.enum[schema.enum.length - 1];
|
|
54425
|
+
}
|
|
54426
|
+
if (lowerFieldName.includes("url") || lowerFieldName.includes("link") || lowerFieldName.includes("endpoint")) {
|
|
54427
|
+
return variant === "empty" ? "" : variant === "maximum" ? "https://very-long-domain-name-for-testing-maximum-length.example.com/path/to/resource?param1=value1¶m2=value2" : variant === "special" ? "https://example.com/path?special=!@#$%^&*()" : this.REALISTIC_DATA.urls[Math.floor(Math.random() * this.REALISTIC_DATA.urls.length)];
|
|
54428
|
+
}
|
|
54429
|
+
if (lowerFieldName.includes("email") || lowerFieldName.includes("mail")) {
|
|
54430
|
+
return variant === "empty" ? "" : variant === "maximum" ? "very.long.email.address.for.testing@subdomain.example-company.co.uk" : variant === "special" ? "user+tag@example.com" : this.REALISTIC_DATA.emails[Math.floor(
|
|
54431
|
+
Math.random() * this.REALISTIC_DATA.emails.length
|
|
54432
|
+
)];
|
|
54433
|
+
}
|
|
54434
|
+
if (lowerFieldName.includes("path") || lowerFieldName.includes("file") || lowerFieldName.includes("directory") || lowerFieldName.includes("folder")) {
|
|
54435
|
+
return variant === "empty" ? "" : variant === "maximum" ? "/very/long/path/to/deeply/nested/directory/structure/for/testing/file.txt" : variant === "special" ? "./path/with spaces/and-special#chars.txt" : this.REALISTIC_DATA.paths[Math.floor(Math.random() * this.REALISTIC_DATA.paths.length)];
|
|
54436
|
+
}
|
|
54437
|
+
if (lowerFieldName.includes("query") || lowerFieldName.includes("search") || lowerFieldName.includes("filter")) {
|
|
54438
|
+
return variant === "empty" ? "test" : variant === "maximum" ? "very long search query with many terms for testing maximum input length handling" : variant === "special" ? 'search with "quotes" and special: characters!' : this.REALISTIC_DATA.queries[Math.floor(
|
|
54439
|
+
Math.random() * this.REALISTIC_DATA.queries.length
|
|
54440
|
+
)];
|
|
54441
|
+
}
|
|
54442
|
+
if (lowerFieldName.includes("id") || lowerFieldName.includes("key") || lowerFieldName.includes("identifier")) {
|
|
54443
|
+
const requiresUuid = lowerFieldName.includes("uuid") || lowerFieldName.includes("page_id") || lowerFieldName.includes("database_id") || lowerFieldName.includes("user_id") || lowerFieldName.includes("block_id") || lowerFieldName.includes("comment_id") || lowerFieldName.includes("workspace_id") || lowerFieldName.includes("notion") || // Check schema description for UUID hints
|
|
54444
|
+
schema.description && (schema.description.toLowerCase().includes("uuid") || schema.description.toLowerCase().includes("universally unique"));
|
|
54445
|
+
if (requiresUuid) {
|
|
54446
|
+
return variant === "empty" ? "00000000-0000-0000-0000-000000000000" : "550e8400-e29b-41d4-a716-446655440000";
|
|
54447
|
+
}
|
|
54448
|
+
return variant === "empty" ? "1" : variant === "maximum" ? "very_long_identifier_string_for_testing_maximum_length_handling_in_system" : this.REALISTIC_DATA.ids[Math.floor(Math.random() * this.REALISTIC_DATA.ids.length)];
|
|
54449
|
+
}
|
|
54450
|
+
if (lowerFieldName.includes("name") || lowerFieldName.includes("title") || lowerFieldName.includes("label")) {
|
|
54451
|
+
return variant === "empty" ? "a" : variant === "maximum" ? "Very Long Name For Testing Maximum String Length Handling In The System" : variant === "special" ? "Name with Special™ Characters® and Émojis 🎉" : this.REALISTIC_DATA.names[Math.floor(Math.random() * this.REALISTIC_DATA.names.length)];
|
|
54452
|
+
}
|
|
54453
|
+
if (lowerFieldName.includes("date") || lowerFieldName.includes("time")) {
|
|
54454
|
+
return variant === "empty" ? "" : this.REALISTIC_DATA.timestamps[0];
|
|
54455
|
+
}
|
|
54456
|
+
return variant === "empty" ? "" : variant === "maximum" ? "x".repeat(100) : variant === "special" ? 'Special chars: !@#$%^&*()_+-=[]{}|;:",.<>?/~`' : "test";
|
|
54457
|
+
// Simple, generic test value that often works
|
|
54458
|
+
case "number":
|
|
54459
|
+
case "integer":
|
|
54460
|
+
if (variant === "maximum") {
|
|
54461
|
+
return schema.maximum || 999999;
|
|
54462
|
+
}
|
|
54463
|
+
if (variant === "empty") {
|
|
54464
|
+
return schema.minimum || 0;
|
|
54465
|
+
}
|
|
54466
|
+
if (lowerFieldName.includes("port")) {
|
|
54467
|
+
return 8080;
|
|
54468
|
+
}
|
|
54469
|
+
if (lowerFieldName.includes("timeout") || lowerFieldName.includes("delay")) {
|
|
54470
|
+
return 5e3;
|
|
54471
|
+
}
|
|
54472
|
+
if (lowerFieldName.includes("count") || lowerFieldName.includes("limit")) {
|
|
54473
|
+
return 10;
|
|
54474
|
+
}
|
|
54475
|
+
if (lowerFieldName.includes("page") || lowerFieldName.includes("offset")) {
|
|
54476
|
+
return 0;
|
|
54477
|
+
}
|
|
54478
|
+
if (lowerFieldName.includes("size") || lowerFieldName.includes("length")) {
|
|
54479
|
+
return 100;
|
|
54480
|
+
}
|
|
54481
|
+
return schema.minimum || 1;
|
|
54482
|
+
case "boolean":
|
|
54483
|
+
return variant === "empty" ? false : true;
|
|
54484
|
+
case "array":
|
|
54485
|
+
if (variant === "empty") {
|
|
54486
|
+
const isMutationField = lowerFieldName.includes("entities") || lowerFieldName.includes("relations") || lowerFieldName.includes("observations") || lowerFieldName.includes("documents");
|
|
54487
|
+
if (isMutationField && schema.items) {
|
|
54488
|
+
const item = this.generateValueFromSchema(schema.items, "empty");
|
|
54489
|
+
return [item];
|
|
54490
|
+
}
|
|
54491
|
+
return [];
|
|
54492
|
+
}
|
|
54493
|
+
if (variant === "maximum") {
|
|
54494
|
+
const count2 = 10;
|
|
54495
|
+
if (schema.items) {
|
|
54496
|
+
return Array(count2).fill(0).map(() => this.generateValueFromSchema(schema.items, variant));
|
|
54497
|
+
}
|
|
54498
|
+
return Array(count2).fill(0).map((_, i) => `item_${i}`);
|
|
54499
|
+
}
|
|
54500
|
+
if (schema.items) {
|
|
54501
|
+
const item = this.generateValueFromSchema(schema.items, variant);
|
|
54502
|
+
return [item];
|
|
54503
|
+
}
|
|
54504
|
+
if (lowerFieldName.includes("tag") || lowerFieldName.includes("label")) {
|
|
54505
|
+
return ["tag1", "tag2", "tag3"];
|
|
54506
|
+
}
|
|
54507
|
+
if (lowerFieldName.includes("id")) {
|
|
54508
|
+
return ["id_1", "id_2", "id_3"];
|
|
54509
|
+
}
|
|
54510
|
+
return this.REALISTIC_DATA.arrays[1];
|
|
54511
|
+
case "object":
|
|
54512
|
+
if (variant === "maximum") {
|
|
54513
|
+
return this.REALISTIC_DATA.jsonObjects[4];
|
|
54514
|
+
}
|
|
54515
|
+
if (lowerFieldName.includes("config") || lowerFieldName.includes("settings")) {
|
|
54516
|
+
return variant === "empty" ? { enabled: false } : { enabled: true, timeout: 5e3, retries: 3 };
|
|
54517
|
+
}
|
|
54518
|
+
if (lowerFieldName.includes("metadata") || lowerFieldName.includes("meta")) {
|
|
54519
|
+
return variant === "empty" ? { version: "1.0.0" } : {
|
|
54520
|
+
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
54521
|
+
version: "1.0.0",
|
|
54522
|
+
author: "test"
|
|
54523
|
+
};
|
|
54524
|
+
}
|
|
54525
|
+
if (lowerFieldName.includes("filter") || lowerFieldName.includes("query")) {
|
|
54526
|
+
return variant === "empty" ? { limit: 1 } : { status: "active", type: "user", limit: 10 };
|
|
54527
|
+
}
|
|
54528
|
+
return variant === "empty" ? { id: 1 } : this.REALISTIC_DATA.jsonObjects[0];
|
|
54529
|
+
default:
|
|
54530
|
+
return "test";
|
|
54531
|
+
}
|
|
54532
|
+
}
|
|
54533
|
+
/**
|
|
54534
|
+
* Check if tool has string inputs
|
|
54535
|
+
*/
|
|
54536
|
+
static hasStringInputs(tool) {
|
|
54537
|
+
if (!tool.inputSchema || tool.inputSchema.type !== "object") {
|
|
54538
|
+
return false;
|
|
54539
|
+
}
|
|
54540
|
+
const properties2 = tool.inputSchema.properties || {};
|
|
54541
|
+
for (const schema of Object.values(properties2)) {
|
|
54542
|
+
if (schema.type === "string") {
|
|
54543
|
+
return true;
|
|
54544
|
+
}
|
|
54545
|
+
}
|
|
54546
|
+
return false;
|
|
54547
|
+
}
|
|
54548
|
+
/**
|
|
54549
|
+
* Generate a single realistic value for backward compatibility
|
|
54550
|
+
*/
|
|
54551
|
+
static generateSingleValue(fieldName, schema) {
|
|
54552
|
+
return this.generateRealisticValue(fieldName, schema, "typical");
|
|
54553
|
+
}
|
|
54554
|
+
/**
|
|
54555
|
+
* Generate value from JSON schema definition
|
|
54556
|
+
*/
|
|
54557
|
+
static generateValueFromSchema(schema, variant) {
|
|
54558
|
+
if (!schema || !schema.type) {
|
|
54559
|
+
return "test";
|
|
54560
|
+
}
|
|
54561
|
+
switch (schema.type) {
|
|
54562
|
+
case "object": {
|
|
54563
|
+
const obj = {};
|
|
54564
|
+
if (schema.properties) {
|
|
54565
|
+
for (const [key, propSchema] of Object.entries(schema.properties)) {
|
|
54566
|
+
obj[key] = this.generateRealisticValue(
|
|
54567
|
+
key,
|
|
54568
|
+
propSchema,
|
|
54569
|
+
variant
|
|
54570
|
+
);
|
|
54571
|
+
}
|
|
54572
|
+
}
|
|
54573
|
+
return obj;
|
|
54574
|
+
}
|
|
54575
|
+
case "array":
|
|
54576
|
+
if (schema.items) {
|
|
54577
|
+
const item = this.generateValueFromSchema(schema.items, variant);
|
|
54578
|
+
return [item];
|
|
54579
|
+
}
|
|
54580
|
+
return [];
|
|
54581
|
+
case "string":
|
|
54582
|
+
return variant === "empty" ? "" : "test";
|
|
54583
|
+
case "number":
|
|
54584
|
+
case "integer":
|
|
54585
|
+
return variant === "empty" ? 0 : 1;
|
|
54586
|
+
case "boolean":
|
|
54587
|
+
return variant === "empty" ? false : true;
|
|
54588
|
+
default:
|
|
54589
|
+
return "test";
|
|
54590
|
+
}
|
|
54591
|
+
}
|
|
54592
|
+
// ============================================================================
|
|
54593
|
+
// Tool Category-Aware Generation
|
|
54594
|
+
// ============================================================================
|
|
54595
|
+
/**
|
|
54596
|
+
* Tool category-specific data pools for when field name doesn't help identify
|
|
54597
|
+
* the expected input type. Used as fallback after field-name detection.
|
|
54598
|
+
*/
|
|
54599
|
+
static TOOL_CATEGORY_DATA = {
|
|
54600
|
+
// Keys must match ToolClassifier category names (lowercase)
|
|
54601
|
+
calculator: {
|
|
54602
|
+
default: ["2+2", "10*5", "100/4", "sqrt(16)", "15-7"]
|
|
54603
|
+
},
|
|
54604
|
+
search_retrieval: {
|
|
54605
|
+
default: [
|
|
54606
|
+
"hello world",
|
|
54607
|
+
"example query",
|
|
54608
|
+
"recent changes",
|
|
54609
|
+
"find documents"
|
|
54610
|
+
]
|
|
54611
|
+
},
|
|
54612
|
+
system_exec: {
|
|
54613
|
+
default: ["echo hello", "pwd", "date", "whoami"]
|
|
54614
|
+
},
|
|
54615
|
+
url_fetcher: {
|
|
54616
|
+
default: [
|
|
54617
|
+
"https://api.github.com",
|
|
54618
|
+
"https://httpbin.org/get",
|
|
54619
|
+
"https://jsonplaceholder.typicode.com/posts/1"
|
|
54620
|
+
]
|
|
54621
|
+
}
|
|
54622
|
+
};
|
|
54623
|
+
/**
|
|
54624
|
+
* Field names that indicate specific data types regardless of tool category.
|
|
54625
|
+
* These take precedence over category-specific generation.
|
|
54626
|
+
*/
|
|
54627
|
+
static SPECIFIC_FIELD_PATTERNS = [
|
|
54628
|
+
/url/i,
|
|
54629
|
+
/endpoint/i,
|
|
54630
|
+
/link/i,
|
|
54631
|
+
/email/i,
|
|
54632
|
+
/mail/i,
|
|
54633
|
+
/path/i,
|
|
54634
|
+
/file/i,
|
|
54635
|
+
/directory/i,
|
|
54636
|
+
/folder/i,
|
|
54637
|
+
/uuid/i,
|
|
54638
|
+
/page_id/i,
|
|
54639
|
+
/database_id/i,
|
|
54640
|
+
/user_id/i,
|
|
54641
|
+
/block_id/i
|
|
54642
|
+
];
|
|
54643
|
+
/**
|
|
54644
|
+
* Generate a value using tool category as hint.
|
|
54645
|
+
* For specific field names (url, email, path, etc.), uses field-name detection.
|
|
54646
|
+
* For generic field names with specific tool categories, uses category-specific inputs.
|
|
54647
|
+
* Otherwise falls back to field-name-based generation.
|
|
54648
|
+
*
|
|
54649
|
+
* @param fieldName The parameter field name
|
|
54650
|
+
* @param schema The parameter schema
|
|
54651
|
+
* @param category The tool category from ToolClassifier (e.g., "CALCULATOR", "SEARCH_RETRIEVAL")
|
|
54652
|
+
* @returns Generated test value appropriate for the tool type
|
|
54653
|
+
*/
|
|
54654
|
+
static generateValueForCategory(fieldName, schema, category) {
|
|
54655
|
+
const isSpecificFieldName = this.SPECIFIC_FIELD_PATTERNS.some(
|
|
54656
|
+
(pattern2) => pattern2.test(fieldName)
|
|
54657
|
+
);
|
|
54658
|
+
if (isSpecificFieldName) {
|
|
54659
|
+
return this.generateSingleValue(fieldName, schema);
|
|
54660
|
+
}
|
|
54661
|
+
const categoryData = this.TOOL_CATEGORY_DATA[category];
|
|
54662
|
+
if (categoryData?.default) {
|
|
54663
|
+
return categoryData.default[0];
|
|
54664
|
+
}
|
|
54665
|
+
return this.generateSingleValue(fieldName, schema);
|
|
54666
|
+
}
|
|
54667
|
+
}
|
|
53611
54668
|
class FunctionalityAssessor extends BaseAssessor {
|
|
54669
|
+
toolClassifier = new ToolClassifier();
|
|
53612
54670
|
/**
|
|
53613
54671
|
* Select tools for testing based on configuration
|
|
53614
54672
|
*/
|
|
@@ -53729,8 +54787,8 @@ class FunctionalityAssessor extends BaseAssessor {
|
|
|
53729
54787
|
}
|
|
53730
54788
|
async testTool(tool, callTool) {
|
|
53731
54789
|
const startTime = Date.now();
|
|
54790
|
+
const { params: testParams, metadata: metadata2 } = this.generateMinimalParams(tool);
|
|
53732
54791
|
try {
|
|
53733
|
-
const testParams = this.generateMinimalParams(tool);
|
|
53734
54792
|
this.log(
|
|
53735
54793
|
`Testing tool: ${tool.name} with params: ${JSON.stringify(testParams)}`
|
|
53736
54794
|
);
|
|
@@ -53752,7 +54810,8 @@ class FunctionalityAssessor extends BaseAssessor {
|
|
|
53752
54810
|
status: "working",
|
|
53753
54811
|
executionTime,
|
|
53754
54812
|
testParameters: testParams,
|
|
53755
|
-
response
|
|
54813
|
+
response,
|
|
54814
|
+
testInputMetadata: metadata2
|
|
53756
54815
|
};
|
|
53757
54816
|
}
|
|
53758
54817
|
return {
|
|
@@ -53762,7 +54821,8 @@ class FunctionalityAssessor extends BaseAssessor {
|
|
|
53762
54821
|
error: this.extractErrorMessage(response),
|
|
53763
54822
|
executionTime,
|
|
53764
54823
|
testParameters: testParams,
|
|
53765
|
-
response
|
|
54824
|
+
response,
|
|
54825
|
+
testInputMetadata: metadata2
|
|
53766
54826
|
};
|
|
53767
54827
|
}
|
|
53768
54828
|
return {
|
|
@@ -53771,7 +54831,8 @@ class FunctionalityAssessor extends BaseAssessor {
|
|
|
53771
54831
|
status: "working",
|
|
53772
54832
|
executionTime,
|
|
53773
54833
|
testParameters: testParams,
|
|
53774
|
-
response
|
|
54834
|
+
response,
|
|
54835
|
+
testInputMetadata: metadata2
|
|
53775
54836
|
};
|
|
53776
54837
|
} catch (error) {
|
|
53777
54838
|
return {
|
|
@@ -53779,24 +54840,48 @@ class FunctionalityAssessor extends BaseAssessor {
|
|
|
53779
54840
|
tested: true,
|
|
53780
54841
|
status: "broken",
|
|
53781
54842
|
error: this.extractErrorMessage(error),
|
|
53782
|
-
executionTime: Date.now() - startTime
|
|
54843
|
+
executionTime: Date.now() - startTime,
|
|
54844
|
+
testInputMetadata: metadata2
|
|
53783
54845
|
};
|
|
53784
54846
|
}
|
|
53785
54847
|
}
|
|
53786
54848
|
generateMinimalParams(tool) {
|
|
53787
|
-
|
|
54849
|
+
const classification = this.toolClassifier.classify(
|
|
54850
|
+
tool.name,
|
|
54851
|
+
tool.description || ""
|
|
54852
|
+
);
|
|
54853
|
+
const primaryCategory = classification.categories[0] || ToolCategory.GENERIC;
|
|
54854
|
+
const emptyResult = {
|
|
54855
|
+
params: {},
|
|
54856
|
+
metadata: {
|
|
54857
|
+
toolCategory: primaryCategory,
|
|
54858
|
+
generationStrategy: "default",
|
|
54859
|
+
fieldSources: {}
|
|
54860
|
+
}
|
|
54861
|
+
};
|
|
54862
|
+
if (!tool.inputSchema) return emptyResult;
|
|
53788
54863
|
const schema = typeof tool.inputSchema === "string" ? this.safeJsonParse(tool.inputSchema) : tool.inputSchema;
|
|
53789
|
-
if (!schema?.properties) return
|
|
54864
|
+
if (!schema?.properties) return emptyResult;
|
|
53790
54865
|
const params = {};
|
|
54866
|
+
const fieldSources = {};
|
|
53791
54867
|
const required2 = schema.required || [];
|
|
53792
54868
|
for (const [key, prop] of Object.entries(
|
|
53793
54869
|
schema.properties
|
|
53794
54870
|
)) {
|
|
53795
54871
|
if (required2.includes(key)) {
|
|
53796
|
-
|
|
54872
|
+
const { value, source, reason } = this.generateSmartParamValueWithMetadata(prop, key, primaryCategory);
|
|
54873
|
+
params[key] = value;
|
|
54874
|
+
fieldSources[key] = { field: key, value, source, reason };
|
|
53797
54875
|
}
|
|
53798
54876
|
}
|
|
53799
|
-
return
|
|
54877
|
+
return {
|
|
54878
|
+
params,
|
|
54879
|
+
metadata: {
|
|
54880
|
+
toolCategory: primaryCategory,
|
|
54881
|
+
generationStrategy: this.determineStrategy(fieldSources),
|
|
54882
|
+
fieldSources
|
|
54883
|
+
}
|
|
54884
|
+
};
|
|
53800
54885
|
}
|
|
53801
54886
|
generateParamValue(prop, fieldName, includeOptional = false) {
|
|
53802
54887
|
const type2 = prop.type;
|
|
@@ -53853,6 +54938,102 @@ class FunctionalityAssessor extends BaseAssessor {
|
|
|
53853
54938
|
return {};
|
|
53854
54939
|
}
|
|
53855
54940
|
}
|
|
54941
|
+
/**
|
|
54942
|
+
* Field names that indicate specific data types regardless of tool category.
|
|
54943
|
+
* These take precedence over category-specific generation.
|
|
54944
|
+
*/
|
|
54945
|
+
static SPECIFIC_FIELD_PATTERNS = [
|
|
54946
|
+
/url/i,
|
|
54947
|
+
/endpoint/i,
|
|
54948
|
+
/link/i,
|
|
54949
|
+
/email/i,
|
|
54950
|
+
/mail/i,
|
|
54951
|
+
/path/i,
|
|
54952
|
+
/file/i,
|
|
54953
|
+
/directory/i,
|
|
54954
|
+
/folder/i,
|
|
54955
|
+
/uuid/i,
|
|
54956
|
+
/page_id/i,
|
|
54957
|
+
/database_id/i,
|
|
54958
|
+
/user_id/i,
|
|
54959
|
+
/block_id/i
|
|
54960
|
+
];
|
|
54961
|
+
/**
|
|
54962
|
+
* Generate smart parameter value with metadata about how it was generated.
|
|
54963
|
+
* Returns value, source type, and reason for downstream consumers.
|
|
54964
|
+
*/
|
|
54965
|
+
generateSmartParamValueWithMetadata(prop, fieldName, category) {
|
|
54966
|
+
if (prop.enum && prop.enum.length > 0) {
|
|
54967
|
+
return {
|
|
54968
|
+
value: prop.enum[0],
|
|
54969
|
+
source: "enum",
|
|
54970
|
+
reason: `First enum value: ${prop.enum[0]}`
|
|
54971
|
+
};
|
|
54972
|
+
}
|
|
54973
|
+
if (prop.format === "uri") {
|
|
54974
|
+
return {
|
|
54975
|
+
value: "https://example.com",
|
|
54976
|
+
source: "format",
|
|
54977
|
+
reason: "URI format detected"
|
|
54978
|
+
};
|
|
54979
|
+
}
|
|
54980
|
+
if (prop.format === "email") {
|
|
54981
|
+
return {
|
|
54982
|
+
value: "test@example.com",
|
|
54983
|
+
source: "format",
|
|
54984
|
+
reason: "Email format detected"
|
|
54985
|
+
};
|
|
54986
|
+
}
|
|
54987
|
+
if (prop.type !== "string") {
|
|
54988
|
+
const value = this.generateParamValue(prop, fieldName);
|
|
54989
|
+
return {
|
|
54990
|
+
value,
|
|
54991
|
+
source: "default",
|
|
54992
|
+
reason: `Default for type: ${prop.type}`
|
|
54993
|
+
};
|
|
54994
|
+
}
|
|
54995
|
+
const isSpecificFieldName = FunctionalityAssessor.SPECIFIC_FIELD_PATTERNS.some(
|
|
54996
|
+
(pattern2) => pattern2.test(fieldName)
|
|
54997
|
+
);
|
|
54998
|
+
if (isSpecificFieldName) {
|
|
54999
|
+
const fieldValue2 = TestDataGenerator.generateSingleValue(fieldName, prop);
|
|
55000
|
+
return {
|
|
55001
|
+
value: fieldValue2,
|
|
55002
|
+
source: "field-name",
|
|
55003
|
+
reason: `Field name pattern: ${fieldName}`
|
|
55004
|
+
};
|
|
55005
|
+
}
|
|
55006
|
+
const categoryData = TestDataGenerator.TOOL_CATEGORY_DATA[category];
|
|
55007
|
+
if (categoryData?.default) {
|
|
55008
|
+
return {
|
|
55009
|
+
value: categoryData.default[0],
|
|
55010
|
+
source: "category",
|
|
55011
|
+
reason: `Category ${category} default value`
|
|
55012
|
+
};
|
|
55013
|
+
}
|
|
55014
|
+
const fieldValue = TestDataGenerator.generateSingleValue(fieldName, prop);
|
|
55015
|
+
if (fieldValue !== "test") {
|
|
55016
|
+
return {
|
|
55017
|
+
value: fieldValue,
|
|
55018
|
+
source: "field-name",
|
|
55019
|
+
reason: `Field name pattern: ${fieldName}`
|
|
55020
|
+
};
|
|
55021
|
+
}
|
|
55022
|
+
return {
|
|
55023
|
+
value: "test",
|
|
55024
|
+
source: "default",
|
|
55025
|
+
reason: "No specific pattern matched"
|
|
55026
|
+
};
|
|
55027
|
+
}
|
|
55028
|
+
/**
|
|
55029
|
+
* Determine overall generation strategy based on field sources
|
|
55030
|
+
*/
|
|
55031
|
+
determineStrategy(fieldSources) {
|
|
55032
|
+
const sources = Object.values(fieldSources).map((f) => f.source);
|
|
55033
|
+
if (sources.includes("category")) return "category-specific";
|
|
55034
|
+
if (sources.includes("field-name")) return "field-name-aware";
|
|
55035
|
+
return "default";
|
|
55036
|
+
}
|
|
53856
55037
|
// Public method for testing purposes - allows tests to verify parameter generation logic
|
|
53857
55038
|
// Always includes optional properties to test full schema
|
|
53858
55039
|
generateTestInput(schema) {
|
|
@@ -54269,377 +55450,6 @@ function getPayloadsForAttack(attackName, limit2) {
|
|
|
54269
55450
|
function getAllAttackPatterns() {
|
|
54270
55451
|
return SECURITY_ATTACK_PATTERNS;
|
|
54271
55452
|
}
|
|
54272
|
-
var ToolCategory = /* @__PURE__ */ ((ToolCategory2) => {
|
|
54273
|
-
ToolCategory2["CALCULATOR"] = "calculator";
|
|
54274
|
-
ToolCategory2["SYSTEM_EXEC"] = "system_exec";
|
|
54275
|
-
ToolCategory2["DATA_ACCESS"] = "data_access";
|
|
54276
|
-
ToolCategory2["TOOL_OVERRIDE"] = "tool_override";
|
|
54277
|
-
ToolCategory2["CONFIG_MODIFIER"] = "config_modifier";
|
|
54278
|
-
ToolCategory2["URL_FETCHER"] = "fetcher";
|
|
54279
|
-
ToolCategory2["UNICODE_PROCESSOR"] = "unicode";
|
|
54280
|
-
ToolCategory2["JSON_PARSER"] = "parser";
|
|
54281
|
-
ToolCategory2["PACKAGE_INSTALLER"] = "installer";
|
|
54282
|
-
ToolCategory2["RUG_PULL"] = "rug_pull";
|
|
54283
|
-
ToolCategory2["SAFE_STORAGE"] = "safe_storage";
|
|
54284
|
-
ToolCategory2["API_WRAPPER"] = "api_wrapper";
|
|
54285
|
-
ToolCategory2["SEARCH_RETRIEVAL"] = "search_retrieval";
|
|
54286
|
-
ToolCategory2["CRUD_CREATION"] = "crud_creation";
|
|
54287
|
-
ToolCategory2["READ_ONLY_INFO"] = "read_only_info";
|
|
54288
|
-
ToolCategory2["GENERIC"] = "generic";
|
|
54289
|
-
return ToolCategory2;
|
|
54290
|
-
})(ToolCategory || {});
|
|
54291
|
-
class ToolClassifier {
|
|
54292
|
-
/**
|
|
54293
|
-
* Classify a tool into one or more categories
|
|
54294
|
-
* Returns multiple categories if tool matches multiple patterns
|
|
54295
|
-
*/
|
|
54296
|
-
classify(toolName, description2) {
|
|
54297
|
-
const categories = [];
|
|
54298
|
-
const confidenceScores = [];
|
|
54299
|
-
const reasons = [];
|
|
54300
|
-
const toolText = `${toolName} ${description2 || ""}`.toLowerCase();
|
|
54301
|
-
if (this.matchesPattern(toolText, [
|
|
54302
|
-
/calculator/i,
|
|
54303
|
-
/compute/i,
|
|
54304
|
-
/math/i,
|
|
54305
|
-
/calc/i,
|
|
54306
|
-
/eval/i,
|
|
54307
|
-
/arithmetic/i,
|
|
54308
|
-
/expression/i
|
|
54309
|
-
])) {
|
|
54310
|
-
categories.push(
|
|
54311
|
-
"calculator"
|
|
54312
|
-
/* CALCULATOR */
|
|
54313
|
-
);
|
|
54314
|
-
confidenceScores.push(90);
|
|
54315
|
-
reasons.push("Calculator pattern detected (arithmetic execution risk)");
|
|
54316
|
-
}
|
|
54317
|
-
if (this.matchesPattern(toolText, [
|
|
54318
|
-
/system.*exec/i,
|
|
54319
|
-
/exec.*tool/i,
|
|
54320
|
-
/command/i,
|
|
54321
|
-
/shell/i,
|
|
54322
|
-
/\brun\b/i,
|
|
54323
|
-
/execute/i,
|
|
54324
|
-
/process/i
|
|
54325
|
-
])) {
|
|
54326
|
-
categories.push(
|
|
54327
|
-
"system_exec"
|
|
54328
|
-
/* SYSTEM_EXEC */
|
|
54329
|
-
);
|
|
54330
|
-
confidenceScores.push(95);
|
|
54331
|
-
reasons.push(
|
|
54332
|
-
"System execution pattern detected (command injection risk)"
|
|
54333
|
-
);
|
|
54334
|
-
}
|
|
54335
|
-
if (this.matchesPattern(toolText, [
|
|
54336
|
-
/leak/i,
|
|
54337
|
-
/\bdata\b/i,
|
|
54338
|
-
/show/i,
|
|
54339
|
-
/\bget\b/i,
|
|
54340
|
-
/\blist\b/i,
|
|
54341
|
-
/display/i,
|
|
54342
|
-
/\benv/i,
|
|
54343
|
-
/secret/i,
|
|
54344
|
-
/\bkey\b/i,
|
|
54345
|
-
/credential/i,
|
|
54346
|
-
/exfiltrat/i
|
|
54347
|
-
])) {
|
|
54348
|
-
categories.push(
|
|
54349
|
-
"data_access"
|
|
54350
|
-
/* DATA_ACCESS */
|
|
54351
|
-
);
|
|
54352
|
-
confidenceScores.push(85);
|
|
54353
|
-
reasons.push("Data access pattern detected (data exfiltration risk)");
|
|
54354
|
-
}
|
|
54355
|
-
if (this.matchesPattern(toolText, [
|
|
54356
|
-
/override/i,
|
|
54357
|
-
/shadow/i,
|
|
54358
|
-
/poison/i,
|
|
54359
|
-
/create.*tool/i,
|
|
54360
|
-
/register.*tool/i,
|
|
54361
|
-
/define.*tool/i,
|
|
54362
|
-
/tool.*creator/i,
|
|
54363
|
-
/add.*tool/i
|
|
54364
|
-
])) {
|
|
54365
|
-
categories.push(
|
|
54366
|
-
"tool_override"
|
|
54367
|
-
/* TOOL_OVERRIDE */
|
|
54368
|
-
);
|
|
54369
|
-
confidenceScores.push(92);
|
|
54370
|
-
reasons.push("Tool override pattern detected (shadowing/poisoning risk)");
|
|
54371
|
-
}
|
|
54372
|
-
if (this.matchesPattern(toolText, [
|
|
54373
|
-
/config/i,
|
|
54374
|
-
/setting/i,
|
|
54375
|
-
/modifier/i,
|
|
54376
|
-
/\badmin\b/i,
|
|
54377
|
-
/privilege/i,
|
|
54378
|
-
/permission/i,
|
|
54379
|
-
/configure/i,
|
|
54380
|
-
/drift/i
|
|
54381
|
-
])) {
|
|
54382
|
-
categories.push(
|
|
54383
|
-
"config_modifier"
|
|
54384
|
-
/* CONFIG_MODIFIER */
|
|
54385
|
-
);
|
|
54386
|
-
confidenceScores.push(88);
|
|
54387
|
-
reasons.push(
|
|
54388
|
-
"Config modification pattern detected (configuration drift risk)"
|
|
54389
|
-
);
|
|
54390
|
-
}
|
|
54391
|
-
if (this.matchesPattern(toolText, [
|
|
54392
|
-
/fetch/i,
|
|
54393
|
-
/\burl\b/i,
|
|
54394
|
-
/http/i,
|
|
54395
|
-
/download/i,
|
|
54396
|
-
/load/i,
|
|
54397
|
-
/retrieve/i,
|
|
54398
|
-
/\bget\b.*url/i,
|
|
54399
|
-
/external/i
|
|
54400
|
-
])) {
|
|
54401
|
-
categories.push(
|
|
54402
|
-
"fetcher"
|
|
54403
|
-
/* URL_FETCHER */
|
|
54404
|
-
);
|
|
54405
|
-
confidenceScores.push(87);
|
|
54406
|
-
reasons.push(
|
|
54407
|
-
"URL fetcher pattern detected (indirect prompt injection risk)"
|
|
54408
|
-
);
|
|
54409
|
-
}
|
|
54410
|
-
if (this.matchesPattern(toolText, [
|
|
54411
|
-
/unicode/i,
|
|
54412
|
-
/encode/i,
|
|
54413
|
-
/decode/i,
|
|
54414
|
-
/charset/i,
|
|
54415
|
-
/utf/i,
|
|
54416
|
-
/hex/i,
|
|
54417
|
-
/escape/i
|
|
54418
|
-
])) {
|
|
54419
|
-
categories.push(
|
|
54420
|
-
"unicode"
|
|
54421
|
-
/* UNICODE_PROCESSOR */
|
|
54422
|
-
);
|
|
54423
|
-
confidenceScores.push(75);
|
|
54424
|
-
reasons.push("Unicode processor pattern detected (bypass encoding risk)");
|
|
54425
|
-
}
|
|
54426
|
-
if (this.matchesPattern(toolText, [
|
|
54427
|
-
/parser/i,
|
|
54428
|
-
/parse/i,
|
|
54429
|
-
/json/i,
|
|
54430
|
-
/xml/i,
|
|
54431
|
-
/yaml/i,
|
|
54432
|
-
/nested/i,
|
|
54433
|
-
/deserialize/i,
|
|
54434
|
-
/unmarshal/i
|
|
54435
|
-
])) {
|
|
54436
|
-
categories.push(
|
|
54437
|
-
"parser"
|
|
54438
|
-
/* JSON_PARSER */
|
|
54439
|
-
);
|
|
54440
|
-
confidenceScores.push(78);
|
|
54441
|
-
reasons.push(
|
|
54442
|
-
"JSON/nested parser pattern detected (nested injection risk)"
|
|
54443
|
-
);
|
|
54444
|
-
}
|
|
54445
|
-
if (this.matchesPattern(toolText, [
|
|
54446
|
-
/install/i,
|
|
54447
|
-
/package/i,
|
|
54448
|
-
/\bnpm\b/i,
|
|
54449
|
-
/\bpip\b/i,
|
|
54450
|
-
/dependency/i,
|
|
54451
|
-
/module/i,
|
|
54452
|
-
/library/i,
|
|
54453
|
-
/\bgem\b/i
|
|
54454
|
-
])) {
|
|
54455
|
-
categories.push(
|
|
54456
|
-
"installer"
|
|
54457
|
-
/* PACKAGE_INSTALLER */
|
|
54458
|
-
);
|
|
54459
|
-
confidenceScores.push(70);
|
|
54460
|
-
reasons.push("Package installer pattern detected (typosquatting risk)");
|
|
54461
|
-
}
|
|
54462
|
-
if (this.matchesPattern(toolText, [
|
|
54463
|
-
/rug.*pull/i,
|
|
54464
|
-
/trust/i,
|
|
54465
|
-
/behavior.*change/i,
|
|
54466
|
-
/malicious.*after/i,
|
|
54467
|
-
/invocation.*count/i
|
|
54468
|
-
])) {
|
|
54469
|
-
categories.push(
|
|
54470
|
-
"rug_pull"
|
|
54471
|
-
/* RUG_PULL */
|
|
54472
|
-
);
|
|
54473
|
-
confidenceScores.push(80);
|
|
54474
|
-
reasons.push("Rug pull pattern detected (behavioral change risk)");
|
|
54475
|
-
}
|
|
54476
|
-
if (this.matchesPattern(toolText, [
|
|
54477
|
-
/firecrawl/i,
|
|
54478
|
-
/\bscrape\b/i,
|
|
54479
|
-
/\bcrawl\b/i,
|
|
54480
|
-
/web.*scraping/i,
|
|
54481
|
-
/api.*wrapper/i,
|
|
54482
|
-
/http.*client/i,
|
|
54483
|
-
/web.*client/i,
|
|
54484
|
-
/rest.*client/i,
|
|
54485
|
-
/graphql.*client/i,
|
|
54486
|
-
/fetch.*web.*content/i
|
|
54487
|
-
])) {
|
|
54488
|
-
categories.push(
|
|
54489
|
-
"api_wrapper"
|
|
54490
|
-
/* API_WRAPPER */
|
|
54491
|
-
);
|
|
54492
|
-
confidenceScores.push(95);
|
|
54493
|
-
reasons.push(
|
|
54494
|
-
"API wrapper pattern detected (safe data passing, not code execution)"
|
|
54495
|
-
);
|
|
54496
|
-
}
|
|
54497
|
-
if (this.matchesPattern(toolText, [
|
|
54498
|
-
/\bsearch\b/i,
|
|
54499
|
-
/\bfind\b/i,
|
|
54500
|
-
/\blookup\b/i,
|
|
54501
|
-
/\bquery\b/i,
|
|
54502
|
-
/retrieve/i,
|
|
54503
|
-
/\blist\b/i,
|
|
54504
|
-
/get.*users/i,
|
|
54505
|
-
/get.*pages/i,
|
|
54506
|
-
/get.*database/i
|
|
54507
|
-
])) {
|
|
54508
|
-
categories.push(
|
|
54509
|
-
"search_retrieval"
|
|
54510
|
-
/* SEARCH_RETRIEVAL */
|
|
54511
|
-
);
|
|
54512
|
-
confidenceScores.push(93);
|
|
54513
|
-
reasons.push(
|
|
54514
|
-
"Search/retrieval pattern detected (returns data, not code execution)"
|
|
54515
|
-
);
|
|
54516
|
-
}
|
|
54517
|
-
if (this.matchesPattern(toolText, [
|
|
54518
|
-
/\bcreate\b/i,
|
|
54519
|
-
/\badd\b/i,
|
|
54520
|
-
/\binsert\b/i,
|
|
54521
|
-
/\bupdate\b/i,
|
|
54522
|
-
/\bmodify\b/i,
|
|
54523
|
-
/\bdelete\b/i,
|
|
54524
|
-
/\bduplicate\b/i,
|
|
54525
|
-
/\bmove\b/i,
|
|
54526
|
-
/\bappend\b/i
|
|
54527
|
-
])) {
|
|
54528
|
-
categories.push(
|
|
54529
|
-
"crud_creation"
|
|
54530
|
-
/* CRUD_CREATION */
|
|
54531
|
-
);
|
|
54532
|
-
confidenceScores.push(92);
|
|
54533
|
-
reasons.push(
|
|
54534
|
-
"CRUD operation pattern detected (data manipulation, not code execution)"
|
|
54535
|
-
);
|
|
54536
|
-
}
|
|
54537
|
-
if (this.matchesPattern(toolText, [
|
|
54538
|
-
/get.*self/i,
|
|
54539
|
-
/get.*teams/i,
|
|
54540
|
-
/get.*info/i,
|
|
54541
|
-
/get.*status/i,
|
|
54542
|
-
/\bwhoami\b/i,
|
|
54543
|
-
/get.*workspace/i,
|
|
54544
|
-
/get.*user/i,
|
|
54545
|
-
/current.*user/i
|
|
54546
|
-
])) {
|
|
54547
|
-
categories.push(
|
|
54548
|
-
"read_only_info"
|
|
54549
|
-
/* READ_ONLY_INFO */
|
|
54550
|
-
);
|
|
54551
|
-
confidenceScores.push(94);
|
|
54552
|
-
reasons.push(
|
|
54553
|
-
"Read-only info pattern detected (intended data exposure, not vulnerability)"
|
|
54554
|
-
);
|
|
54555
|
-
}
|
|
54556
|
-
if (this.matchesPattern(toolText, [
|
|
54557
|
-
/safe.*storage/i,
|
|
54558
|
-
/safe.*search/i,
|
|
54559
|
-
/safe.*list/i,
|
|
54560
|
-
/safe.*info/i,
|
|
54561
|
-
/safe.*echo/i,
|
|
54562
|
-
/safe.*validate/i,
|
|
54563
|
-
/safe.*tool/i
|
|
54564
|
-
])) {
|
|
54565
|
-
categories.push(
|
|
54566
|
-
"safe_storage"
|
|
54567
|
-
/* SAFE_STORAGE */
|
|
54568
|
-
);
|
|
54569
|
-
confidenceScores.push(99);
|
|
54570
|
-
reasons.push(
|
|
54571
|
-
"Safe tool pattern detected (control group - should be safe)"
|
|
54572
|
-
);
|
|
54573
|
-
}
|
|
54574
|
-
if (categories.length === 0) {
|
|
54575
|
-
categories.push(
|
|
54576
|
-
"generic"
|
|
54577
|
-
/* GENERIC */
|
|
54578
|
-
);
|
|
54579
|
-
confidenceScores.push(50);
|
|
54580
|
-
reasons.push("No specific pattern match, using generic tests");
|
|
54581
|
-
}
|
|
54582
|
-
const avgConfidence = confidenceScores.reduce((a, b) => a + b, 0) / confidenceScores.length;
|
|
54583
|
-
return {
|
|
54584
|
-
toolName,
|
|
54585
|
-
categories,
|
|
54586
|
-
confidence: Math.round(avgConfidence),
|
|
54587
|
-
reasoning: reasons.join("; ")
|
|
54588
|
-
};
|
|
54589
|
-
}
|
|
54590
|
-
/**
|
|
54591
|
-
* Check if text matches any of the provided patterns
|
|
54592
|
-
*/
|
|
54593
|
-
matchesPattern(text, patterns) {
|
|
54594
|
-
return patterns.some((pattern2) => pattern2.test(text));
|
|
54595
|
-
}
|
|
54596
|
-
/**
|
|
54597
|
-
* Get all tool categories (for testing/debugging)
|
|
54598
|
-
*/
|
|
54599
|
-
static getAllCategories() {
|
|
54600
|
-
return Object.values(ToolCategory);
|
|
54601
|
-
}
|
|
54602
|
-
/**
|
|
54603
|
-
* Get risk level for a category
|
|
54604
|
-
*/
|
|
54605
|
-
static getRiskLevel(category) {
|
|
54606
|
-
const highRiskCategories = [
|
|
54607
|
-
"calculator",
|
|
54608
|
-
"system_exec",
|
|
54609
|
-
"data_access",
|
|
54610
|
-
"tool_override",
|
|
54611
|
-
"config_modifier",
|
|
54612
|
-
"fetcher"
|
|
54613
|
-
/* URL_FETCHER */
|
|
54614
|
-
];
|
|
54615
|
-
const mediumRiskCategories = [
|
|
54616
|
-
"unicode",
|
|
54617
|
-
"parser",
|
|
54618
|
-
"installer",
|
|
54619
|
-
"rug_pull"
|
|
54620
|
-
/* RUG_PULL */
|
|
54621
|
-
];
|
|
54622
|
-
const lowRiskCategories = [
|
|
54623
|
-
"api_wrapper",
|
|
54624
|
-
"search_retrieval",
|
|
54625
|
-
"crud_creation",
|
|
54626
|
-
"read_only_info",
|
|
54627
|
-
"safe_storage",
|
|
54628
|
-
"generic"
|
|
54629
|
-
/* GENERIC */
|
|
54630
|
-
];
|
|
54631
|
-
if (highRiskCategories.includes(category)) return "HIGH";
|
|
54632
|
-
if (mediumRiskCategories.includes(category)) return "MEDIUM";
|
|
54633
|
-
if (lowRiskCategories.includes(category)) return "LOW";
|
|
54634
|
-
return "LOW";
|
|
54635
|
-
}
|
|
54636
|
-
/**
|
|
54637
|
-
* Classify multiple tools at once
|
|
54638
|
-
*/
|
|
54639
|
-
classifyBatch(tools) {
|
|
54640
|
-
return tools.map((tool) => this.classify(tool.name, tool.description));
|
|
54641
|
-
}
|
|
54642
|
-
}
|
|
54643
55453
|
class SecurityAssessor extends BaseAssessor {
|
|
54644
55454
|
async assess(context) {
|
|
54645
55455
|
const toolsToTest = this.selectToolsForTesting(context.tools);
|
|
@@ -54809,6 +55619,18 @@ class SecurityAssessor extends BaseAssessor {
|
|
|
54809
55619
|
this.log(
|
|
54810
55620
|
`🚨 VULNERABILITY: ${tool.name} - ${attackPattern.attackName} (${payload.payloadType}: ${payload.description})`
|
|
54811
55621
|
);
|
|
55622
|
+
if (context.onProgress) {
|
|
55623
|
+
context.onProgress({
|
|
55624
|
+
type: "vulnerability_found",
|
|
55625
|
+
tool: tool.name,
|
|
55626
|
+
pattern: attackPattern.attackName,
|
|
55627
|
+
confidence: result.confidence || "medium",
|
|
55628
|
+
evidence: result.evidence || "Vulnerability detected",
|
|
55629
|
+
riskLevel: payload.riskLevel,
|
|
55630
|
+
requiresReview: result.requiresManualReview || false,
|
|
55631
|
+
payload: payload.payload
|
|
55632
|
+
});
|
|
55633
|
+
}
|
|
54812
55634
|
}
|
|
54813
55635
|
} catch (error) {
|
|
54814
55636
|
this.logError(
|
|
@@ -54926,6 +55748,18 @@ class SecurityAssessor extends BaseAssessor {
|
|
|
54926
55748
|
this.log(
|
|
54927
55749
|
`🚨 VULNERABILITY: ${tool.name} - ${attackPattern.attackName}`
|
|
54928
55750
|
);
|
|
55751
|
+
if (context.onProgress) {
|
|
55752
|
+
context.onProgress({
|
|
55753
|
+
type: "vulnerability_found",
|
|
55754
|
+
tool: tool.name,
|
|
55755
|
+
pattern: attackPattern.attackName,
|
|
55756
|
+
confidence: result.confidence || "medium",
|
|
55757
|
+
evidence: result.evidence || "Vulnerability detected",
|
|
55758
|
+
riskLevel: payload.riskLevel,
|
|
55759
|
+
requiresReview: result.requiresManualReview || false,
|
|
55760
|
+
payload: payload.payload
|
|
55761
|
+
});
|
|
55762
|
+
}
|
|
54929
55763
|
}
|
|
54930
55764
|
} catch (error) {
|
|
54931
55765
|
this.logError(
|
|
@@ -60906,13 +61740,13 @@ const App = () => {
|
|
|
60906
61740
|
) });
|
|
60907
61741
|
if (window.location.pathname === "/oauth/callback") {
|
|
60908
61742
|
const OAuthCallback = React.lazy(
|
|
60909
|
-
() => __vitePreload(() => import("./OAuthCallback-
|
|
61743
|
+
() => __vitePreload(() => import("./OAuthCallback-DA2koy6X.js"), true ? [] : void 0)
|
|
60910
61744
|
);
|
|
60911
61745
|
return /* @__PURE__ */ jsxRuntimeExports.jsx(reactExports.Suspense, { fallback: /* @__PURE__ */ jsxRuntimeExports.jsx("div", { children: "Loading..." }), children: /* @__PURE__ */ jsxRuntimeExports.jsx(OAuthCallback, { onConnect: onOAuthConnect }) });
|
|
60912
61746
|
}
|
|
60913
61747
|
if (window.location.pathname === "/oauth/callback/debug") {
|
|
60914
61748
|
const OAuthDebugCallback = React.lazy(
|
|
60915
|
-
() => __vitePreload(() => import("./OAuthDebugCallback-
|
|
61749
|
+
() => __vitePreload(() => import("./OAuthDebugCallback-Bx60PQTT.js"), true ? [] : void 0)
|
|
60916
61750
|
);
|
|
60917
61751
|
return /* @__PURE__ */ jsxRuntimeExports.jsx(reactExports.Suspense, { fallback: /* @__PURE__ */ jsxRuntimeExports.jsx("div", { children: "Loading..." }), children: /* @__PURE__ */ jsxRuntimeExports.jsx(OAuthDebugCallback, { onConnect: onOAuthDebugConnect }) });
|
|
60918
61752
|
}
|