@vfarcic/dot-ai 0.111.0 → 0.113.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/ai-provider-factory.d.ts +0 -10
- package/dist/core/ai-provider-factory.d.ts.map +1 -1
- package/dist/core/ai-provider-factory.js +14 -24
- package/dist/core/ai-provider.interface.d.ts +28 -1
- package/dist/core/ai-provider.interface.d.ts.map +1 -1
- package/dist/core/capabilities.d.ts +1 -1
- package/dist/core/capabilities.d.ts.map +1 -1
- package/dist/core/capabilities.js +7 -4
- package/dist/core/capability-scan-workflow.js +2 -2
- package/dist/core/embedding-service.d.ts +35 -2
- package/dist/core/embedding-service.d.ts.map +1 -1
- package/dist/core/embedding-service.js +228 -15
- package/dist/core/model-config.d.ts +23 -0
- package/dist/core/model-config.d.ts.map +1 -0
- package/dist/core/model-config.js +28 -0
- package/dist/core/platform-operations.d.ts.map +1 -1
- package/dist/core/platform-operations.js +3 -5
- package/dist/core/platform-utils.d.ts +13 -2
- package/dist/core/platform-utils.d.ts.map +1 -1
- package/dist/core/platform-utils.js +91 -9
- package/dist/core/providers/anthropic-provider.d.ts +6 -1
- package/dist/core/providers/anthropic-provider.d.ts.map +1 -1
- package/dist/core/providers/anthropic-provider.js +99 -27
- package/dist/core/providers/provider-debug-utils.d.ts +53 -20
- package/dist/core/providers/provider-debug-utils.d.ts.map +1 -1
- package/dist/core/providers/provider-debug-utils.js +106 -51
- package/dist/core/providers/vercel-provider.d.ts +6 -1
- package/dist/core/providers/vercel-provider.d.ts.map +1 -1
- package/dist/core/providers/vercel-provider.js +212 -130
- package/dist/core/schema.d.ts +1 -101
- package/dist/core/schema.d.ts.map +1 -1
- package/dist/core/schema.js +20 -154
- package/dist/core/unified-creation-session.d.ts.map +1 -1
- package/dist/core/unified-creation-session.js +15 -7
- package/dist/evaluation/dataset-analyzer.d.ts +118 -0
- package/dist/evaluation/dataset-analyzer.d.ts.map +1 -0
- package/dist/evaluation/dataset-analyzer.js +234 -0
- package/dist/evaluation/datasets/loader.d.ts +42 -0
- package/dist/evaluation/datasets/loader.d.ts.map +1 -0
- package/dist/evaluation/datasets/loader.js +104 -0
- package/dist/evaluation/eval-runner.d.ts +9 -0
- package/dist/evaluation/eval-runner.d.ts.map +1 -0
- package/dist/evaluation/eval-runner.js +399 -0
- package/dist/evaluation/evaluators/base-comparative.d.ts +94 -0
- package/dist/evaluation/evaluators/base-comparative.d.ts.map +1 -0
- package/dist/evaluation/evaluators/base-comparative.js +187 -0
- package/dist/evaluation/evaluators/base.d.ts +47 -0
- package/dist/evaluation/evaluators/base.d.ts.map +1 -0
- package/dist/evaluation/evaluators/base.js +10 -0
- package/dist/evaluation/evaluators/capability-comparative.d.ts +32 -0
- package/dist/evaluation/evaluators/capability-comparative.d.ts.map +1 -0
- package/dist/evaluation/evaluators/capability-comparative.js +104 -0
- package/dist/evaluation/evaluators/pattern-comparative.d.ts +31 -0
- package/dist/evaluation/evaluators/pattern-comparative.d.ts.map +1 -0
- package/dist/evaluation/evaluators/pattern-comparative.js +97 -0
- package/dist/evaluation/evaluators/policy-comparative.d.ts +31 -0
- package/dist/evaluation/evaluators/policy-comparative.d.ts.map +1 -0
- package/dist/evaluation/evaluators/policy-comparative.js +97 -0
- package/dist/evaluation/evaluators/recommendation-comparative.d.ts +25 -0
- package/dist/evaluation/evaluators/recommendation-comparative.d.ts.map +1 -0
- package/dist/evaluation/evaluators/recommendation-comparative.js +55 -0
- package/dist/evaluation/evaluators/remediation-comparative.d.ts +25 -0
- package/dist/evaluation/evaluators/remediation-comparative.d.ts.map +1 -0
- package/dist/evaluation/evaluators/remediation-comparative.js +54 -0
- package/dist/evaluation/platform-synthesizer.d.ts +54 -0
- package/dist/evaluation/platform-synthesizer.d.ts.map +1 -0
- package/dist/evaluation/platform-synthesizer.js +368 -0
- package/dist/evaluation/run-platform-synthesis.d.ts +9 -0
- package/dist/evaluation/run-platform-synthesis.d.ts.map +1 -0
- package/dist/evaluation/run-platform-synthesis.js +45 -0
- package/dist/interfaces/mcp.d.ts.map +1 -1
- package/dist/interfaces/mcp.js +23 -29
- package/dist/interfaces/rest-api.d.ts.map +1 -1
- package/dist/tools/answer-question.d.ts +2 -0
- package/dist/tools/answer-question.d.ts.map +1 -1
- package/dist/tools/answer-question.js +18 -11
- package/dist/tools/generate-manifests.d.ts +2 -0
- package/dist/tools/generate-manifests.d.ts.map +1 -1
- package/dist/tools/generate-manifests.js +11 -12
- package/dist/tools/organizational-data.d.ts +1 -0
- package/dist/tools/organizational-data.d.ts.map +1 -1
- package/dist/tools/organizational-data.js +2 -1
- package/dist/tools/recommend.d.ts +1 -0
- package/dist/tools/recommend.d.ts.map +1 -1
- package/dist/tools/recommend.js +13 -21
- package/dist/tools/remediate.d.ts +3 -0
- package/dist/tools/remediate.d.ts.map +1 -1
- package/dist/tools/remediate.js +35 -14
- package/dist/tools/test-docs.d.ts +1 -0
- package/dist/tools/test-docs.d.ts.map +1 -1
- package/dist/tools/test-docs.js +4 -2
- package/dist/tools/version.d.ts +5 -1
- package/dist/tools/version.d.ts.map +1 -1
- package/dist/tools/version.js +23 -8
- package/package.json +19 -1
package/dist/core/schema.js
CHANGED
|
@@ -46,6 +46,7 @@ const vector_db_service_1 = require("./vector-db-service");
|
|
|
46
46
|
const capability_vector_service_1 = require("./capability-vector-service");
|
|
47
47
|
const policy_vector_service_1 = require("./policy-vector-service");
|
|
48
48
|
const shared_prompt_loader_1 = require("./shared-prompt-loader");
|
|
49
|
+
const platform_utils_1 = require("./platform-utils");
|
|
49
50
|
/**
|
|
50
51
|
* SchemaParser converts kubectl explain output to structured ResourceSchema
|
|
51
52
|
*/
|
|
@@ -312,7 +313,7 @@ class ResourceRecommender {
|
|
|
312
313
|
/**
|
|
313
314
|
* Find the best resource solution(s) for user intent using two-phase analysis
|
|
314
315
|
*/
|
|
315
|
-
async findBestSolutions(intent, _explainResource) {
|
|
316
|
+
async findBestSolutions(intent, _explainResource, interaction_id) {
|
|
316
317
|
if (!this.aiProvider.isInitialized()) {
|
|
317
318
|
throw new Error('AI provider not initialized. API key required for AI-powered resource ranking.');
|
|
318
319
|
}
|
|
@@ -351,18 +352,16 @@ class ResourceRecommender {
|
|
|
351
352
|
const capabilityFilteredResources = relevantCapabilities.map(cap => ({
|
|
352
353
|
kind: this.extractKindFromResourceName(cap.data.resourceName),
|
|
353
354
|
group: this.extractGroupFromResourceName(cap.data.resourceName),
|
|
354
|
-
apiVersion: this.constructApiVersionFromResourceName(cap.data.resourceName),
|
|
355
355
|
resourceName: cap.data.resourceName,
|
|
356
|
-
|
|
357
|
-
capabilities: cap.data // Include capability data for AI decision-making
|
|
356
|
+
capabilities: cap.data // Include capability data for AI decision-making (includes namespaced, etc.)
|
|
358
357
|
}));
|
|
359
358
|
// Phase 1: Add missing pattern-suggested resources to available resources list
|
|
360
359
|
const enhancedResources = await this.addMissingPatternResources(capabilityFilteredResources, relevantPatterns);
|
|
361
360
|
// Phase 2: AI assembles and ranks complete solutions (replaces separate selection + ranking phases)
|
|
362
|
-
const solutions = await this.assembleAndRankSolutions(intent, enhancedResources, relevantPatterns);
|
|
361
|
+
const solutions = await this.assembleAndRankSolutions(intent, enhancedResources, relevantPatterns, interaction_id);
|
|
363
362
|
// Phase 3: Generate questions for each solution
|
|
364
363
|
for (const solution of solutions) {
|
|
365
|
-
solution.questions = await this.generateQuestionsWithAI(intent, solution, _explainResource);
|
|
364
|
+
solution.questions = await this.generateQuestionsWithAI(intent, solution, _explainResource, interaction_id);
|
|
366
365
|
}
|
|
367
366
|
return solutions;
|
|
368
367
|
}
|
|
@@ -373,9 +372,12 @@ class ResourceRecommender {
|
|
|
373
372
|
/**
|
|
374
373
|
* Phase 2: AI assembles and ranks complete solutions (replaces separate selection + ranking)
|
|
375
374
|
*/
|
|
376
|
-
async assembleAndRankSolutions(intent, availableResources, patterns) {
|
|
375
|
+
async assembleAndRankSolutions(intent, availableResources, patterns, interaction_id) {
|
|
377
376
|
const prompt = await this.loadSolutionAssemblyPrompt(intent, availableResources, patterns);
|
|
378
|
-
const response = await this.aiProvider.sendMessage(prompt, 'solution-assembly'
|
|
377
|
+
const response = await this.aiProvider.sendMessage(prompt, 'recommend-solution-assembly', {
|
|
378
|
+
user_intent: intent ? `Kubernetes solution assembly for: ${intent}` : 'Kubernetes solution assembly',
|
|
379
|
+
interaction_id: interaction_id || 'recommend_solution_assembly'
|
|
380
|
+
});
|
|
379
381
|
return this.parseSimpleSolutionResponse(response.content);
|
|
380
382
|
}
|
|
381
383
|
/**
|
|
@@ -384,7 +386,7 @@ class ResourceRecommender {
|
|
|
384
386
|
parseSimpleSolutionResponse(aiResponse) {
|
|
385
387
|
try {
|
|
386
388
|
// Use robust JSON extraction
|
|
387
|
-
const parsed =
|
|
389
|
+
const parsed = (0, platform_utils_1.extractJsonFromAIResponse)(aiResponse);
|
|
388
390
|
const solutions = parsed.solutions.map((solution) => {
|
|
389
391
|
const isDebugMode = process.env.DOT_AI_DEBUG === 'true';
|
|
390
392
|
if (isDebugMode) {
|
|
@@ -429,9 +431,8 @@ class ResourceRecommender {
|
|
|
429
431
|
const template = (0, shared_prompt_loader_1.loadPrompt)('resource-selection');
|
|
430
432
|
// Format resources for the prompt with capability information
|
|
431
433
|
const resourcesText = resources.map((resource, index) => {
|
|
432
|
-
return `${index}: ${resource.kind.toUpperCase()}
|
|
434
|
+
return `${index}: ${resource.kind.toUpperCase()}
|
|
433
435
|
Group: ${resource.group || 'core'}
|
|
434
|
-
Namespaced: ${resource.namespaced}
|
|
435
436
|
Resource Name: ${resource.resourceName}
|
|
436
437
|
Capabilities: ${Array.isArray(resource.capabilities.capabilities) ? resource.capabilities.capabilities.join(', ') : 'Not specified'}
|
|
437
438
|
Providers: ${Array.isArray(resource.capabilities.providers) ? resource.capabilities.providers.join(', ') : resource.capabilities.providers || 'kubernetes'}
|
|
@@ -480,14 +481,10 @@ class ResourceRecommender {
|
|
|
480
481
|
const parts = suggestedResource.split('.');
|
|
481
482
|
const kind = parts[0]; // Use resource name as-is: resourcegroups, servicemonitors, etc.
|
|
482
483
|
const group = parts.length > 1 ? parts.slice(1).join('.') : '';
|
|
483
|
-
const version = 'v1beta1'; // Default version for CRDs, could be enhanced
|
|
484
|
-
const apiVersion = group ? `${group}/${version}` : version;
|
|
485
484
|
missingPatternResources.push({
|
|
486
485
|
kind,
|
|
487
486
|
group,
|
|
488
|
-
apiVersion,
|
|
489
487
|
resourceName,
|
|
490
|
-
namespaced: true, // Default assumption for pattern resources
|
|
491
488
|
capabilities: {
|
|
492
489
|
resourceName,
|
|
493
490
|
description: `Resource suggested by organizational pattern: ${pattern.description}`,
|
|
@@ -547,17 +544,8 @@ class ResourceRecommender {
|
|
|
547
544
|
// Return everything after the first dot
|
|
548
545
|
return resourceName.substring(resourceName.indexOf('.') + 1);
|
|
549
546
|
}
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
*/
|
|
553
|
-
constructApiVersionFromResourceName(resourceName) {
|
|
554
|
-
if (!resourceName.includes('.')) {
|
|
555
|
-
return 'v1'; // Core resources typically use v1
|
|
556
|
-
}
|
|
557
|
-
// For CRDs, construct group/version format
|
|
558
|
-
const group = this.extractGroupFromResourceName(resourceName);
|
|
559
|
-
return `${group}/v1beta1`; // Default to v1beta1 for CRDs
|
|
560
|
-
}
|
|
547
|
+
// Note: constructApiVersionFromResourceName method removed - no longer needed
|
|
548
|
+
// API versions are extracted from kubectl explain schema content during manifest generation
|
|
561
549
|
/**
|
|
562
550
|
* Phase 0: Search for relevant organizational patterns using multi-concept approach
|
|
563
551
|
* Returns empty array if Vector DB is not available - this is completely optional
|
|
@@ -581,98 +569,6 @@ class ResourceRecommender {
|
|
|
581
569
|
}
|
|
582
570
|
// REMOVED: selectResourceCandidates - replaced by single-phase assembleAndRankSolutions
|
|
583
571
|
// REMOVED: fetchDetailedSchemas - no longer needed in single-phase architecture
|
|
584
|
-
/**
|
|
585
|
-
const basic = `${index}: ${resource.kind} (${resource.apiVersion})
|
|
586
|
-
Group: ${resource.group || 'core'}
|
|
587
|
-
Namespaced: ${resource.namespaced}`;
|
|
588
|
-
|
|
589
|
-
// Include rich capability context if available (from capability-based pre-filtering)
|
|
590
|
-
if (resource.capabilities) {
|
|
591
|
-
const cap = resource.capabilities;
|
|
592
|
-
return `${basic}
|
|
593
|
-
Resource Name: ${resource.resourceName || 'Not specified'}
|
|
594
|
-
Capabilities: ${cap.capabilities?.join(', ') || 'Not specified'}
|
|
595
|
-
Providers: ${cap.providers?.join(', ') || 'Not specified'}
|
|
596
|
-
Complexity: ${cap.complexity || 'Not specified'}
|
|
597
|
-
Use Case: ${cap.useCase || 'Not specified'}
|
|
598
|
-
Description: ${cap.description || 'Not specified'}
|
|
599
|
-
Confidence: ${cap.confidence || 'N/A'}`;
|
|
600
|
-
}
|
|
601
|
-
|
|
602
|
-
return basic;
|
|
603
|
-
}).join('\n\n');
|
|
604
|
-
|
|
605
|
-
// Format organizational patterns for AI context
|
|
606
|
-
const patternsContext = patterns.length > 0
|
|
607
|
-
? patterns.map(pattern =>
|
|
608
|
-
`- ID: ${pattern.id}
|
|
609
|
-
Description: ${pattern.description}
|
|
610
|
-
Suggested Resources: ${pattern.suggestedResources?.join(', ') || 'Not specified'}
|
|
611
|
-
Rationale: ${pattern.rationale}
|
|
612
|
-
Triggers: ${pattern.triggers?.join(', ') || 'None'}`
|
|
613
|
-
).join('\n')
|
|
614
|
-
: 'No organizational patterns found for this request.';
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
const template = loadPrompt('resource-selection');
|
|
618
|
-
|
|
619
|
-
const selectionPrompt = template
|
|
620
|
-
.replace('{intent}', intent)
|
|
621
|
-
.replace('{resources}', resourceSummary)
|
|
622
|
-
.replace('{patterns}', patternsContext);
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
const response = await this.aiProvider.sendMessage(selectionPrompt, 'resource-selection');
|
|
626
|
-
|
|
627
|
-
try {
|
|
628
|
-
// Extract JSON from response with robust parsing
|
|
629
|
-
let jsonContent = response.content;
|
|
630
|
-
|
|
631
|
-
// First try to find JSON array wrapped in code blocks
|
|
632
|
-
const codeBlockMatch = response.content.match(/```(?:json)?\s*(\[[\s\S]*?\])\s*```/);
|
|
633
|
-
if (codeBlockMatch) {
|
|
634
|
-
jsonContent = codeBlockMatch[1];
|
|
635
|
-
} else {
|
|
636
|
-
// Try to find JSON array that starts with [ and find the matching closing ]
|
|
637
|
-
const startIndex = response.content.indexOf('[');
|
|
638
|
-
if (startIndex !== -1) {
|
|
639
|
-
let bracketCount = 0;
|
|
640
|
-
let endIndex = startIndex;
|
|
641
|
-
|
|
642
|
-
for (let i = startIndex; i < response.content.length; i++) {
|
|
643
|
-
if (response.content[i] === '[') bracketCount++;
|
|
644
|
-
if (response.content[i] === ']') bracketCount--;
|
|
645
|
-
if (bracketCount === 0) {
|
|
646
|
-
endIndex = i;
|
|
647
|
-
break;
|
|
648
|
-
}
|
|
649
|
-
}
|
|
650
|
-
|
|
651
|
-
if (bracketCount === 0) {
|
|
652
|
-
jsonContent = response.content.substring(startIndex, endIndex + 1);
|
|
653
|
-
}
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
|
|
657
|
-
const selectedResources = JSON.parse(jsonContent.trim());
|
|
658
|
-
|
|
659
|
-
if (!Array.isArray(selectedResources)) {
|
|
660
|
-
throw new Error('AI response is not an array');
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
// Validate that each resource has required fields
|
|
664
|
-
for (const resource of selectedResources) {
|
|
665
|
-
if (!resource.kind || !resource.apiVersion) {
|
|
666
|
-
throw new Error(`AI selected invalid resource: ${JSON.stringify(resource)}`);
|
|
667
|
-
}
|
|
668
|
-
}
|
|
669
|
-
|
|
670
|
-
return selectedResources;
|
|
671
|
-
} catch (error) {
|
|
672
|
-
throw new Error(`AI failed to select resources in valid JSON format. Error: ${(error as Error).message}. AI response: "${response.content.substring(0, 200)}..."`);
|
|
673
|
-
}
|
|
674
|
-
}
|
|
675
|
-
|
|
676
572
|
/**
|
|
677
573
|
* Phase 2: Fetch detailed schemas for selected candidates
|
|
678
574
|
*/
|
|
@@ -780,43 +676,10 @@ class ResourceRecommender {
|
|
|
780
676
|
};
|
|
781
677
|
}
|
|
782
678
|
}
|
|
783
|
-
/**
|
|
784
|
-
* Extract JSON object from AI response with robust parsing
|
|
785
|
-
*/
|
|
786
|
-
extractJsonFromAIResponse(aiResponse) {
|
|
787
|
-
let jsonContent = aiResponse;
|
|
788
|
-
// First try to find JSON wrapped in code blocks
|
|
789
|
-
const codeBlockMatch = aiResponse.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/);
|
|
790
|
-
if (codeBlockMatch) {
|
|
791
|
-
jsonContent = codeBlockMatch[1];
|
|
792
|
-
}
|
|
793
|
-
else {
|
|
794
|
-
// Try to find JSON that starts with { and find the matching closing }
|
|
795
|
-
const startIndex = aiResponse.indexOf('{');
|
|
796
|
-
if (startIndex !== -1) {
|
|
797
|
-
let braceCount = 0;
|
|
798
|
-
let endIndex = startIndex;
|
|
799
|
-
for (let i = startIndex; i < aiResponse.length; i++) {
|
|
800
|
-
if (aiResponse[i] === '{')
|
|
801
|
-
braceCount++;
|
|
802
|
-
if (aiResponse[i] === '}')
|
|
803
|
-
braceCount--;
|
|
804
|
-
if (braceCount === 0) {
|
|
805
|
-
endIndex = i;
|
|
806
|
-
break;
|
|
807
|
-
}
|
|
808
|
-
}
|
|
809
|
-
if (braceCount === 0) {
|
|
810
|
-
jsonContent = aiResponse.substring(startIndex, endIndex + 1);
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
|
-
}
|
|
814
|
-
return JSON.parse(jsonContent.trim());
|
|
815
|
-
}
|
|
816
679
|
/**
|
|
817
680
|
* Generate contextual questions using AI based on user intent and solution resources
|
|
818
681
|
*/
|
|
819
|
-
async generateQuestionsWithAI(intent, solution, _explainResource) {
|
|
682
|
+
async generateQuestionsWithAI(intent, solution, _explainResource, interaction_id) {
|
|
820
683
|
try {
|
|
821
684
|
// Discover cluster options for dynamic questions
|
|
822
685
|
const clusterOptions = await this.discoverClusterOptions();
|
|
@@ -903,9 +766,12 @@ Available Node Labels: ${clusterOptions.nodeLabels.length > 0 ? clusterOptions.n
|
|
|
903
766
|
.replace('{resource_details}', resourceDetails)
|
|
904
767
|
.replace('{cluster_options}', clusterOptionsText)
|
|
905
768
|
.replace('{policy_context}', policyContextText);
|
|
906
|
-
const response = await this.aiProvider.sendMessage(questionPrompt, 'question-generation'
|
|
769
|
+
const response = await this.aiProvider.sendMessage(questionPrompt, 'recommend-question-generation', {
|
|
770
|
+
user_intent: `Generate deployment questions for: ${intent}`,
|
|
771
|
+
interaction_id: interaction_id || 'recommend_question_generation'
|
|
772
|
+
});
|
|
907
773
|
// Use robust JSON extraction
|
|
908
|
-
const questions =
|
|
774
|
+
const questions = (0, platform_utils_1.extractJsonFromAIResponse)(response.content);
|
|
909
775
|
// Validate the response structure
|
|
910
776
|
if (!questions.required || !questions.basic || !questions.advanced || !questions.open) {
|
|
911
777
|
throw new Error('Invalid question structure from AI');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"unified-creation-session.d.ts","sourceRoot":"","sources":["../../src/core/unified-creation-session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAQH,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"unified-creation-session.d.ts","sourceRoot":"","sources":["../../src/core/unified-creation-session.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAQH,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAKlD,OAAO,EACL,sBAAsB,EACtB,2BAA2B,EAC3B,iCAAiC,EACjC,UAAU,EAIX,MAAM,0BAA0B,CAAC;AAKlC,qBAAa,6BAA6B;IACxC,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,SAAS,CAAsB;gBAE3B,UAAU,EAAE,UAAU,EAAE,SAAS,CAAC,EAAE,mBAAmB;IAKnE;;OAEG;IACH,aAAa,CAAC,IAAI,EAAE,GAAG,GAAG,sBAAsB;IAqBhD;;OAEG;IACH,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,sBAAsB,GAAG,IAAI;IAiBxE;;OAEG;IACH,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,GAAG,sBAAsB;IA2GvF;;OAEG;IACG,mBAAmB,CAAC,OAAO,EAAE,sBAAsB,EAAE,IAAI,CAAC,EAAE,GAAG,GAAG,OAAO,CAAC,2BAA2B,GAAG,iCAAiC,CAAC;IAoGhJ;;OAEG;YACW,4BAA4B;IA4C1C;;OAEG;YACW,gCAAgC;IAyC9C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA6E1B;;OAEG;YACW,gBAAgB;IAsF9B;;OAEG;YACW,4BAA4B;IAoH1C;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAY1B;;OAEG;YACW,qBAAqB;IAqCnC;;OAEG;YACW,mBAAmB;IAyLjC;;OAEG;YACW,uBAAuB;IAmFrC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAU5B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAY9B;;OAEG;IACH,OAAO,CAAC,WAAW;IAgBnB;;OAEG;IACH,OAAO,CAAC,iBAAiB;CAG1B"}
|
|
@@ -50,6 +50,7 @@ const capability_vector_service_1 = require("./capability-vector-service");
|
|
|
50
50
|
const discovery_1 = require("./discovery");
|
|
51
51
|
const schema_1 = require("./schema");
|
|
52
52
|
const version_1 = require("../tools/version");
|
|
53
|
+
const platform_utils_1 = require("./platform-utils");
|
|
53
54
|
const yaml = __importStar(require("js-yaml"));
|
|
54
55
|
const unified_creation_types_1 = require("./unified-creation-types");
|
|
55
56
|
const pattern_operations_1 = require("./pattern-operations");
|
|
@@ -225,7 +226,7 @@ class UnifiedCreationSessionManager {
|
|
|
225
226
|
nextStep: (0, unified_creation_types_1.getNextStep)('triggers', this.config) || undefined
|
|
226
227
|
};
|
|
227
228
|
case 'trigger-expansion':
|
|
228
|
-
return await this.generateTriggerExpansionStep(session);
|
|
229
|
+
return await this.generateTriggerExpansionStep(session, args?.interaction_id);
|
|
229
230
|
case 'resources':
|
|
230
231
|
if (this.config.entityType === 'pattern') {
|
|
231
232
|
return {
|
|
@@ -291,12 +292,12 @@ class UnifiedCreationSessionManager {
|
|
|
291
292
|
/**
|
|
292
293
|
* Generate trigger expansion step with AI suggestions
|
|
293
294
|
*/
|
|
294
|
-
async generateTriggerExpansionStep(session) {
|
|
295
|
+
async generateTriggerExpansionStep(session, interaction_id) {
|
|
295
296
|
const description = session.data.description || '';
|
|
296
297
|
const initialTriggers = session.data.initialTriggers || [];
|
|
297
298
|
try {
|
|
298
299
|
// Generate expanded triggers internally using AI
|
|
299
|
-
const expandedTriggers = await this.generateInternalTriggerExpansion(initialTriggers, description);
|
|
300
|
+
const expandedTriggers = await this.generateInternalTriggerExpansion(initialTriggers, description, interaction_id);
|
|
300
301
|
// Combine initial and expanded triggers into full list
|
|
301
302
|
const fullTriggerList = [...initialTriggers, ...expandedTriggers];
|
|
302
303
|
if (expandedTriggers.length === 0) {
|
|
@@ -333,7 +334,7 @@ class UnifiedCreationSessionManager {
|
|
|
333
334
|
/**
|
|
334
335
|
* Generate trigger expansion using internal AI
|
|
335
336
|
*/
|
|
336
|
-
async generateInternalTriggerExpansion(initialTriggers, description) {
|
|
337
|
+
async generateInternalTriggerExpansion(initialTriggers, description, interaction_id) {
|
|
337
338
|
const { createAIProvider } = require('./ai-provider-factory');
|
|
338
339
|
const aiProvider = createAIProvider();
|
|
339
340
|
if (!aiProvider.isInitialized()) {
|
|
@@ -345,7 +346,10 @@ class UnifiedCreationSessionManager {
|
|
|
345
346
|
description
|
|
346
347
|
});
|
|
347
348
|
try {
|
|
348
|
-
const response = await aiProvider.sendMessage(prompt,
|
|
349
|
+
const response = await aiProvider.sendMessage(prompt, `${this.config.entityType}-trigger-expansion`, {
|
|
350
|
+
user_intent: `Create organizational ${this.config.entityType} for ${initialTriggers.join(', ') || 'unspecified domain'}`,
|
|
351
|
+
interaction_id: interaction_id || 'trigger_expansion'
|
|
352
|
+
});
|
|
349
353
|
const expandedText = response.content.trim();
|
|
350
354
|
if (!expandedText || expandedText.toLowerCase().includes('no relevant') || expandedText.toLowerCase().includes('no additional')) {
|
|
351
355
|
return [];
|
|
@@ -735,9 +739,13 @@ The policy intent has been stored in the database. The Kyverno policy was not ap
|
|
|
735
739
|
// Call AI provider internally to generate Kyverno policy
|
|
736
740
|
const { createAIProvider } = require('./ai-provider-factory');
|
|
737
741
|
const aiProvider = createAIProvider();
|
|
738
|
-
const response = await aiProvider.sendMessage(prompt, 'kyverno-generation'
|
|
742
|
+
const response = await aiProvider.sendMessage(prompt, 'policy-kyverno-generation', {
|
|
743
|
+
user_intent: data.description ? `Create Kyverno policy: ${data.description}` : `Generate Kyverno policy for ${finalTriggers.join(', ')}`,
|
|
744
|
+
interaction_id: args?.interaction_id || 'kyverno_generation'
|
|
745
|
+
});
|
|
739
746
|
// Response should be clean YAML with analysis comments
|
|
740
|
-
|
|
747
|
+
// Extract YAML from code blocks if wrapped (using shared utility)
|
|
748
|
+
const kyvernoPolicy = (0, platform_utils_1.extractContentFromMarkdownCodeBlocks)(response.content, 'yaml');
|
|
741
749
|
// Save policy to file immediately after generation
|
|
742
750
|
const yamlPath = path.join(policySessionDir, `${session.sessionId}-kyverno.yaml`);
|
|
743
751
|
fs.writeFileSync(yamlPath, kyvernoPolicy, 'utf8');
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dataset Analyzer for Multi-Model Comparison
|
|
3
|
+
*
|
|
4
|
+
* Analyzes evaluation datasets to group them by scenario and extract
|
|
5
|
+
* model responses for comparative evaluation.
|
|
6
|
+
*/
|
|
7
|
+
export interface DatasetSample {
|
|
8
|
+
input: {
|
|
9
|
+
issue: string;
|
|
10
|
+
[key: string]: any;
|
|
11
|
+
};
|
|
12
|
+
output: string;
|
|
13
|
+
performance: {
|
|
14
|
+
duration_ms: number;
|
|
15
|
+
input_tokens: number;
|
|
16
|
+
output_tokens: number;
|
|
17
|
+
total_tokens: number;
|
|
18
|
+
model_version: string;
|
|
19
|
+
sdk: string;
|
|
20
|
+
iterations?: number;
|
|
21
|
+
tool_calls_executed?: number;
|
|
22
|
+
cache_read_tokens?: number;
|
|
23
|
+
cache_creation_tokens?: number;
|
|
24
|
+
};
|
|
25
|
+
metadata: {
|
|
26
|
+
timestamp: string;
|
|
27
|
+
complexity: string;
|
|
28
|
+
tags: string[];
|
|
29
|
+
source: string;
|
|
30
|
+
tool: string;
|
|
31
|
+
test_scenario: string;
|
|
32
|
+
failure_analysis: string;
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
export interface ModelResponse {
|
|
36
|
+
model: string;
|
|
37
|
+
response: string;
|
|
38
|
+
performance: {
|
|
39
|
+
duration_ms: number;
|
|
40
|
+
input_tokens: number;
|
|
41
|
+
output_tokens: number;
|
|
42
|
+
total_tokens: number;
|
|
43
|
+
iterations?: number;
|
|
44
|
+
tool_calls_executed?: number;
|
|
45
|
+
cache_read_tokens?: number;
|
|
46
|
+
cache_creation_tokens?: number;
|
|
47
|
+
};
|
|
48
|
+
metadata: {
|
|
49
|
+
timestamp: string;
|
|
50
|
+
complexity: string;
|
|
51
|
+
test_scenario: string;
|
|
52
|
+
issue?: string;
|
|
53
|
+
interaction_count?: number;
|
|
54
|
+
failure_analysis?: {
|
|
55
|
+
failure_type: string;
|
|
56
|
+
failure_reason: string;
|
|
57
|
+
time_to_failure: number;
|
|
58
|
+
};
|
|
59
|
+
all_failures?: Array<{
|
|
60
|
+
failure_type: string;
|
|
61
|
+
failure_reason: string;
|
|
62
|
+
time_to_failure: number;
|
|
63
|
+
}>;
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
export interface ComparisonScenario {
|
|
67
|
+
issue: string;
|
|
68
|
+
interaction_id: string;
|
|
69
|
+
tool: string;
|
|
70
|
+
models: ModelResponse[];
|
|
71
|
+
}
|
|
72
|
+
export declare class DatasetAnalyzer {
|
|
73
|
+
private datasetDir;
|
|
74
|
+
constructor(datasetDir?: string);
|
|
75
|
+
/**
|
|
76
|
+
* Find all available datasets for a specific tool
|
|
77
|
+
*/
|
|
78
|
+
findDatasets(tool: string): string[];
|
|
79
|
+
/**
|
|
80
|
+
* Parse dataset filename to extract components
|
|
81
|
+
* Format: {tool}_{interaction_id}_{sdk}_{model}_{timestamp}.jsonl
|
|
82
|
+
*/
|
|
83
|
+
parseDatasetFilename(filename: string): {
|
|
84
|
+
tool: string;
|
|
85
|
+
interaction_id: string;
|
|
86
|
+
sdk: string;
|
|
87
|
+
model: string;
|
|
88
|
+
timestamp: string;
|
|
89
|
+
} | null;
|
|
90
|
+
/**
|
|
91
|
+
* Load and parse a dataset file
|
|
92
|
+
*/
|
|
93
|
+
loadDataset(filepath: string): DatasetSample | null;
|
|
94
|
+
/**
|
|
95
|
+
* Group datasets by scenario for comparative evaluation
|
|
96
|
+
* Returns scenarios that have data from multiple models
|
|
97
|
+
* Groups by both tool and interaction_id to create separate evaluations for each phase
|
|
98
|
+
*/
|
|
99
|
+
groupByScenario(tool: string): ComparisonScenario[];
|
|
100
|
+
/**
|
|
101
|
+
* Combine multiple interactions per model into a single response for evaluation
|
|
102
|
+
*/
|
|
103
|
+
private combineModelInteractions;
|
|
104
|
+
/**
|
|
105
|
+
* Get summary of available models across all scenarios for a tool
|
|
106
|
+
*/
|
|
107
|
+
getAvailableModels(tool: string): string[];
|
|
108
|
+
/**
|
|
109
|
+
* Get statistics about dataset availability
|
|
110
|
+
*/
|
|
111
|
+
getDatasetStats(tool: string): {
|
|
112
|
+
totalDatasets: number;
|
|
113
|
+
availableModels: string[];
|
|
114
|
+
scenariosWithMultipleModels: number;
|
|
115
|
+
interactionTypes: string[];
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
//# sourceMappingURL=dataset-analyzer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dataset-analyzer.d.ts","sourceRoot":"","sources":["../../src/evaluation/dataset-analyzer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;KACpB,CAAC;IACF,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE;QACX,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,GAAG,EAAE,MAAM,CAAC;QACZ,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,QAAQ,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,EAAE,MAAM,CAAC;QACb,aAAa,EAAE,MAAM,CAAC;QACtB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE;QACX,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,mBAAmB,CAAC,EAAE,MAAM,CAAC;QAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;KAChC,CAAC;IACF,QAAQ,EAAE;QACR,SAAS,EAAE,MAAM,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,gBAAgB,CAAC,EAAE;YACjB,YAAY,EAAE,MAAM,CAAC;YACrB,cAAc,EAAE,MAAM,CAAC;YACvB,eAAe,EAAE,MAAM,CAAC;SACzB,CAAC;QACF,YAAY,CAAC,EAAE,KAAK,CAAC;YACnB,YAAY,EAAE,MAAM,CAAC;YACrB,cAAc,EAAE,MAAM,CAAC;YACvB,eAAe,EAAE,MAAM,CAAC;SACzB,CAAC,CAAC;KACJ,CAAC;CACH;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,aAAa,EAAE,CAAC;CACzB;AAED,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAS;gBAEf,UAAU,GAAE,MAA0B;IAIlD;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAOpC;;;OAGG;IACH,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG;QACtC,IAAI,EAAE,MAAM,CAAC;QACb,cAAc,EAAE,MAAM,CAAC;QACvB,GAAG,EAAE,MAAM,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;KACnB,GAAG,IAAI;IAuBR;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAYnD;;;;OAIG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,kBAAkB,EAAE;IA2FnD;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAqDhC;;OAEG;IACH,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAc1C;;OAEG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG;QAC7B,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,2BAA2B,EAAE,MAAM,CAAC;QACpC,gBAAgB,EAAE,MAAM,EAAE,CAAC;KAC5B;CAmBF"}
|