@jagilber-org/index-server 1.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1218 -0
- package/CODE_OF_CONDUCT.md +49 -0
- package/CONTRIBUTING.md +75 -0
- package/LICENSE +21 -0
- package/README.md +523 -0
- package/SECURITY.md +50 -0
- package/dist/config/configUtils.d.ts +11 -0
- package/dist/config/configUtils.js +87 -0
- package/dist/config/dashboardConfig.d.ts +45 -0
- package/dist/config/dashboardConfig.js +63 -0
- package/dist/config/defaultValues.d.ts +61 -0
- package/dist/config/defaultValues.js +70 -0
- package/dist/config/dirConstants.d.ts +17 -0
- package/dist/config/dirConstants.js +28 -0
- package/dist/config/featureConfig.d.ts +61 -0
- package/dist/config/featureConfig.js +121 -0
- package/dist/config/runtimeConfig.d.ts +145 -0
- package/dist/config/runtimeConfig.js +334 -0
- package/dist/config/serverConfig.d.ts +90 -0
- package/dist/config/serverConfig.js +164 -0
- package/dist/dashboard/analytics/AnalyticsEngine.d.ts +142 -0
- package/dist/dashboard/analytics/AnalyticsEngine.js +373 -0
- package/dist/dashboard/analytics/BusinessIntelligence.d.ts +187 -0
- package/dist/dashboard/analytics/BusinessIntelligence.js +594 -0
- package/dist/dashboard/client/admin.html +2150 -0
- package/dist/dashboard/client/chunks/mermaid-layout-elk.esm.min/chunk-SP2CHFBE.mjs +1 -0
- package/dist/dashboard/client/chunks/mermaid-layout-elk.esm.min/render-T6MDALS3.mjs +27 -0
- package/dist/dashboard/client/css/admin.css +1466 -0
- package/dist/dashboard/client/js/admin.boot.js +359 -0
- package/dist/dashboard/client/js/admin.config.js +196 -0
- package/dist/dashboard/client/js/admin.embeddings.js +425 -0
- package/dist/dashboard/client/js/admin.graph.js +583 -0
- package/dist/dashboard/client/js/admin.instances.js +120 -0
- package/dist/dashboard/client/js/admin.instructions.js +552 -0
- package/dist/dashboard/client/js/admin.logs.js +113 -0
- package/dist/dashboard/client/js/admin.maintenance.js +354 -0
- package/dist/dashboard/client/js/admin.messaging.js +635 -0
- package/dist/dashboard/client/js/admin.monitor.js +181 -0
- package/dist/dashboard/client/js/admin.overview.js +221 -0
- package/dist/dashboard/client/js/admin.performance.js +61 -0
- package/dist/dashboard/client/js/admin.sessions.js +293 -0
- package/dist/dashboard/client/js/admin.sqlite.js +366 -0
- package/dist/dashboard/client/js/admin.utils.js +49 -0
- package/dist/dashboard/client/js/chart.umd.js +14 -0
- package/dist/dashboard/client/js/elk.bundled.js +6696 -0
- package/dist/dashboard/client/js/marked.umd.js +74 -0
- package/dist/dashboard/client/js/mermaid.min.js +3022 -0
- package/dist/dashboard/client/mermaid-layout-elk.esm.min.mjs +1 -0
- package/dist/dashboard/export/DataExporter.d.ts +169 -0
- package/dist/dashboard/export/DataExporter.js +737 -0
- package/dist/dashboard/export/exporters/csvExporter.d.ts +11 -0
- package/dist/dashboard/export/exporters/csvExporter.js +46 -0
- package/dist/dashboard/export/exporters/exportTypes.d.ts +89 -0
- package/dist/dashboard/export/exporters/exportTypes.js +5 -0
- package/dist/dashboard/export/exporters/jsonExporter.d.ts +7 -0
- package/dist/dashboard/export/exporters/jsonExporter.js +22 -0
- package/dist/dashboard/export/exporters/xmlExporter.d.ts +17 -0
- package/dist/dashboard/export/exporters/xmlExporter.js +175 -0
- package/dist/dashboard/integration/APIIntegration.d.ts +41 -0
- package/dist/dashboard/integration/APIIntegration.js +95 -0
- package/dist/dashboard/security/SecurityMonitor.d.ts +167 -0
- package/dist/dashboard/security/SecurityMonitor.js +559 -0
- package/dist/dashboard/server/AdminPanel.d.ts +183 -0
- package/dist/dashboard/server/AdminPanel.js +792 -0
- package/dist/dashboard/server/AdminPanelConfig.d.ts +42 -0
- package/dist/dashboard/server/AdminPanelConfig.js +80 -0
- package/dist/dashboard/server/AdminPanelState.d.ts +47 -0
- package/dist/dashboard/server/AdminPanelState.js +214 -0
- package/dist/dashboard/server/ApiRoutes.d.ts +17 -0
- package/dist/dashboard/server/ApiRoutes.js +149 -0
- package/dist/dashboard/server/DashboardServer.d.ts +49 -0
- package/dist/dashboard/server/DashboardServer.js +159 -0
- package/dist/dashboard/server/FileMetricsStorage.d.ts +49 -0
- package/dist/dashboard/server/FileMetricsStorage.js +195 -0
- package/dist/dashboard/server/HttpTransport.d.ts +23 -0
- package/dist/dashboard/server/HttpTransport.js +116 -0
- package/dist/dashboard/server/InstanceManager.d.ts +53 -0
- package/dist/dashboard/server/InstanceManager.js +284 -0
- package/dist/dashboard/server/KnowledgeStore.d.ts +35 -0
- package/dist/dashboard/server/KnowledgeStore.js +105 -0
- package/dist/dashboard/server/LeaderElection.d.ts +81 -0
- package/dist/dashboard/server/LeaderElection.js +268 -0
- package/dist/dashboard/server/MetricsCollector.d.ts +200 -0
- package/dist/dashboard/server/MetricsCollector.js +803 -0
- package/dist/dashboard/server/SessionPersistenceManager.d.ts +88 -0
- package/dist/dashboard/server/SessionPersistenceManager.js +457 -0
- package/dist/dashboard/server/ThinClient.d.ts +64 -0
- package/dist/dashboard/server/ThinClient.js +237 -0
- package/dist/dashboard/server/WebSocketManager.d.ts +161 -0
- package/dist/dashboard/server/WebSocketManager.js +463 -0
- package/dist/dashboard/server/httpLifecycle.d.ts +17 -0
- package/dist/dashboard/server/httpLifecycle.js +35 -0
- package/dist/dashboard/server/legacyDashboardHtml.d.ts +9 -0
- package/dist/dashboard/server/legacyDashboardHtml.js +618 -0
- package/dist/dashboard/server/legacyDashboardStyles.d.ts +5 -0
- package/dist/dashboard/server/legacyDashboardStyles.js +490 -0
- package/dist/dashboard/server/metricsAggregation.d.ts +252 -0
- package/dist/dashboard/server/metricsAggregation.js +206 -0
- package/dist/dashboard/server/metricsSerializer.d.ts +25 -0
- package/dist/dashboard/server/metricsSerializer.js +195 -0
- package/dist/dashboard/server/routes/admin.routes.d.ts +16 -0
- package/dist/dashboard/server/routes/admin.routes.js +596 -0
- package/dist/dashboard/server/routes/alerts.routes.d.ts +7 -0
- package/dist/dashboard/server/routes/alerts.routes.js +93 -0
- package/dist/dashboard/server/routes/api.feedback.routes.d.ts +73 -0
- package/dist/dashboard/server/routes/api.feedback.routes.js +171 -0
- package/dist/dashboard/server/routes/api.instructions.routes.d.ts +101 -0
- package/dist/dashboard/server/routes/api.instructions.routes.js +213 -0
- package/dist/dashboard/server/routes/api.usage.routes.d.ts +57 -0
- package/dist/dashboard/server/routes/api.usage.routes.js +374 -0
- package/dist/dashboard/server/routes/embeddings.routes.d.ts +6 -0
- package/dist/dashboard/server/routes/embeddings.routes.js +246 -0
- package/dist/dashboard/server/routes/graph.routes.d.ts +6 -0
- package/dist/dashboard/server/routes/graph.routes.js +280 -0
- package/dist/dashboard/server/routes/index.d.ts +38 -0
- package/dist/dashboard/server/routes/index.js +194 -0
- package/dist/dashboard/server/routes/instances.routes.d.ts +6 -0
- package/dist/dashboard/server/routes/instances.routes.js +35 -0
- package/dist/dashboard/server/routes/instructions.routes.d.ts +8 -0
- package/dist/dashboard/server/routes/instructions.routes.js +336 -0
- package/dist/dashboard/server/routes/knowledge.routes.d.ts +6 -0
- package/dist/dashboard/server/routes/knowledge.routes.js +82 -0
- package/dist/dashboard/server/routes/logs.routes.d.ts +6 -0
- package/dist/dashboard/server/routes/logs.routes.js +164 -0
- package/dist/dashboard/server/routes/messaging.routes.d.ts +16 -0
- package/dist/dashboard/server/routes/messaging.routes.js +293 -0
- package/dist/dashboard/server/routes/metrics.routes.d.ts +10 -0
- package/dist/dashboard/server/routes/metrics.routes.js +346 -0
- package/dist/dashboard/server/routes/scripts.routes.d.ts +9 -0
- package/dist/dashboard/server/routes/scripts.routes.js +84 -0
- package/dist/dashboard/server/routes/sqlite.routes.d.ts +9 -0
- package/dist/dashboard/server/routes/sqlite.routes.js +569 -0
- package/dist/dashboard/server/routes/status.routes.d.ts +7 -0
- package/dist/dashboard/server/routes/status.routes.js +183 -0
- package/dist/dashboard/server/routes/synthetic.routes.d.ts +7 -0
- package/dist/dashboard/server/routes/synthetic.routes.js +195 -0
- package/dist/dashboard/server/routes/tools.routes.d.ts +6 -0
- package/dist/dashboard/server/routes/tools.routes.js +46 -0
- package/dist/dashboard/server/routes/usage.routes.d.ts +6 -0
- package/dist/dashboard/server/routes/usage.routes.js +25 -0
- package/dist/dashboard/server/wsInit.d.ts +16 -0
- package/dist/dashboard/server/wsInit.js +35 -0
- package/dist/externalClientLib.d.ts +1 -0
- package/dist/externalClientLib.js +2 -0
- package/dist/minimal/index.d.ts +1 -0
- package/dist/minimal/index.js +140 -0
- package/dist/models/SessionPersistence.d.ts +115 -0
- package/dist/models/SessionPersistence.js +66 -0
- package/dist/models/instruction.d.ts +45 -0
- package/dist/models/instruction.js +2 -0
- package/dist/perf/benchmark.d.ts +1 -0
- package/dist/perf/benchmark.js +50 -0
- package/dist/portableClientWrapper.d.ts +1 -0
- package/dist/portableClientWrapper.js +2 -0
- package/dist/schemas/index.d.ts +128 -0
- package/dist/schemas/index.js +371 -0
- package/dist/scripts/runPerformanceBaseline.d.ts +1 -0
- package/dist/scripts/runPerformanceBaseline.js +17 -0
- package/dist/server/handshakeManager.d.ts +25 -0
- package/dist/server/handshakeManager.js +472 -0
- package/dist/server/index-server.d.ts +56 -0
- package/dist/server/index-server.js +822 -0
- package/dist/server/registry.d.ts +44 -0
- package/dist/server/registry.js +236 -0
- package/dist/server/sdkServer.d.ts +8 -0
- package/dist/server/sdkServer.js +299 -0
- package/dist/server/shutdownGuard.d.ts +41 -0
- package/dist/server/shutdownGuard.js +52 -0
- package/dist/server/thin-client.d.ts +22 -0
- package/dist/server/thin-client.js +111 -0
- package/dist/server/transport.d.ts +41 -0
- package/dist/server/transport.js +312 -0
- package/dist/server/transportFactory.d.ts +21 -0
- package/dist/server/transportFactory.js +429 -0
- package/dist/services/atomicFs.d.ts +22 -0
- package/dist/services/atomicFs.js +103 -0
- package/dist/services/auditLog.d.ts +38 -0
- package/dist/services/auditLog.js +142 -0
- package/dist/services/autoBackup.d.ts +14 -0
- package/dist/services/autoBackup.js +171 -0
- package/dist/services/autoSplit.d.ts +32 -0
- package/dist/services/autoSplit.js +113 -0
- package/dist/services/backupZip.d.ts +25 -0
- package/dist/services/backupZip.js +110 -0
- package/dist/services/bootstrapGating.d.ts +123 -0
- package/dist/services/bootstrapGating.js +221 -0
- package/dist/services/canonical.d.ts +23 -0
- package/dist/services/canonical.js +65 -0
- package/dist/services/categoryRules.d.ts +7 -0
- package/dist/services/categoryRules.js +37 -0
- package/dist/services/classificationService.d.ts +42 -0
- package/dist/services/classificationService.js +168 -0
- package/dist/services/embeddingService.d.ts +62 -0
- package/dist/services/embeddingService.js +259 -0
- package/dist/services/errors.d.ts +22 -0
- package/dist/services/errors.js +31 -0
- package/dist/services/featureFlags.d.ts +25 -0
- package/dist/services/featureFlags.js +89 -0
- package/dist/services/features.d.ts +13 -0
- package/dist/services/features.js +35 -0
- package/dist/services/handlers/instructions.add.d.ts +1 -0
- package/dist/services/handlers/instructions.add.js +496 -0
- package/dist/services/handlers/instructions.groom.d.ts +1 -0
- package/dist/services/handlers/instructions.groom.js +523 -0
- package/dist/services/handlers/instructions.import.d.ts +1 -0
- package/dist/services/handlers/instructions.import.js +173 -0
- package/dist/services/handlers/instructions.patch.d.ts +1 -0
- package/dist/services/handlers/instructions.patch.js +167 -0
- package/dist/services/handlers/instructions.query.d.ts +163 -0
- package/dist/services/handlers/instructions.query.js +522 -0
- package/dist/services/handlers/instructions.reload.d.ts +1 -0
- package/dist/services/handlers/instructions.reload.js +13 -0
- package/dist/services/handlers/instructions.remove.d.ts +1 -0
- package/dist/services/handlers/instructions.remove.js +118 -0
- package/dist/services/handlers/instructions.shared.d.ts +31 -0
- package/dist/services/handlers/instructions.shared.js +124 -0
- package/dist/services/handlers.activation.d.ts +1 -0
- package/dist/services/handlers.activation.js +203 -0
- package/dist/services/handlers.bootstrap.d.ts +1 -0
- package/dist/services/handlers.bootstrap.js +38 -0
- package/dist/services/handlers.dashboardConfig.d.ts +34 -0
- package/dist/services/handlers.dashboardConfig.js +108 -0
- package/dist/services/handlers.diagnostics.d.ts +1 -0
- package/dist/services/handlers.diagnostics.js +64 -0
- package/dist/services/handlers.feedback.d.ts +15 -0
- package/dist/services/handlers.feedback.js +378 -0
- package/dist/services/handlers.gates.d.ts +1 -0
- package/dist/services/handlers.gates.js +46 -0
- package/dist/services/handlers.graph.d.ts +53 -0
- package/dist/services/handlers.graph.js +231 -0
- package/dist/services/handlers.help.d.ts +1 -0
- package/dist/services/handlers.help.js +119 -0
- package/dist/services/handlers.instructionSchema.d.ts +1 -0
- package/dist/services/handlers.instructionSchema.js +227 -0
- package/dist/services/handlers.instructions.d.ts +8 -0
- package/dist/services/handlers.instructions.js +14 -0
- package/dist/services/handlers.instructionsDiagnostics.d.ts +1 -0
- package/dist/services/handlers.instructionsDiagnostics.js +14 -0
- package/dist/services/handlers.integrity.d.ts +1 -0
- package/dist/services/handlers.integrity.js +35 -0
- package/dist/services/handlers.manifest.d.ts +1 -0
- package/dist/services/handlers.manifest.js +24 -0
- package/dist/services/handlers.messaging.d.ts +12 -0
- package/dist/services/handlers.messaging.js +203 -0
- package/dist/services/handlers.metrics.d.ts +1 -0
- package/dist/services/handlers.metrics.js +43 -0
- package/dist/services/handlers.promote.d.ts +1 -0
- package/dist/services/handlers.promote.js +306 -0
- package/dist/services/handlers.prompt.d.ts +1 -0
- package/dist/services/handlers.prompt.js +7 -0
- package/dist/services/handlers.search.d.ts +69 -0
- package/dist/services/handlers.search.js +645 -0
- package/dist/services/handlers.testPrimitive.d.ts +1 -0
- package/dist/services/handlers.testPrimitive.js +5 -0
- package/dist/services/handlers.trace.d.ts +1 -0
- package/dist/services/handlers.trace.js +31 -0
- package/dist/services/handlers.usage.d.ts +1 -0
- package/dist/services/handlers.usage.js +11 -0
- package/dist/services/hotScore.d.ts +137 -0
- package/dist/services/hotScore.js +244 -0
- package/dist/services/indexContext.d.ts +117 -0
- package/dist/services/indexContext.js +968 -0
- package/dist/services/indexLoader.d.ts +44 -0
- package/dist/services/indexLoader.js +921 -0
- package/dist/services/indexRepository.d.ts +32 -0
- package/dist/services/indexRepository.js +71 -0
- package/dist/services/indexingService.d.ts +1 -0
- package/dist/services/indexingService.js +2 -0
- package/dist/services/instructions.dispatcher.d.ts +1 -0
- package/dist/services/instructions.dispatcher.js +231 -0
- package/dist/services/logPrefix.d.ts +1 -0
- package/dist/services/logPrefix.js +30 -0
- package/dist/services/logger.d.ts +52 -0
- package/dist/services/logger.js +268 -0
- package/dist/services/manifestManager.d.ts +82 -0
- package/dist/services/manifestManager.js +200 -0
- package/dist/services/messaging/agentMailbox.d.ts +60 -0
- package/dist/services/messaging/agentMailbox.js +353 -0
- package/dist/services/messaging/messagingPersistence.d.ts +20 -0
- package/dist/services/messaging/messagingPersistence.js +111 -0
- package/dist/services/messaging/messagingTypes.d.ts +150 -0
- package/dist/services/messaging/messagingTypes.js +66 -0
- package/dist/services/ownershipService.d.ts +1 -0
- package/dist/services/ownershipService.js +38 -0
- package/dist/services/performanceBaseline.d.ts +19 -0
- package/dist/services/performanceBaseline.js +210 -0
- package/dist/services/preflight.d.ts +12 -0
- package/dist/services/preflight.js +79 -0
- package/dist/services/promptReviewService.d.ts +44 -0
- package/dist/services/promptReviewService.js +101 -0
- package/dist/services/responseEnvelope.d.ts +6 -0
- package/dist/services/responseEnvelope.js +25 -0
- package/dist/services/seedBootstrap.d.ts +34 -0
- package/dist/services/seedBootstrap.js +427 -0
- package/dist/services/storage/factory.d.ts +17 -0
- package/dist/services/storage/factory.js +35 -0
- package/dist/services/storage/hashUtils.d.ts +11 -0
- package/dist/services/storage/hashUtils.js +35 -0
- package/dist/services/storage/index.d.ts +12 -0
- package/dist/services/storage/index.js +18 -0
- package/dist/services/storage/jsonFileStore.d.ts +32 -0
- package/dist/services/storage/jsonFileStore.js +241 -0
- package/dist/services/storage/migrationEngine.d.ts +35 -0
- package/dist/services/storage/migrationEngine.js +93 -0
- package/dist/services/storage/sqliteMessageStore.d.ts +53 -0
- package/dist/services/storage/sqliteMessageStore.js +146 -0
- package/dist/services/storage/sqliteSchema.d.ts +12 -0
- package/dist/services/storage/sqliteSchema.js +122 -0
- package/dist/services/storage/sqliteStore.d.ts +41 -0
- package/dist/services/storage/sqliteStore.js +339 -0
- package/dist/services/storage/sqliteUsageStore.d.ts +35 -0
- package/dist/services/storage/sqliteUsageStore.js +94 -0
- package/dist/services/storage/types.d.ts +171 -0
- package/dist/services/storage/types.js +12 -0
- package/dist/services/toolHandlers.d.ts +23 -0
- package/dist/services/toolHandlers.js +50 -0
- package/dist/services/toolRegistry.d.ts +20 -0
- package/dist/services/toolRegistry.js +490 -0
- package/dist/services/toolRegistry.zod.d.ts +10 -0
- package/dist/services/toolRegistry.zod.js +323 -0
- package/dist/services/tracing.d.ts +26 -0
- package/dist/services/tracing.js +260 -0
- package/dist/services/usageBuckets.d.ts +161 -0
- package/dist/services/usageBuckets.js +364 -0
- package/dist/services/validationService.d.ts +38 -0
- package/dist/services/validationService.js +125 -0
- package/dist/utils/BufferRing.d.ts +203 -0
- package/dist/utils/BufferRing.js +551 -0
- package/dist/utils/BufferRingExamples.d.ts +55 -0
- package/dist/utils/BufferRingExamples.js +188 -0
- package/dist/utils/envUtils.d.ts +42 -0
- package/dist/utils/envUtils.js +80 -0
- package/dist/utils/memoryMonitor.d.ts +83 -0
- package/dist/utils/memoryMonitor.js +275 -0
- package/dist/versioning/schemaVersion.d.ts +6 -0
- package/dist/versioning/schemaVersion.js +93 -0
- package/package.json +134 -0
- package/schemas/README.md +13 -0
- package/schemas/feedback-entry.schema.json +27 -0
- package/schemas/graph-export-v2.schema.json +60 -0
- package/schemas/index-server.code-schema.json +38477 -0
- package/schemas/instruction.schema.json +262 -0
- package/schemas/json-schema/SessionPersistence-persisted-admin-session.schema.json +54 -0
- package/schemas/json-schema/SessionPersistence-persisted-session-history-entry.schema.json +51 -0
- package/schemas/json-schema/SessionPersistence-persisted-web-socket-connection.schema.json +54 -0
- package/schemas/json-schema/SessionPersistence-session-persistence-config.schema.json +110 -0
- package/schemas/json-schema/SessionPersistence-session-persistence-data.schema.json +229 -0
- package/schemas/json-schema/SessionPersistence-session-persistence-manifest.schema.json +109 -0
- package/schemas/json-schema/SessionPersistence-session-persistence-metadata.schema.json +55 -0
- package/schemas/json-schema/instruction-audience-scope.schema.json +14 -0
- package/schemas/json-schema/instruction-content-type.schema.json +17 -0
- package/schemas/json-schema/instruction-instruction-entry.schema.json +206 -0
- package/schemas/json-schema/instruction-requirement-level.schema.json +16 -0
- package/schemas/manifest.json +78 -0
- package/schemas/manifest.schema.json +33 -0
- package/schemas/usage-batch.schema.json +16 -0
- package/schemas/usage-buckets.schema.json +30 -0
- package/schemas/usage-event.schema.json +17 -0
- package/scripts/copy-dashboard-assets.mjs +170 -0
- package/scripts/setup-hooks.cjs +28 -0
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* API Usage Routes — Request execution and monitoring.
|
|
3
|
+
*
|
|
4
|
+
* Manages API request execution (retry, rate-limit enforcement, auth, transforms)
|
|
5
|
+
* and the monitoring event bus. Depends on EndpointManager for endpoint config.
|
|
6
|
+
*/
|
|
7
|
+
import type { EndpointManager, APIAuthentication, RetryConfig, DataMapping } from './api.instructions.routes.js';
|
|
8
|
+
export interface APIRequest {
|
|
9
|
+
id: string;
|
|
10
|
+
endpointId: string;
|
|
11
|
+
method: string;
|
|
12
|
+
url: string;
|
|
13
|
+
headers: Record<string, string>;
|
|
14
|
+
body?: unknown;
|
|
15
|
+
timestamp: number;
|
|
16
|
+
timeout: number;
|
|
17
|
+
}
|
|
18
|
+
export interface APIResponse {
|
|
19
|
+
requestId: string;
|
|
20
|
+
status: number;
|
|
21
|
+
statusText: string;
|
|
22
|
+
headers: Record<string, string>;
|
|
23
|
+
body: unknown;
|
|
24
|
+
responseTime: number;
|
|
25
|
+
timestamp: number;
|
|
26
|
+
error?: string;
|
|
27
|
+
}
|
|
28
|
+
export interface APIMonitoringEvent {
|
|
29
|
+
type: string;
|
|
30
|
+
timestamp: number;
|
|
31
|
+
data: Record<string, unknown>;
|
|
32
|
+
}
|
|
33
|
+
export declare class UsageManager {
|
|
34
|
+
private readonly endpointMgr;
|
|
35
|
+
private readonly monitoringCallbacks;
|
|
36
|
+
constructor(endpointMgr: EndpointManager);
|
|
37
|
+
executeRequest(endpointId: string, data?: unknown, overrides?: Partial<APIRequest>): Promise<APIResponse>;
|
|
38
|
+
executeWithRetry(request: APIRequest, retryConfig: RetryConfig): Promise<APIResponse>;
|
|
39
|
+
performHTTPRequest(request: APIRequest): Promise<APIResponse>;
|
|
40
|
+
private simulateHTTPRequest;
|
|
41
|
+
private shouldRetry;
|
|
42
|
+
private shouldRetryError;
|
|
43
|
+
private calculateRetryDelay;
|
|
44
|
+
applyAuthentication(request: APIRequest, auth: APIAuthentication): void;
|
|
45
|
+
resolveConfigValue(value: string): string;
|
|
46
|
+
applySafeTransform(data: any, transformSpec: string): any;
|
|
47
|
+
applySafeFieldTransform(value: any, transform: string): any;
|
|
48
|
+
applyFieldMappings(data: unknown, mappings: DataMapping['fieldMappings']): unknown;
|
|
49
|
+
private getNestedValue;
|
|
50
|
+
private setNestedValue;
|
|
51
|
+
private validateData;
|
|
52
|
+
private collectMetrics;
|
|
53
|
+
logAPIEvent(type: string, data: Record<string, unknown>): void;
|
|
54
|
+
onAPIEvent(callback: (event: APIMonitoringEvent) => void): void;
|
|
55
|
+
processRequestQueues(): void;
|
|
56
|
+
sleep(ms: number): Promise<void>;
|
|
57
|
+
}
|
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* API Usage Routes — Request execution and monitoring.
|
|
4
|
+
*
|
|
5
|
+
* Manages API request execution (retry, rate-limit enforcement, auth, transforms)
|
|
6
|
+
* and the monitoring event bus. Depends on EndpointManager for endpoint config.
|
|
7
|
+
*/
|
|
8
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
+
exports.UsageManager = void 0;
|
|
10
|
+
// ── UsageManager ─────────────────────────────────────────────────────────────────────
|
|
11
|
+
class UsageManager {
|
|
12
|
+
endpointMgr;
|
|
13
|
+
monitoringCallbacks = [];
|
|
14
|
+
constructor(endpointMgr) {
|
|
15
|
+
this.endpointMgr = endpointMgr;
|
|
16
|
+
}
|
|
17
|
+
async executeRequest(endpointId, data, overrides) {
|
|
18
|
+
const endpoint = this.endpointMgr.getEndpoint(endpointId);
|
|
19
|
+
if (!endpoint)
|
|
20
|
+
throw new Error(`Endpoint not found: ${endpointId}`);
|
|
21
|
+
if (!this.endpointMgr.checkRateLimit(endpointId)) {
|
|
22
|
+
throw new Error(`Rate limit exceeded for endpoint: ${endpointId}`);
|
|
23
|
+
}
|
|
24
|
+
const requestId = `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
|
|
25
|
+
const request = {
|
|
26
|
+
id: requestId,
|
|
27
|
+
endpointId,
|
|
28
|
+
method: endpoint.method,
|
|
29
|
+
url: endpoint.url,
|
|
30
|
+
headers: { ...endpoint.headers, ...(overrides?.headers || {}) },
|
|
31
|
+
body: data,
|
|
32
|
+
timestamp: Date.now(),
|
|
33
|
+
timeout: overrides?.timeout || endpoint.timeout
|
|
34
|
+
};
|
|
35
|
+
this.applyAuthentication(request, endpoint.authentication);
|
|
36
|
+
if (data && endpoint.dataMapping.requestTransform) {
|
|
37
|
+
try {
|
|
38
|
+
request.body = this.applySafeTransform(data, endpoint.dataMapping.requestTransform);
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
console.error('Request transform error:', error);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
if (endpoint.validation.validateRequest && endpoint.validation.requestSchema) {
|
|
45
|
+
const valid = this.validateData(request.body, endpoint.validation.requestSchema);
|
|
46
|
+
if (!valid && endpoint.validation.strictMode) {
|
|
47
|
+
throw new Error('Request validation failed');
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
if (endpoint.monitoring.logRequests) {
|
|
51
|
+
this.logAPIEvent('request', { endpointId, requestId, url: request.url, method: request.method });
|
|
52
|
+
}
|
|
53
|
+
const response = await this.executeWithRetry(request, endpoint.retryConfig);
|
|
54
|
+
if (endpoint.dataMapping.responseTransform) {
|
|
55
|
+
try {
|
|
56
|
+
response.body = this.applySafeTransform(response.body, endpoint.dataMapping.responseTransform);
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
console.error('Response transform error:', error);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
if (endpoint.dataMapping.fieldMappings.length > 0) {
|
|
63
|
+
response.body = this.applyFieldMappings(response.body, endpoint.dataMapping.fieldMappings);
|
|
64
|
+
}
|
|
65
|
+
if (endpoint.validation.validateResponse && endpoint.validation.responseSchema) {
|
|
66
|
+
const valid = this.validateData(response.body, endpoint.validation.responseSchema);
|
|
67
|
+
if (!valid && endpoint.validation.strictMode) {
|
|
68
|
+
console.warn('Response validation failed for endpoint:', endpointId);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (endpoint.monitoring.logResponses) {
|
|
72
|
+
this.logAPIEvent('response', {
|
|
73
|
+
endpointId,
|
|
74
|
+
requestId,
|
|
75
|
+
status: response.status,
|
|
76
|
+
responseTime: response.responseTime
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
if (response.error && endpoint.monitoring.logErrors) {
|
|
80
|
+
this.logAPIEvent('error', { endpointId, requestId, error: response.error, status: response.status });
|
|
81
|
+
}
|
|
82
|
+
if (endpoint.monitoring.collectMetrics) {
|
|
83
|
+
this.collectMetrics(endpointId, response);
|
|
84
|
+
}
|
|
85
|
+
return response;
|
|
86
|
+
}
|
|
87
|
+
async executeWithRetry(request, retryConfig) {
|
|
88
|
+
let lastError = null;
|
|
89
|
+
let attempt = 0;
|
|
90
|
+
while (attempt <= retryConfig.maxAttempts) {
|
|
91
|
+
try {
|
|
92
|
+
const response = await this.performHTTPRequest(request);
|
|
93
|
+
if (this.shouldRetry(response, retryConfig)) {
|
|
94
|
+
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
95
|
+
}
|
|
96
|
+
return response;
|
|
97
|
+
}
|
|
98
|
+
catch (error) {
|
|
99
|
+
lastError = error instanceof Error ? error : new Error(String(error));
|
|
100
|
+
attempt++;
|
|
101
|
+
if (attempt > retryConfig.maxAttempts)
|
|
102
|
+
break;
|
|
103
|
+
if (!this.shouldRetryError(lastError, retryConfig))
|
|
104
|
+
break;
|
|
105
|
+
const delay = this.calculateRetryDelay(attempt, retryConfig);
|
|
106
|
+
await this.sleep(delay);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
requestId: request.id,
|
|
111
|
+
status: 0,
|
|
112
|
+
statusText: 'Request Failed',
|
|
113
|
+
headers: {},
|
|
114
|
+
body: null,
|
|
115
|
+
responseTime: 0,
|
|
116
|
+
timestamp: Date.now(),
|
|
117
|
+
error: lastError?.message || 'Unknown error'
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
async performHTTPRequest(request) {
|
|
121
|
+
const startTime = Date.now();
|
|
122
|
+
try {
|
|
123
|
+
const response = await this.simulateHTTPRequest(request);
|
|
124
|
+
return {
|
|
125
|
+
requestId: request.id,
|
|
126
|
+
status: response.status,
|
|
127
|
+
statusText: response.statusText,
|
|
128
|
+
headers: response.headers,
|
|
129
|
+
body: response.body,
|
|
130
|
+
responseTime: Date.now() - startTime,
|
|
131
|
+
timestamp: Date.now()
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
throw new Error(`Network error: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
async simulateHTTPRequest(request) {
|
|
139
|
+
await this.sleep(Math.random() * 100 + 50);
|
|
140
|
+
const scenarios = [
|
|
141
|
+
{ weight: 0.8, status: 200, statusText: 'OK', body: { success: true, data: 'Response data' } },
|
|
142
|
+
{ weight: 0.1, status: 429, statusText: 'Too Many Requests', body: { error: 'Rate limit exceeded' } },
|
|
143
|
+
{ weight: 0.05, status: 500, statusText: 'Internal Server Error', body: { error: 'Server error' } },
|
|
144
|
+
{ weight: 0.05, status: 503, statusText: 'Service Unavailable', body: { error: 'Service unavailable' } }
|
|
145
|
+
];
|
|
146
|
+
const random = Math.random();
|
|
147
|
+
let cumulative = 0;
|
|
148
|
+
for (const scenario of scenarios) {
|
|
149
|
+
cumulative += scenario.weight;
|
|
150
|
+
if (random <= cumulative) {
|
|
151
|
+
return { status: scenario.status, statusText: scenario.statusText, headers: { 'Content-Type': 'application/json' }, body: scenario.body };
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return { status: 200, statusText: 'OK', headers: { 'Content-Type': 'application/json' }, body: { success: true, data: request.body } };
|
|
155
|
+
}
|
|
156
|
+
shouldRetry(response, retryConfig) {
|
|
157
|
+
if (!retryConfig.enabled)
|
|
158
|
+
return false;
|
|
159
|
+
if (response.status >= 500 || response.status === 429) {
|
|
160
|
+
return retryConfig.retryOn.includes('server_error') || retryConfig.retryOn.includes('rate_limit');
|
|
161
|
+
}
|
|
162
|
+
return false;
|
|
163
|
+
}
|
|
164
|
+
shouldRetryError(error, retryConfig) {
|
|
165
|
+
if (!retryConfig.enabled)
|
|
166
|
+
return false;
|
|
167
|
+
const message = error.message.toLowerCase();
|
|
168
|
+
if (message.includes('timeout') && retryConfig.retryOn.includes('timeout'))
|
|
169
|
+
return true;
|
|
170
|
+
if ((message.includes('network') || message.includes('fetch')) && retryConfig.retryOn.includes('network_error'))
|
|
171
|
+
return true;
|
|
172
|
+
return false;
|
|
173
|
+
}
|
|
174
|
+
calculateRetryDelay(attempt, retryConfig) {
|
|
175
|
+
let delay;
|
|
176
|
+
switch (retryConfig.backoffStrategy) {
|
|
177
|
+
case 'linear':
|
|
178
|
+
delay = retryConfig.baseDelay * attempt;
|
|
179
|
+
break;
|
|
180
|
+
case 'exponential':
|
|
181
|
+
delay = retryConfig.baseDelay * Math.pow(2, attempt - 1);
|
|
182
|
+
break;
|
|
183
|
+
case 'fixed':
|
|
184
|
+
default:
|
|
185
|
+
delay = retryConfig.baseDelay;
|
|
186
|
+
break;
|
|
187
|
+
}
|
|
188
|
+
return Math.min(delay, retryConfig.maxDelay);
|
|
189
|
+
}
|
|
190
|
+
applyAuthentication(request, auth) {
|
|
191
|
+
switch (auth.type) {
|
|
192
|
+
case 'api_key': {
|
|
193
|
+
const keyName = auth.config.keyName;
|
|
194
|
+
const keyValue = this.resolveConfigValue(auth.config.keyValue);
|
|
195
|
+
request.headers[keyName] = keyValue;
|
|
196
|
+
break;
|
|
197
|
+
}
|
|
198
|
+
case 'bearer_token': {
|
|
199
|
+
const token = this.resolveConfigValue(auth.config.token);
|
|
200
|
+
request.headers['Authorization'] = `Bearer ${token}`;
|
|
201
|
+
break;
|
|
202
|
+
}
|
|
203
|
+
case 'basic_auth': {
|
|
204
|
+
const username = this.resolveConfigValue(auth.config.username);
|
|
205
|
+
const password = this.resolveConfigValue(auth.config.password);
|
|
206
|
+
const credentials = Buffer.from(`${username}:${password}`).toString('base64');
|
|
207
|
+
request.headers['Authorization'] = `Basic ${credentials}`;
|
|
208
|
+
break;
|
|
209
|
+
}
|
|
210
|
+
case 'oauth2':
|
|
211
|
+
console.warn('OAuth2 authentication not fully implemented');
|
|
212
|
+
break;
|
|
213
|
+
case 'custom': {
|
|
214
|
+
const customHeaders = auth.config.headers;
|
|
215
|
+
Object.assign(request.headers, customHeaders);
|
|
216
|
+
break;
|
|
217
|
+
}
|
|
218
|
+
case 'none':
|
|
219
|
+
default:
|
|
220
|
+
break;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
resolveConfigValue(value) {
|
|
224
|
+
if (value.startsWith('${') && value.endsWith('}')) {
|
|
225
|
+
const envVar = value.slice(2, -1);
|
|
226
|
+
return process.env[envVar] || value;
|
|
227
|
+
}
|
|
228
|
+
return value;
|
|
229
|
+
}
|
|
230
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
231
|
+
applySafeTransform(data, transformSpec) {
|
|
232
|
+
if (!data || !transformSpec)
|
|
233
|
+
return data;
|
|
234
|
+
const dotPath = transformSpec.trim().match(/^(?:data|response|value)\.(.+)$/);
|
|
235
|
+
if (dotPath) {
|
|
236
|
+
const parts = dotPath[1].split('.');
|
|
237
|
+
let result = data;
|
|
238
|
+
for (const part of parts) {
|
|
239
|
+
if (result == null || typeof result !== 'object')
|
|
240
|
+
return data;
|
|
241
|
+
result = result[part];
|
|
242
|
+
}
|
|
243
|
+
return result;
|
|
244
|
+
}
|
|
245
|
+
return data;
|
|
246
|
+
}
|
|
247
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
248
|
+
applySafeFieldTransform(value, transform) {
|
|
249
|
+
const t = transform.trim();
|
|
250
|
+
if (t === 'String(value)' || t === 'value.toString()')
|
|
251
|
+
return String(value);
|
|
252
|
+
if (t === 'Number(value)' || t === 'parseInt(value)')
|
|
253
|
+
return Number(value);
|
|
254
|
+
if (t === 'Boolean(value)')
|
|
255
|
+
return Boolean(value);
|
|
256
|
+
if (t === 'JSON.parse(value)') {
|
|
257
|
+
try {
|
|
258
|
+
return JSON.parse(value);
|
|
259
|
+
}
|
|
260
|
+
catch {
|
|
261
|
+
return value;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
if (t === 'JSON.stringify(value)')
|
|
265
|
+
return JSON.stringify(value);
|
|
266
|
+
if (t === 'value.toLowerCase()')
|
|
267
|
+
return typeof value === 'string' ? value.toLowerCase() : value;
|
|
268
|
+
if (t === 'value.toUpperCase()')
|
|
269
|
+
return typeof value === 'string' ? value.toUpperCase() : value;
|
|
270
|
+
if (t === 'value.trim()')
|
|
271
|
+
return typeof value === 'string' ? value.trim() : value;
|
|
272
|
+
const dotPath = t.match(/^value\.(.+)$/);
|
|
273
|
+
if (dotPath) {
|
|
274
|
+
let result = value;
|
|
275
|
+
for (const part of dotPath[1].split('.')) {
|
|
276
|
+
if (result == null || typeof result !== 'object')
|
|
277
|
+
return value;
|
|
278
|
+
result = result[part];
|
|
279
|
+
}
|
|
280
|
+
return result;
|
|
281
|
+
}
|
|
282
|
+
return value;
|
|
283
|
+
}
|
|
284
|
+
applyFieldMappings(data, mappings) {
|
|
285
|
+
if (!data || typeof data !== 'object' || !Array.isArray(mappings))
|
|
286
|
+
return data;
|
|
287
|
+
const result = { ...data };
|
|
288
|
+
mappings.forEach(mapping => {
|
|
289
|
+
const sourceValue = this.getNestedValue(result, mapping.source);
|
|
290
|
+
let targetValue = sourceValue;
|
|
291
|
+
if (mapping.transform) {
|
|
292
|
+
try {
|
|
293
|
+
targetValue = this.applySafeFieldTransform(sourceValue, mapping.transform);
|
|
294
|
+
}
|
|
295
|
+
catch (error) {
|
|
296
|
+
console.error('Field mapping transform error:', error);
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
this.setNestedValue(result, mapping.target, targetValue);
|
|
300
|
+
});
|
|
301
|
+
return result;
|
|
302
|
+
}
|
|
303
|
+
getNestedValue(obj, path) {
|
|
304
|
+
return path.split('.').reduce((current, key) => {
|
|
305
|
+
return current && typeof current === 'object' ? current[key] : undefined;
|
|
306
|
+
}, obj);
|
|
307
|
+
}
|
|
308
|
+
setNestedValue(obj, path, value) {
|
|
309
|
+
const keys = path.split('.');
|
|
310
|
+
const lastKey = keys.pop();
|
|
311
|
+
const target = keys.reduce((current, key) => {
|
|
312
|
+
if (!current[key] || typeof current[key] !== 'object')
|
|
313
|
+
current[key] = {};
|
|
314
|
+
return current[key];
|
|
315
|
+
}, obj);
|
|
316
|
+
target[lastKey] = value;
|
|
317
|
+
}
|
|
318
|
+
validateData(data, _schema) {
|
|
319
|
+
try {
|
|
320
|
+
if (typeof data !== 'object' || data === null)
|
|
321
|
+
return false;
|
|
322
|
+
return true;
|
|
323
|
+
}
|
|
324
|
+
catch {
|
|
325
|
+
return false;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
collectMetrics(endpointId, response) {
|
|
329
|
+
this.logAPIEvent('metrics', {
|
|
330
|
+
endpointId,
|
|
331
|
+
responseTime: response.responseTime,
|
|
332
|
+
status: response.status,
|
|
333
|
+
success: response.status >= 200 && response.status < 300
|
|
334
|
+
});
|
|
335
|
+
}
|
|
336
|
+
logAPIEvent(type, data) {
|
|
337
|
+
const event = { type, timestamp: Date.now(), data };
|
|
338
|
+
this.monitoringCallbacks.forEach(callback => {
|
|
339
|
+
try {
|
|
340
|
+
callback(event);
|
|
341
|
+
}
|
|
342
|
+
catch (error) {
|
|
343
|
+
console.error('API monitoring callback error:', error);
|
|
344
|
+
}
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
onAPIEvent(callback) {
|
|
348
|
+
this.monitoringCallbacks.push(callback);
|
|
349
|
+
}
|
|
350
|
+
processRequestQueues() {
|
|
351
|
+
this.endpointMgr.requestQueue.forEach((requests, endpointId) => {
|
|
352
|
+
if (requests.length === 0)
|
|
353
|
+
return;
|
|
354
|
+
const endpoint = this.endpointMgr.getEndpoint(endpointId);
|
|
355
|
+
if (!endpoint?.rateLimit.enabled)
|
|
356
|
+
return;
|
|
357
|
+
if (this.endpointMgr.checkRateLimit(endpointId)) {
|
|
358
|
+
const request = requests.shift();
|
|
359
|
+
if (request) {
|
|
360
|
+
this.executeRequest(endpointId, request.body, {
|
|
361
|
+
headers: request.headers,
|
|
362
|
+
timeout: request.timeout
|
|
363
|
+
}).catch(error => {
|
|
364
|
+
console.error('Queued request execution error:', error);
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
sleep(ms) {
|
|
371
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
exports.UsageManager = UsageManager;
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Embeddings Routes
|
|
4
|
+
* Route: GET /embeddings/projection — PCA-project embeddings to 2D for visualization.
|
|
5
|
+
*/
|
|
6
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
7
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
8
|
+
};
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.createEmbeddingsRoutes = createEmbeddingsRoutes;
|
|
11
|
+
const express_1 = require("express");
|
|
12
|
+
const node_fs_1 = __importDefault(require("node:fs"));
|
|
13
|
+
const runtimeConfig_js_1 = require("../../../config/runtimeConfig.js");
|
|
14
|
+
const embeddingService_js_1 = require("../../../services/embeddingService.js");
|
|
15
|
+
const indexContext_js_1 = require("../../../services/indexContext.js");
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
// Category derivation from instruction ID (shared rules)
|
|
18
|
+
// ---------------------------------------------------------------------------
|
|
19
|
+
const categoryRules_js_1 = require("../../../services/categoryRules.js");
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
// PCA via power iteration (no dependencies)
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
function mean(vectors) {
|
|
24
|
+
const n = vectors.length;
|
|
25
|
+
const d = vectors[0].length;
|
|
26
|
+
const m = new Array(d).fill(0);
|
|
27
|
+
for (const v of vectors)
|
|
28
|
+
for (let i = 0; i < d; i++)
|
|
29
|
+
m[i] += v[i];
|
|
30
|
+
for (let i = 0; i < d; i++)
|
|
31
|
+
m[i] /= n;
|
|
32
|
+
return m;
|
|
33
|
+
}
|
|
34
|
+
function subtract(a, b) {
|
|
35
|
+
return a.map((v, i) => v - b[i]);
|
|
36
|
+
}
|
|
37
|
+
function dot(a, b) {
|
|
38
|
+
let s = 0;
|
|
39
|
+
for (let i = 0; i < a.length; i++)
|
|
40
|
+
s += a[i] * b[i];
|
|
41
|
+
return s;
|
|
42
|
+
}
|
|
43
|
+
function norm(v) {
|
|
44
|
+
return Math.sqrt(dot(v, v));
|
|
45
|
+
}
|
|
46
|
+
function scale(v, s) {
|
|
47
|
+
return v.map(x => x * s);
|
|
48
|
+
}
|
|
49
|
+
/** Compute the top principal component via power iteration. */
|
|
50
|
+
function powerIteration(centered, dims, iterations = 100) {
|
|
51
|
+
// random init
|
|
52
|
+
let pc = Array.from({ length: dims }, () => Math.random() - 0.5);
|
|
53
|
+
const n = norm(pc);
|
|
54
|
+
if (n > 0)
|
|
55
|
+
pc = scale(pc, 1 / n);
|
|
56
|
+
for (let iter = 0; iter < iterations; iter++) {
|
|
57
|
+
const newPc = new Array(dims).fill(0);
|
|
58
|
+
for (const row of centered) {
|
|
59
|
+
const d = dot(row, pc);
|
|
60
|
+
for (let j = 0; j < dims; j++)
|
|
61
|
+
newPc[j] += d * row[j];
|
|
62
|
+
}
|
|
63
|
+
const len = norm(newPc);
|
|
64
|
+
if (len === 0)
|
|
65
|
+
break;
|
|
66
|
+
pc = scale(newPc, 1 / len);
|
|
67
|
+
}
|
|
68
|
+
return pc;
|
|
69
|
+
}
|
|
70
|
+
/** Deflate: remove component along given direction */
|
|
71
|
+
function deflate(centered, pc) {
|
|
72
|
+
return centered.map(row => {
|
|
73
|
+
const proj = dot(row, pc);
|
|
74
|
+
return row.map((v, i) => v - proj * pc[i]);
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
/** PCA project to 2D */
|
|
78
|
+
function pcaProject(vectors) {
|
|
79
|
+
if (vectors.length === 0)
|
|
80
|
+
return [];
|
|
81
|
+
const dims = vectors[0].length;
|
|
82
|
+
if (vectors.length === 1) {
|
|
83
|
+
return [{ x: 0, y: 0 }];
|
|
84
|
+
}
|
|
85
|
+
const mu = mean(vectors);
|
|
86
|
+
let centered = vectors.map(v => subtract(v, mu));
|
|
87
|
+
const pc1 = powerIteration(centered, dims);
|
|
88
|
+
centered = deflate(centered, pc1);
|
|
89
|
+
const pc2 = powerIteration(centered, dims);
|
|
90
|
+
return vectors.map(v => {
|
|
91
|
+
const c = subtract(v, mu);
|
|
92
|
+
return { x: dot(c, pc1), y: dot(c, pc2) };
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
// Cosine similarity stats
|
|
97
|
+
// ---------------------------------------------------------------------------
|
|
98
|
+
function cosineSimilarity(a, b) {
|
|
99
|
+
const na = norm(a);
|
|
100
|
+
const nb = norm(b);
|
|
101
|
+
if (na === 0 || nb === 0)
|
|
102
|
+
return 0;
|
|
103
|
+
return dot(a, b) / (na * nb);
|
|
104
|
+
}
|
|
105
|
+
function computeStats(ids, vectors, topK = 10) {
|
|
106
|
+
let sumSim = 0;
|
|
107
|
+
let minSim = 1;
|
|
108
|
+
let maxSim = -1;
|
|
109
|
+
let pairCount = 0;
|
|
110
|
+
let sumNorm = 0;
|
|
111
|
+
const topPairs = [];
|
|
112
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
113
|
+
sumNorm += norm(vectors[i]);
|
|
114
|
+
}
|
|
115
|
+
// Sample up to 5000 pairs for large collections
|
|
116
|
+
const maxPairs = 5000;
|
|
117
|
+
const allPairs = (vectors.length * (vectors.length - 1)) / 2;
|
|
118
|
+
const sampleRate = allPairs > maxPairs ? maxPairs / allPairs : 1;
|
|
119
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
120
|
+
for (let j = i + 1; j < vectors.length; j++) {
|
|
121
|
+
if (sampleRate < 1 && Math.random() > sampleRate)
|
|
122
|
+
continue;
|
|
123
|
+
const sim = cosineSimilarity(vectors[i], vectors[j]);
|
|
124
|
+
sumSim += sim;
|
|
125
|
+
if (sim < minSim)
|
|
126
|
+
minSim = sim;
|
|
127
|
+
if (sim > maxSim)
|
|
128
|
+
maxSim = sim;
|
|
129
|
+
pairCount++;
|
|
130
|
+
if (topPairs.length < topK || sim > topPairs[topPairs.length - 1].similarity) {
|
|
131
|
+
topPairs.push({ a: ids[i], b: ids[j], similarity: Math.round(sim * 1000) / 1000 });
|
|
132
|
+
topPairs.sort((a, b) => b.similarity - a.similarity);
|
|
133
|
+
if (topPairs.length > topK)
|
|
134
|
+
topPairs.pop();
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return {
|
|
139
|
+
stats: {
|
|
140
|
+
avgCosineSim: pairCount > 0 ? Math.round((sumSim / pairCount) * 1000) / 1000 : 0,
|
|
141
|
+
minCosineSim: pairCount > 0 ? Math.round(minSim * 1000) / 1000 : 0,
|
|
142
|
+
maxCosineSim: pairCount > 0 ? Math.round(maxSim * 1000) / 1000 : 0,
|
|
143
|
+
avgNorm: vectors.length > 0 ? Math.round((sumNorm / vectors.length) * 1000) / 1000 : 0,
|
|
144
|
+
},
|
|
145
|
+
similarPairs: topPairs,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
// ---------------------------------------------------------------------------
|
|
149
|
+
// Route factory
|
|
150
|
+
// ---------------------------------------------------------------------------
|
|
151
|
+
function createEmbeddingsRoutes(embeddingPathOverride) {
|
|
152
|
+
const router = (0, express_1.Router)();
|
|
153
|
+
router.get('/embeddings/projection', (_req, res) => {
|
|
154
|
+
try {
|
|
155
|
+
const embeddingPath = embeddingPathOverride ?? (0, runtimeConfig_js_1.getRuntimeConfig)().semantic.embeddingPath ?? '';
|
|
156
|
+
if (!embeddingPath || !node_fs_1.default.existsSync(embeddingPath)) {
|
|
157
|
+
return res.status(404).json({
|
|
158
|
+
success: false,
|
|
159
|
+
error: 'Embeddings file not found',
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
const raw = node_fs_1.default.readFileSync(embeddingPath, 'utf-8');
|
|
163
|
+
const data = JSON.parse(raw);
|
|
164
|
+
const ids = Object.keys(data.embeddings);
|
|
165
|
+
const vectors = ids.map(id => data.embeddings[id]);
|
|
166
|
+
if (ids.length === 0) {
|
|
167
|
+
return res.json({
|
|
168
|
+
success: true,
|
|
169
|
+
count: 0,
|
|
170
|
+
dimensions: 0,
|
|
171
|
+
model: data.modelName,
|
|
172
|
+
points: [],
|
|
173
|
+
stats: { avgCosineSim: 0, minCosineSim: 0, maxCosineSim: 0, avgNorm: 0 },
|
|
174
|
+
similarPairs: [],
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
const dims = vectors[0].length;
|
|
178
|
+
const projected = pcaProject(vectors);
|
|
179
|
+
const points = ids.map((id, i) => ({
|
|
180
|
+
id,
|
|
181
|
+
x: Math.round(projected[i].x * 10000) / 10000,
|
|
182
|
+
y: Math.round(projected[i].y * 10000) / 10000,
|
|
183
|
+
category: (0, categoryRules_js_1.deriveCategory)(id),
|
|
184
|
+
norm: Math.round(norm(vectors[i]) * 10000) / 10000,
|
|
185
|
+
}));
|
|
186
|
+
const { stats, similarPairs } = computeStats(ids, vectors);
|
|
187
|
+
return res.json({
|
|
188
|
+
success: true,
|
|
189
|
+
count: ids.length,
|
|
190
|
+
dimensions: dims,
|
|
191
|
+
model: data.modelName,
|
|
192
|
+
points,
|
|
193
|
+
stats,
|
|
194
|
+
similarPairs,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
catch (err) {
|
|
198
|
+
return res.status(500).json({
|
|
199
|
+
success: false,
|
|
200
|
+
error: 'Failed to compute projection',
|
|
201
|
+
message: err.message,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
// POST /embeddings/compute — trigger embedding computation for all instructions
|
|
206
|
+
router.post('/embeddings/compute', async (_req, res) => {
|
|
207
|
+
try {
|
|
208
|
+
const config = (0, runtimeConfig_js_1.getRuntimeConfig)();
|
|
209
|
+
const sem = config.semantic;
|
|
210
|
+
if (!sem.enabled) {
|
|
211
|
+
return res.status(400).json({
|
|
212
|
+
success: false,
|
|
213
|
+
error: 'Semantic embeddings are disabled',
|
|
214
|
+
hint: 'Set INDEX_SERVER_SEMANTIC_ENABLED=1 and restart the server',
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
const state = (0, indexContext_js_1.ensureLoaded)();
|
|
218
|
+
if (!state || !state.list || state.list.length === 0) {
|
|
219
|
+
return res.status(400).json({
|
|
220
|
+
success: false,
|
|
221
|
+
error: 'No instructions loaded in index',
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
const start = performance.now();
|
|
225
|
+
const embeddings = await (0, embeddingService_js_1.getInstructionEmbeddings)(state.list, state.hash, sem.embeddingPath, sem.model, sem.cacheDir, sem.device, sem.localOnly);
|
|
226
|
+
const elapsed = (performance.now() - start).toFixed(0);
|
|
227
|
+
const count = Object.keys(embeddings).length;
|
|
228
|
+
return res.json({
|
|
229
|
+
success: true,
|
|
230
|
+
count,
|
|
231
|
+
model: sem.model,
|
|
232
|
+
device: sem.device,
|
|
233
|
+
elapsedMs: Number(elapsed),
|
|
234
|
+
embeddingPath: sem.embeddingPath,
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
catch (err) {
|
|
238
|
+
return res.status(500).json({
|
|
239
|
+
success: false,
|
|
240
|
+
error: 'Failed to compute embeddings',
|
|
241
|
+
message: err.message,
|
|
242
|
+
});
|
|
243
|
+
}
|
|
244
|
+
});
|
|
245
|
+
return router;
|
|
246
|
+
}
|