@skillsmith/core 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/src/analysis/CodebaseAnalyzer.d.ts.map +1 -1
- package/dist/src/analysis/CodebaseAnalyzer.js.map +1 -1
- package/dist/src/analysis/adapters/__tests__/python.test.js.map +1 -1
- package/dist/src/analysis/adapters/java-parsers.d.ts +64 -0
- package/dist/src/analysis/adapters/java-parsers.d.ts.map +1 -0
- package/dist/src/analysis/adapters/java-parsers.js +114 -0
- package/dist/src/analysis/adapters/java-parsers.js.map +1 -0
- package/dist/src/analysis/adapters/java.d.ts +1 -51
- package/dist/src/analysis/adapters/java.d.ts.map +1 -1
- package/dist/src/analysis/adapters/java.js +94 -205
- package/dist/src/analysis/adapters/java.js.map +1 -1
- package/dist/src/analysis/adapters/python-frameworks.d.ts +21 -0
- package/dist/src/analysis/adapters/python-frameworks.d.ts.map +1 -0
- package/dist/src/analysis/adapters/python-frameworks.js +86 -0
- package/dist/src/analysis/adapters/python-frameworks.js.map +1 -0
- package/dist/src/analysis/adapters/python.d.ts +0 -50
- package/dist/src/analysis/adapters/python.d.ts.map +1 -1
- package/dist/src/analysis/adapters/python.js +5 -120
- package/dist/src/analysis/adapters/python.js.map +1 -1
- package/dist/src/analysis/adapters/rust-parsers.d.ts +33 -0
- package/dist/src/analysis/adapters/rust-parsers.d.ts.map +1 -0
- package/dist/src/analysis/adapters/rust-parsers.js +94 -0
- package/dist/src/analysis/adapters/rust-parsers.js.map +1 -0
- package/dist/src/analysis/adapters/rust.d.ts +1 -43
- package/dist/src/analysis/adapters/rust.d.ts.map +1 -1
- package/dist/src/analysis/adapters/rust.js +74 -173
- package/dist/src/analysis/adapters/rust.js.map +1 -1
- package/dist/src/analysis/language-detector.d.ts +4 -20
- package/dist/src/analysis/language-detector.d.ts.map +1 -1
- package/dist/src/analysis/language-detector.js +2 -399
- package/dist/src/analysis/language-detector.js.map +1 -1
- package/dist/src/analysis/language-detector.patterns.d.ts +17 -0
- package/dist/src/analysis/language-detector.patterns.d.ts.map +1 -0
- package/dist/src/analysis/language-detector.patterns.js +413 -0
- package/dist/src/analysis/language-detector.patterns.js.map +1 -0
- package/dist/src/analysis/language-detector.types.d.ts +48 -0
- package/dist/src/analysis/language-detector.types.d.ts.map +1 -0
- package/dist/src/analysis/language-detector.types.js +6 -0
- package/dist/src/analysis/language-detector.types.js.map +1 -0
- package/dist/src/analysis/metrics.d.ts +22 -219
- package/dist/src/analysis/metrics.d.ts.map +1 -1
- package/dist/src/analysis/metrics.implementations.d.ts +39 -0
- package/dist/src/analysis/metrics.implementations.d.ts.map +1 -0
- package/dist/src/analysis/metrics.implementations.js +114 -0
- package/dist/src/analysis/metrics.implementations.js.map +1 -0
- package/dist/src/analysis/metrics.js +21 -224
- package/dist/src/analysis/metrics.js.map +1 -1
- package/dist/src/analysis/metrics.types.d.ts +79 -0
- package/dist/src/analysis/metrics.types.d.ts.map +1 -0
- package/dist/src/analysis/metrics.types.js +6 -0
- package/dist/src/analysis/metrics.types.js.map +1 -0
- package/dist/src/analysis/worker-pool.d.ts +25 -52
- package/dist/src/analysis/worker-pool.d.ts.map +1 -1
- package/dist/src/analysis/worker-pool.js +29 -181
- package/dist/src/analysis/worker-pool.js.map +1 -1
- package/dist/src/analysis/worker-types.d.ts +64 -0
- package/dist/src/analysis/worker-types.d.ts.map +1 -0
- package/dist/src/analysis/worker-types.js +11 -0
- package/dist/src/analysis/worker-types.js.map +1 -0
- package/dist/src/analysis/worker-utils.d.ts +49 -0
- package/dist/src/analysis/worker-utils.d.ts.map +1 -0
- package/dist/src/analysis/worker-utils.js +206 -0
- package/dist/src/analysis/worker-utils.js.map +1 -0
- package/dist/src/analytics/AnalyticsRepository.d.ts +1 -5
- package/dist/src/analytics/AnalyticsRepository.d.ts.map +1 -1
- package/dist/src/analytics/AnalyticsRepository.helpers.d.ts +28 -0
- package/dist/src/analytics/AnalyticsRepository.helpers.d.ts.map +1 -0
- package/dist/src/analytics/AnalyticsRepository.helpers.js +90 -0
- package/dist/src/analytics/AnalyticsRepository.helpers.js.map +1 -0
- package/dist/src/analytics/AnalyticsRepository.js +16 -129
- package/dist/src/analytics/AnalyticsRepository.js.map +1 -1
- package/dist/src/analytics/AnalyticsRepository.types.d.ts +82 -0
- package/dist/src/analytics/AnalyticsRepository.types.d.ts.map +1 -0
- package/dist/src/analytics/AnalyticsRepository.types.js +8 -0
- package/dist/src/analytics/AnalyticsRepository.types.js.map +1 -0
- package/dist/src/analytics/ROIDashboardService.js +4 -2
- package/dist/src/analytics/ROIDashboardService.js.map +1 -1
- package/dist/src/api/cache.d.ts.map +1 -1
- package/dist/src/api/cache.js.map +1 -1
- package/dist/src/api/client.d.ts +6 -121
- package/dist/src/api/client.d.ts.map +1 -1
- package/dist/src/api/client.js +29 -129
- package/dist/src/api/client.js.map +1 -1
- package/dist/src/api/schemas.d.ts +122 -0
- package/dist/src/api/schemas.d.ts.map +1 -0
- package/dist/src/api/schemas.js +71 -0
- package/dist/src/api/schemas.js.map +1 -0
- package/dist/src/api/types.d.ts +1 -1
- package/dist/src/api/types.d.ts.map +1 -1
- package/dist/src/api/utils.d.ts +35 -0
- package/dist/src/api/utils.d.ts.map +1 -0
- package/dist/src/api/utils.js +86 -0
- package/dist/src/api/utils.js.map +1 -0
- package/dist/src/benchmarks/BenchmarkRunner.d.ts.map +1 -1
- package/dist/src/benchmarks/BenchmarkRunner.js +1 -1
- package/dist/src/benchmarks/BenchmarkRunner.js.map +1 -1
- package/dist/src/billing/BillingService.d.ts +15 -53
- package/dist/src/billing/BillingService.d.ts.map +1 -1
- package/dist/src/billing/BillingService.helpers.d.ts +15 -0
- package/dist/src/billing/BillingService.helpers.d.ts.map +1 -0
- package/dist/src/billing/BillingService.helpers.js +45 -0
- package/dist/src/billing/BillingService.helpers.js.map +1 -0
- package/dist/src/billing/BillingService.js +49 -179
- package/dist/src/billing/BillingService.js.map +1 -1
- package/dist/src/billing/BillingService.types.d.ts +52 -0
- package/dist/src/billing/BillingService.types.d.ts.map +1 -0
- package/dist/src/billing/BillingService.types.js +6 -0
- package/dist/src/billing/BillingService.types.js.map +1 -0
- package/dist/src/billing/GDPRComplianceService.d.ts +2 -97
- package/dist/src/billing/GDPRComplianceService.d.ts.map +1 -1
- package/dist/src/billing/GDPRComplianceService.js.map +1 -1
- package/dist/src/billing/StripeClient.d.ts +6 -64
- package/dist/src/billing/StripeClient.d.ts.map +1 -1
- package/dist/src/billing/StripeClient.js +14 -71
- package/dist/src/billing/StripeClient.js.map +1 -1
- package/dist/src/billing/StripeReconciliationJob.d.ts +2 -47
- package/dist/src/billing/StripeReconciliationJob.d.ts.map +1 -1
- package/dist/src/billing/StripeReconciliationJob.js +1 -41
- package/dist/src/billing/StripeReconciliationJob.js.map +1 -1
- package/dist/src/billing/StripeWebhookHandler.d.ts +3 -46
- package/dist/src/billing/StripeWebhookHandler.d.ts.map +1 -1
- package/dist/src/billing/StripeWebhookHandler.js +14 -261
- package/dist/src/billing/StripeWebhookHandler.js.map +1 -1
- package/dist/src/billing/gdpr-types.d.ts +103 -0
- package/dist/src/billing/gdpr-types.d.ts.map +1 -0
- package/dist/src/billing/gdpr-types.js +7 -0
- package/dist/src/billing/gdpr-types.js.map +1 -0
- package/dist/src/billing/reconciliation-helpers.d.ts +16 -0
- package/dist/src/billing/reconciliation-helpers.d.ts.map +1 -0
- package/dist/src/billing/reconciliation-helpers.js +53 -0
- package/dist/src/billing/reconciliation-helpers.js.map +1 -0
- package/dist/src/billing/reconciliation-types.d.ts +71 -0
- package/dist/src/billing/reconciliation-types.d.ts.map +1 -0
- package/dist/src/billing/reconciliation-types.js +7 -0
- package/dist/src/billing/reconciliation-types.js.map +1 -0
- package/dist/src/billing/stripe-client-types.d.ts +45 -0
- package/dist/src/billing/stripe-client-types.d.ts.map +1 -0
- package/dist/src/billing/stripe-client-types.js +7 -0
- package/dist/src/billing/stripe-client-types.js.map +1 -0
- package/dist/src/billing/stripe-helpers.d.ts +17 -0
- package/dist/src/billing/stripe-helpers.d.ts.map +1 -0
- package/dist/src/billing/stripe-helpers.js +50 -0
- package/dist/src/billing/stripe-helpers.js.map +1 -0
- package/dist/src/billing/webhook-handlers.d.ts +56 -0
- package/dist/src/billing/webhook-handlers.d.ts.map +1 -0
- package/dist/src/billing/webhook-handlers.js +303 -0
- package/dist/src/billing/webhook-handlers.js.map +1 -0
- package/dist/src/billing/webhook-types.d.ts +42 -0
- package/dist/src/billing/webhook-types.d.ts.map +1 -0
- package/dist/src/billing/webhook-types.js +7 -0
- package/dist/src/billing/webhook-types.js.map +1 -0
- package/dist/src/config/index.d.ts +118 -0
- package/dist/src/config/index.d.ts.map +1 -0
- package/dist/src/config/index.js +172 -0
- package/dist/src/config/index.js.map +1 -0
- package/dist/src/db/migration-types.d.ts +101 -0
- package/dist/src/db/migration-types.d.ts.map +1 -0
- package/dist/src/db/migration-types.js +6 -0
- package/dist/src/db/migration-types.js.map +1 -0
- package/dist/src/db/migration.d.ts +32 -0
- package/dist/src/db/migration.d.ts.map +1 -0
- package/dist/src/db/migration.js +239 -0
- package/dist/src/db/migration.js.map +1 -0
- package/dist/src/db/schema.d.ts +2 -2
- package/dist/src/db/schema.d.ts.map +1 -1
- package/dist/src/db/schema.js +23 -1
- package/dist/src/db/schema.js.map +1 -1
- package/dist/src/embeddings/embedding-types.d.ts +44 -0
- package/dist/src/embeddings/embedding-types.d.ts.map +1 -0
- package/dist/src/embeddings/embedding-types.js +6 -0
- package/dist/src/embeddings/embedding-types.js.map +1 -0
- package/dist/src/embeddings/embedding-utils.d.ts +42 -0
- package/dist/src/embeddings/embedding-utils.d.ts.map +1 -0
- package/dist/src/embeddings/embedding-utils.js +121 -0
- package/dist/src/embeddings/embedding-utils.js.map +1 -0
- package/dist/src/embeddings/hnsw-store.d.ts +8 -518
- package/dist/src/embeddings/hnsw-store.d.ts.map +1 -1
- package/dist/src/embeddings/hnsw-store.helpers.d.ts +83 -0
- package/dist/src/embeddings/hnsw-store.helpers.d.ts.map +1 -0
- package/dist/src/embeddings/hnsw-store.helpers.js +147 -0
- package/dist/src/embeddings/hnsw-store.helpers.js.map +1 -0
- package/dist/src/embeddings/hnsw-store.js +66 -517
- package/dist/src/embeddings/hnsw-store.js.map +1 -1
- package/dist/src/embeddings/hnsw-store.types.d.ts +154 -0
- package/dist/src/embeddings/hnsw-store.types.d.ts.map +1 -0
- package/dist/src/embeddings/hnsw-store.types.js +26 -0
- package/dist/src/embeddings/hnsw-store.types.js.map +1 -0
- package/dist/src/embeddings/index.d.ts +36 -112
- package/dist/src/embeddings/index.d.ts.map +1 -1
- package/dist/src/embeddings/index.js +37 -210
- package/dist/src/embeddings/index.js.map +1 -1
- package/dist/src/errors.d.ts.map +1 -1
- package/dist/src/errors.js +1 -1
- package/dist/src/errors.js.map +1 -1
- package/dist/src/exports/repositories.d.ts +18 -0
- package/dist/src/exports/repositories.d.ts.map +1 -0
- package/dist/src/exports/repositories.js +30 -0
- package/dist/src/exports/repositories.js.map +1 -0
- package/dist/src/exports/services.d.ts +30 -0
- package/dist/src/exports/services.d.ts.map +1 -0
- package/dist/src/exports/services.js +70 -0
- package/dist/src/exports/services.js.map +1 -0
- package/dist/src/exports/types.d.ts +21 -0
- package/dist/src/exports/types.d.ts.map +1 -0
- package/dist/src/exports/types.js +12 -0
- package/dist/src/exports/types.js.map +1 -0
- package/dist/src/index.d.ts +10 -47
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +31 -48
- package/dist/src/index.js.map +1 -1
- package/dist/src/indexer/GitHubIndexer.d.ts.map +1 -1
- package/dist/src/indexer/GitHubIndexer.js +17 -4
- package/dist/src/indexer/GitHubIndexer.js.map +1 -1
- package/dist/src/indexer/SwarmIndexer.d.ts.map +1 -1
- package/dist/src/indexer/SwarmIndexer.js +1 -1
- package/dist/src/indexer/SwarmIndexer.js.map +1 -1
- package/dist/src/learning/PatternStore.d.ts +6 -426
- package/dist/src/learning/PatternStore.d.ts.map +1 -1
- package/dist/src/learning/PatternStore.helpers.d.ts +88 -0
- package/dist/src/learning/PatternStore.helpers.d.ts.map +1 -0
- package/dist/src/learning/PatternStore.helpers.js +286 -0
- package/dist/src/learning/PatternStore.helpers.js.map +1 -0
- package/dist/src/learning/PatternStore.js +52 -598
- package/dist/src/learning/PatternStore.js.map +1 -1
- package/dist/src/learning/PatternStore.queries.d.ts +83 -0
- package/dist/src/learning/PatternStore.queries.d.ts.map +1 -0
- package/dist/src/learning/PatternStore.queries.js +189 -0
- package/dist/src/learning/PatternStore.queries.js.map +1 -0
- package/dist/src/learning/PatternStore.types.d.ts +308 -0
- package/dist/src/learning/PatternStore.types.d.ts.map +1 -0
- package/dist/src/learning/PatternStore.types.js +44 -0
- package/dist/src/learning/PatternStore.types.js.map +1 -0
- package/dist/src/learning/ReasoningBankIntegration.d.ts +5 -345
- package/dist/src/learning/ReasoningBankIntegration.d.ts.map +1 -1
- package/dist/src/learning/ReasoningBankIntegration.helpers.d.ts +49 -0
- package/dist/src/learning/ReasoningBankIntegration.helpers.d.ts.map +1 -0
- package/dist/src/learning/ReasoningBankIntegration.helpers.js +180 -0
- package/dist/src/learning/ReasoningBankIntegration.helpers.js.map +1 -0
- package/dist/src/learning/ReasoningBankIntegration.js +21 -390
- package/dist/src/learning/ReasoningBankIntegration.js.map +1 -1
- package/dist/src/learning/ReasoningBankIntegration.types.d.ts +167 -0
- package/dist/src/learning/ReasoningBankIntegration.types.d.ts.map +1 -0
- package/dist/src/learning/ReasoningBankIntegration.types.js +51 -0
- package/dist/src/learning/ReasoningBankIntegration.types.js.map +1 -0
- package/dist/src/learning/interfaces.d.ts.map +1 -1
- package/dist/src/pipeline/DailyIndexPipeline.d.ts +4 -176
- package/dist/src/pipeline/DailyIndexPipeline.d.ts.map +1 -1
- package/dist/src/pipeline/DailyIndexPipeline.js +9 -83
- package/dist/src/pipeline/DailyIndexPipeline.js.map +1 -1
- package/dist/src/pipeline/pipeline-types.d.ts +132 -0
- package/dist/src/pipeline/pipeline-types.d.ts.map +1 -0
- package/dist/src/pipeline/pipeline-types.js +6 -0
- package/dist/src/pipeline/pipeline-types.js.map +1 -0
- package/dist/src/repositories/IndexerRepository.d.ts +1 -0
- package/dist/src/repositories/IndexerRepository.d.ts.map +1 -1
- package/dist/src/repositories/IndexerRepository.js +6 -0
- package/dist/src/repositories/IndexerRepository.js.map +1 -1
- package/dist/src/repositories/SkillRepository.d.ts.map +1 -1
- package/dist/src/repositories/SkillRepository.js +20 -5
- package/dist/src/repositories/SkillRepository.js.map +1 -1
- package/dist/src/routing/SONARouter.d.ts +3 -101
- package/dist/src/routing/SONARouter.d.ts.map +1 -1
- package/dist/src/routing/SONARouter.helpers.d.ts +112 -0
- package/dist/src/routing/SONARouter.helpers.d.ts.map +1 -0
- package/dist/src/routing/SONARouter.helpers.js +184 -0
- package/dist/src/routing/SONARouter.helpers.js.map +1 -0
- package/dist/src/routing/SONARouter.js +16 -318
- package/dist/src/routing/SONARouter.js.map +1 -1
- package/dist/src/routing/expert-types.d.ts +109 -0
- package/dist/src/routing/expert-types.d.ts.map +1 -0
- package/dist/src/routing/expert-types.js +162 -0
- package/dist/src/routing/expert-types.js.map +1 -0
- package/dist/src/routing/request-types.d.ts +113 -0
- package/dist/src/routing/request-types.d.ts.map +1 -0
- package/dist/src/routing/request-types.js +9 -0
- package/dist/src/routing/request-types.js.map +1 -0
- package/dist/src/routing/types.d.ts +5 -207
- package/dist/src/routing/types.d.ts.map +1 -1
- package/dist/src/routing/types.js +3 -153
- package/dist/src/routing/types.js.map +1 -1
- package/dist/src/scripts/github-import/github-auth.d.ts +17 -0
- package/dist/src/scripts/github-import/github-auth.d.ts.map +1 -0
- package/dist/src/scripts/github-import/github-auth.js +201 -0
- package/dist/src/scripts/github-import/github-auth.js.map +1 -0
- package/dist/src/scripts/github-import/github-client.d.ts +5 -5
- package/dist/src/scripts/github-import/github-client.d.ts.map +1 -1
- package/dist/src/scripts/github-import/github-client.js +5 -204
- package/dist/src/scripts/github-import/github-client.js.map +1 -1
- package/dist/src/scripts/ingest-lenny-skills.d.ts +40 -0
- package/dist/src/scripts/ingest-lenny-skills.d.ts.map +1 -0
- package/dist/src/scripts/ingest-lenny-skills.js +346 -0
- package/dist/src/scripts/ingest-lenny-skills.js.map +1 -0
- package/dist/src/scripts/merge-skills.d.ts +5 -27
- package/dist/src/scripts/merge-skills.d.ts.map +1 -1
- package/dist/src/scripts/merge-skills.js +4 -144
- package/dist/src/scripts/merge-skills.js.map +1 -1
- package/dist/src/scripts/merge-types.d.ts +72 -0
- package/dist/src/scripts/merge-types.d.ts.map +1 -0
- package/dist/src/scripts/merge-types.js +7 -0
- package/dist/src/scripts/merge-types.js.map +1 -0
- package/dist/src/scripts/merge-utils.d.ts +23 -0
- package/dist/src/scripts/merge-utils.d.ts.map +1 -0
- package/dist/src/scripts/merge-utils.js +150 -0
- package/dist/src/scripts/merge-utils.js.map +1 -0
- package/dist/src/scripts/review-categories.d.ts +41 -0
- package/dist/src/scripts/review-categories.d.ts.map +1 -0
- package/dist/src/scripts/review-categories.js +316 -0
- package/dist/src/scripts/review-categories.js.map +1 -0
- package/dist/src/scripts/review-lenny-skills.d.ts +26 -0
- package/dist/src/scripts/review-lenny-skills.d.ts.map +1 -0
- package/dist/src/scripts/review-lenny-skills.js +215 -0
- package/dist/src/scripts/review-lenny-skills.js.map +1 -0
- package/dist/src/scripts/sync-to-supabase.d.ts +10 -0
- package/dist/src/scripts/sync-to-supabase.d.ts.map +1 -0
- package/dist/src/scripts/sync-to-supabase.js +93 -0
- package/dist/src/scripts/sync-to-supabase.js.map +1 -0
- package/dist/src/security/AuditLogger.d.ts +4 -75
- package/dist/src/security/AuditLogger.d.ts.map +1 -1
- package/dist/src/security/AuditLogger.js +3 -12
- package/dist/src/security/AuditLogger.js.map +1 -1
- package/dist/src/security/__tests__/pathValidation.test.js.map +1 -1
- package/dist/src/security/audit-types.d.ts +96 -0
- package/dist/src/security/audit-types.d.ts.map +1 -0
- package/dist/src/security/audit-types.js +15 -0
- package/dist/src/security/audit-types.js.map +1 -0
- package/dist/src/security/sanitization.d.ts +1 -85
- package/dist/src/security/sanitization.d.ts.map +1 -1
- package/dist/src/security/sanitization.js +2 -133
- package/dist/src/security/sanitization.js.map +1 -1
- package/dist/src/security/scanner/SecurityScanner.d.ts +12 -88
- package/dist/src/security/scanner/SecurityScanner.d.ts.map +1 -1
- package/dist/src/security/scanner/SecurityScanner.formatters.d.ts +49 -0
- package/dist/src/security/scanner/SecurityScanner.formatters.d.ts.map +1 -0
- package/dist/src/security/scanner/SecurityScanner.formatters.js +139 -0
- package/dist/src/security/scanner/SecurityScanner.formatters.js.map +1 -0
- package/dist/src/security/scanner/SecurityScanner.helpers.d.ts +40 -0
- package/dist/src/security/scanner/SecurityScanner.helpers.d.ts.map +1 -0
- package/dist/src/security/scanner/SecurityScanner.helpers.js +142 -0
- package/dist/src/security/scanner/SecurityScanner.helpers.js.map +1 -0
- package/dist/src/security/scanner/SecurityScanner.js +26 -255
- package/dist/src/security/scanner/SecurityScanner.js.map +1 -1
- package/dist/src/security/stripe-validators.d.ts +92 -0
- package/dist/src/security/stripe-validators.d.ts.map +1 -0
- package/dist/src/security/stripe-validators.js +139 -0
- package/dist/src/security/stripe-validators.js.map +1 -0
- package/dist/src/services/SearchService.d.ts +5 -29
- package/dist/src/services/SearchService.d.ts.map +1 -1
- package/dist/src/services/SearchService.helpers.d.ts +42 -0
- package/dist/src/services/SearchService.helpers.d.ts.map +1 -0
- package/dist/src/services/SearchService.helpers.js +128 -0
- package/dist/src/services/SearchService.helpers.js.map +1 -0
- package/dist/src/services/SearchService.js +142 -128
- package/dist/src/services/SearchService.js.map +1 -1
- package/dist/src/services/SearchService.types.d.ts +53 -0
- package/dist/src/services/SearchService.types.d.ts.map +1 -0
- package/dist/src/services/SearchService.types.js +7 -0
- package/dist/src/services/SearchService.types.js.map +1 -0
- package/dist/src/services/SkillAnalyzer.d.ts +37 -0
- package/dist/src/services/SkillAnalyzer.d.ts.map +1 -0
- package/dist/src/services/SkillAnalyzer.helpers.d.ts +103 -0
- package/dist/src/services/SkillAnalyzer.helpers.d.ts.map +1 -0
- package/dist/src/services/SkillAnalyzer.helpers.js +237 -0
- package/dist/src/services/SkillAnalyzer.helpers.js.map +1 -0
- package/dist/src/services/SkillAnalyzer.js +270 -0
- package/dist/src/services/SkillAnalyzer.js.map +1 -0
- package/dist/src/services/SkillAnalyzer.types.d.ts +110 -0
- package/dist/src/services/SkillAnalyzer.types.d.ts.map +1 -0
- package/dist/src/services/SkillAnalyzer.types.js +9 -0
- package/dist/src/services/SkillAnalyzer.types.js.map +1 -0
- package/dist/src/services/SkillDecomposer.d.ts +37 -0
- package/dist/src/services/SkillDecomposer.d.ts.map +1 -0
- package/dist/src/services/SkillDecomposer.helpers.d.ts +75 -0
- package/dist/src/services/SkillDecomposer.helpers.d.ts.map +1 -0
- package/dist/src/services/SkillDecomposer.helpers.js +277 -0
- package/dist/src/services/SkillDecomposer.helpers.js.map +1 -0
- package/dist/src/services/SkillDecomposer.js +116 -0
- package/dist/src/services/SkillDecomposer.js.map +1 -0
- package/dist/src/services/SkillDecomposer.types.d.ts +104 -0
- package/dist/src/services/SkillDecomposer.types.d.ts.map +1 -0
- package/dist/src/services/SkillDecomposer.types.js +9 -0
- package/dist/src/services/SkillDecomposer.types.js.map +1 -0
- package/dist/src/services/SubagentGenerator.d.ts +77 -0
- package/dist/src/services/SubagentGenerator.d.ts.map +1 -0
- package/dist/src/services/SubagentGenerator.js +352 -0
- package/dist/src/services/SubagentGenerator.js.map +1 -0
- package/dist/src/services/TaskRunner.d.ts +227 -0
- package/dist/src/services/TaskRunner.d.ts.map +1 -0
- package/dist/src/services/TaskRunner.js +423 -0
- package/dist/src/services/TaskRunner.js.map +1 -0
- package/dist/src/services/TransformationService.d.ts +167 -0
- package/dist/src/services/TransformationService.d.ts.map +1 -0
- package/dist/src/services/TransformationService.js +315 -0
- package/dist/src/services/TransformationService.js.map +1 -0
- package/dist/src/services/__tests__/SkillAnalyzer.test.d.ts +6 -0
- package/dist/src/services/__tests__/SkillAnalyzer.test.d.ts.map +1 -0
- package/dist/src/services/__tests__/SkillAnalyzer.test.js +177 -0
- package/dist/src/services/__tests__/SkillAnalyzer.test.js.map +1 -0
- package/dist/src/services/__tests__/SkillDecomposer.test.d.ts +6 -0
- package/dist/src/services/__tests__/SkillDecomposer.test.d.ts.map +1 -0
- package/dist/src/services/__tests__/SkillDecomposer.test.js +235 -0
- package/dist/src/services/__tests__/SkillDecomposer.test.js.map +1 -0
- package/dist/src/services/__tests__/SubagentGenerator.test.d.ts +6 -0
- package/dist/src/services/__tests__/SubagentGenerator.test.d.ts.map +1 -0
- package/dist/src/services/__tests__/SubagentGenerator.test.js +174 -0
- package/dist/src/services/__tests__/SubagentGenerator.test.js.map +1 -0
- package/dist/src/services/__tests__/TaskRunner.test.d.ts +7 -0
- package/dist/src/services/__tests__/TaskRunner.test.d.ts.map +1 -0
- package/dist/src/services/__tests__/TaskRunner.test.js +316 -0
- package/dist/src/services/__tests__/TaskRunner.test.js.map +1 -0
- package/dist/src/services/__tests__/TransformationService.test.d.ts +6 -0
- package/dist/src/services/__tests__/TransformationService.test.d.ts.map +1 -0
- package/dist/src/services/__tests__/TransformationService.test.js +244 -0
- package/dist/src/services/__tests__/TransformationService.test.js.map +1 -0
- package/dist/src/session/SessionContext.js +1 -1
- package/dist/src/session/SessionContext.js.map +1 -1
- package/dist/src/session/SessionManager.d.ts +3 -67
- package/dist/src/session/SessionManager.d.ts.map +1 -1
- package/dist/src/session/SessionManager.helpers.d.ts +68 -0
- package/dist/src/session/SessionManager.helpers.d.ts.map +1 -0
- package/dist/src/session/SessionManager.helpers.js +152 -0
- package/dist/src/session/SessionManager.helpers.js.map +1 -0
- package/dist/src/session/SessionManager.js +58 -163
- package/dist/src/session/SessionManager.js.map +1 -1
- package/dist/src/session/SessionManager.types.d.ts +88 -0
- package/dist/src/session/SessionManager.types.d.ts.map +1 -0
- package/dist/src/session/SessionManager.types.js +27 -0
- package/dist/src/session/SessionManager.types.js.map +1 -0
- package/dist/src/session/SessionRecovery.d.ts.map +1 -1
- package/dist/src/session/SessionRecovery.js +3 -2
- package/dist/src/session/SessionRecovery.js.map +1 -1
- package/dist/src/sources/RawUrlSourceAdapter.js.map +1 -1
- package/dist/src/sources/SourceAdapterRegistry.js +1 -1
- package/dist/src/sources/SourceAdapterRegistry.js.map +1 -1
- package/dist/src/telemetry/metric-helpers.d.ts +47 -0
- package/dist/src/telemetry/metric-helpers.d.ts.map +1 -0
- package/dist/src/telemetry/metric-helpers.js +96 -0
- package/dist/src/telemetry/metric-helpers.js.map +1 -0
- package/dist/src/telemetry/metric-types.d.ts +73 -0
- package/dist/src/telemetry/metric-types.d.ts.map +1 -0
- package/dist/src/telemetry/metric-types.js +9 -0
- package/dist/src/telemetry/metric-types.js.map +1 -0
- package/dist/src/telemetry/metrics.d.ts +8 -119
- package/dist/src/telemetry/metrics.d.ts.map +1 -1
- package/dist/src/telemetry/metrics.js +20 -193
- package/dist/src/telemetry/metrics.js.map +1 -1
- package/dist/src/telemetry/span-utils.d.ts +30 -0
- package/dist/src/telemetry/span-utils.d.ts.map +1 -0
- package/dist/src/telemetry/span-utils.js +62 -0
- package/dist/src/telemetry/span-utils.js.map +1 -0
- package/dist/src/telemetry/tracer-types.d.ts +91 -0
- package/dist/src/telemetry/tracer-types.d.ts.map +1 -0
- package/dist/src/telemetry/tracer-types.js +6 -0
- package/dist/src/telemetry/tracer-types.js.map +1 -0
- package/dist/src/telemetry/tracer.d.ts +5 -122
- package/dist/src/telemetry/tracer.d.ts.map +1 -1
- package/dist/src/telemetry/tracer.js +24 -170
- package/dist/src/telemetry/tracer.js.map +1 -1
- package/dist/src/testing/MultiLLMProvider.d.ts +17 -293
- package/dist/src/testing/MultiLLMProvider.d.ts.map +1 -1
- package/dist/src/testing/MultiLLMProvider.helpers.d.ts +54 -0
- package/dist/src/testing/MultiLLMProvider.helpers.d.ts.map +1 -0
- package/dist/src/testing/MultiLLMProvider.helpers.js +126 -0
- package/dist/src/testing/MultiLLMProvider.helpers.js.map +1 -0
- package/dist/src/testing/MultiLLMProvider.js +50 -323
- package/dist/src/testing/MultiLLMProvider.js.map +1 -1
- package/dist/src/testing/MultiLLMProvider.types.d.ts +253 -0
- package/dist/src/testing/MultiLLMProvider.types.d.ts.map +1 -0
- package/dist/src/testing/MultiLLMProvider.types.js +111 -0
- package/dist/src/testing/MultiLLMProvider.types.js.map +1 -0
- package/dist/src/triggers/TriggerDetector.d.ts +6 -142
- package/dist/src/triggers/TriggerDetector.d.ts.map +1 -1
- package/dist/src/triggers/TriggerDetector.js +15 -247
- package/dist/src/triggers/TriggerDetector.js.map +1 -1
- package/dist/src/triggers/__tests__/TriggerDetector.test.js +1 -1
- package/dist/src/triggers/__tests__/TriggerDetector.test.js.map +1 -1
- package/dist/src/triggers/trigger-constants.d.ts +22 -0
- package/dist/src/triggers/trigger-constants.d.ts.map +1 -0
- package/dist/src/triggers/trigger-constants.js +185 -0
- package/dist/src/triggers/trigger-constants.js.map +1 -0
- package/dist/src/triggers/trigger-types.d.ts +90 -0
- package/dist/src/triggers/trigger-types.d.ts.map +1 -0
- package/dist/src/triggers/trigger-types.js +6 -0
- package/dist/src/triggers/trigger-types.js.map +1 -0
- package/dist/src/types/skill.d.ts +33 -1
- package/dist/src/types/skill.d.ts.map +1 -1
- package/dist/src/types/skill.js +11 -1
- package/dist/src/types/skill.js.map +1 -1
- package/dist/src/types.d.ts +28 -2
- package/dist/src/types.d.ts.map +1 -1
- package/dist/src/types.js +1 -0
- package/dist/src/types.js.map +1 -1
- package/dist/src/utils/retry.js +2 -2
- package/dist/src/utils/retry.js.map +1 -1
- package/dist/src/validation/index.d.ts +5 -105
- package/dist/src/validation/index.d.ts.map +1 -1
- package/dist/src/validation/index.js +9 -421
- package/dist/src/validation/index.js.map +1 -1
- package/dist/src/validation/input-validators.d.ts +68 -0
- package/dist/src/validation/input-validators.d.ts.map +1 -0
- package/dist/src/validation/input-validators.js +126 -0
- package/dist/src/validation/input-validators.js.map +1 -0
- package/dist/src/validation/path-validators.d.ts +23 -0
- package/dist/src/validation/path-validators.d.ts.map +1 -0
- package/dist/src/validation/path-validators.js +47 -0
- package/dist/src/validation/path-validators.js.map +1 -0
- package/dist/src/validation/url-validators.d.ts +55 -0
- package/dist/src/validation/url-validators.d.ts.map +1 -0
- package/dist/src/validation/url-validators.js +267 -0
- package/dist/src/validation/url-validators.js.map +1 -0
- package/dist/src/validation/validation-error.d.ts +14 -0
- package/dist/src/validation/validation-error.d.ts.map +1 -0
- package/dist/src/validation/validation-error.js +19 -0
- package/dist/src/validation/validation-error.js.map +1 -0
- package/dist/src/webhooks/WebhookHandler.d.ts.map +1 -1
- package/dist/src/webhooks/WebhookHandler.js.map +1 -1
- package/dist/src/webhooks/WebhookPayload.d.ts +4 -215
- package/dist/src/webhooks/WebhookPayload.d.ts.map +1 -1
- package/dist/src/webhooks/WebhookPayload.js +1 -122
- package/dist/src/webhooks/WebhookPayload.js.map +1 -1
- package/dist/src/webhooks/WebhookQueue.d.ts +2 -140
- package/dist/src/webhooks/WebhookQueue.d.ts.map +1 -1
- package/dist/src/webhooks/WebhookQueue.js +4 -11
- package/dist/src/webhooks/WebhookQueue.js.map +1 -1
- package/dist/src/webhooks/WebhookQueue.types.d.ts +149 -0
- package/dist/src/webhooks/WebhookQueue.types.d.ts.map +1 -0
- package/dist/src/webhooks/WebhookQueue.types.js +10 -0
- package/dist/src/webhooks/WebhookQueue.types.js.map +1 -0
- package/dist/src/webhooks/WebhookQueue.utils.d.ts +44 -0
- package/dist/src/webhooks/WebhookQueue.utils.d.ts.map +1 -0
- package/dist/src/webhooks/WebhookQueue.utils.js +58 -0
- package/dist/src/webhooks/WebhookQueue.utils.js.map +1 -0
- package/dist/src/webhooks/webhook-schemas.d.ts +213 -0
- package/dist/src/webhooks/webhook-schemas.d.ts.map +1 -0
- package/dist/src/webhooks/webhook-schemas.js +132 -0
- package/dist/src/webhooks/webhook-schemas.js.map +1 -0
- package/dist/src/webhooks/webhook-types.d.ts +222 -0
- package/dist/src/webhooks/webhook-types.d.ts.map +1 -0
- package/dist/src/webhooks/webhook-types.js +10 -0
- package/dist/src/webhooks/webhook-types.js.map +1 -0
- package/dist/tests/Analytics.integration.test.js +2 -2
- package/dist/tests/Analytics.integration.test.js.map +1 -1
- package/dist/tests/AnalyticsRepository.test.js +3 -5
- package/dist/tests/AnalyticsRepository.test.js.map +1 -1
- package/dist/tests/AnalyticsStorage.test.js.map +1 -1
- package/dist/tests/ApiPartialResponses.test.d.ts +12 -0
- package/dist/tests/ApiPartialResponses.test.d.ts.map +1 -0
- package/dist/tests/ApiPartialResponses.test.js +202 -0
- package/dist/tests/ApiPartialResponses.test.js.map +1 -0
- package/dist/tests/AuditLogger.test.js.map +1 -1
- package/dist/tests/BenchmarkRunner.test.js +6 -6
- package/dist/tests/BenchmarkRunner.test.js.map +1 -1
- package/dist/tests/CacheSecurity.test.js.map +1 -1
- package/dist/tests/CodebaseAnalyzer.test.js.map +1 -1
- package/dist/tests/DailyIndexPipeline.test.js.map +1 -1
- package/dist/tests/EmbeddingService.test.js.map +1 -1
- package/dist/tests/GitHubIndexer.test.js +4 -3
- package/dist/tests/GitHubIndexer.test.js.map +1 -1
- package/dist/tests/MemoryProfiler.test.js +3 -3
- package/dist/tests/MemoryProfiler.test.js.map +1 -1
- package/dist/tests/QualityScorer.test.js.map +1 -1
- package/dist/tests/RateLimiter.test.js.map +1 -1
- package/dist/tests/ScraperAdapters.test.js.map +1 -1
- package/dist/tests/SecurityScanner.test.js +4 -4
- package/dist/tests/SecurityScanner.test.js.map +1 -1
- package/dist/tests/SessionManager.security.test.js +1 -1
- package/dist/tests/SessionManager.security.test.js.map +1 -1
- package/dist/tests/SessionManager.test.js +4 -4
- package/dist/tests/SessionManager.test.js.map +1 -1
- package/dist/tests/SwarmIndexer.test.js +1 -1
- package/dist/tests/SwarmIndexer.test.js.map +1 -1
- package/dist/tests/TieredCache.test.js +2 -11
- package/dist/tests/TieredCache.test.js.map +1 -1
- package/dist/tests/WebhookHandler.test.js.map +1 -1
- package/dist/tests/fixtures/api-responses/index.d.ts +119 -0
- package/dist/tests/fixtures/api-responses/index.d.ts.map +1 -0
- package/dist/tests/fixtures/api-responses/index.js +419 -0
- package/dist/tests/fixtures/api-responses/index.js.map +1 -0
- package/dist/tests/integration/neural/neural-fixtures.d.ts +19 -0
- package/dist/tests/integration/neural/neural-fixtures.d.ts.map +1 -0
- package/dist/tests/integration/neural/neural-fixtures.js +49 -0
- package/dist/tests/integration/neural/neural-fixtures.js.map +1 -0
- package/dist/tests/integration/neural/neural-mocks-privacy.d.ts +78 -0
- package/dist/tests/integration/neural/neural-mocks-privacy.d.ts.map +1 -0
- package/dist/tests/integration/neural/neural-mocks-privacy.js +146 -0
- package/dist/tests/integration/neural/neural-mocks-privacy.js.map +1 -0
- package/dist/tests/integration/neural/neural-mocks.d.ts +90 -0
- package/dist/tests/integration/neural/neural-mocks.d.ts.map +1 -0
- package/dist/tests/integration/neural/neural-mocks.js +288 -0
- package/dist/tests/integration/neural/neural-mocks.js.map +1 -0
- package/dist/tests/integration/neural/neural-test-utils.d.ts +27 -0
- package/dist/tests/integration/neural/neural-test-utils.d.ts.map +1 -0
- package/dist/tests/integration/neural/neural-test-utils.js +35 -0
- package/dist/tests/integration/neural/neural-test-utils.js.map +1 -0
- package/dist/tests/integration/neural/setup.d.ts +6 -165
- package/dist/tests/integration/neural/setup.d.ts.map +1 -1
- package/dist/tests/integration/neural/setup.js +9 -477
- package/dist/tests/integration/neural/setup.js.map +1 -1
- package/dist/tests/logger.test.js +1 -1
- package/dist/tests/logger.test.js.map +1 -1
- package/dist/tests/performance/LargeScalePerformance.test.js +1 -1
- package/dist/tests/performance/LargeScalePerformance.test.js.map +1 -1
- package/dist/tests/sanitization.test.js.map +1 -1
- package/dist/tests/session/SessionManager.helpers.test.d.ts +8 -0
- package/dist/tests/session/SessionManager.helpers.test.d.ts.map +1 -0
- package/dist/tests/session/SessionManager.helpers.test.js +105 -0
- package/dist/tests/session/SessionManager.helpers.test.js.map +1 -0
- package/dist/tests/sources.test.js.map +1 -1
- package/dist/tests/webhooks/WebhookHandler.idempotency.test.js +1 -1
- package/dist/tests/webhooks/WebhookHandler.idempotency.test.js.map +1 -1
- package/dist/tests/webhooks/WebhookPayload.security.test.js.map +1 -1
- package/package.json +11 -11
|
@@ -1,210 +1,57 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* SMI-1519: HNSW Embedding Store
|
|
3
3
|
*
|
|
4
|
-
* High-performance vector storage using HNSW
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* Features:
|
|
8
|
-
* - O(log n) similarity search vs O(n) brute-force (150x faster)
|
|
9
|
-
* - SQLite for metadata persistence (skill_id, text, created_at)
|
|
10
|
-
* - Graceful fallback to brute-force if HNSW unavailable
|
|
11
|
-
* - Compatible with existing EmbeddingService interface
|
|
12
|
-
* - Uses claude-flow V3 VectorDB API with automatic fallback
|
|
13
|
-
*
|
|
14
|
-
* Enable via environment variable: SKILLSMITH_USE_HNSW=true
|
|
4
|
+
* High-performance vector storage using HNSW index for fast ANN search.
|
|
5
|
+
* Uses claude-flow V3 VectorDB API with automatic fallback to brute-force.
|
|
15
6
|
*
|
|
7
|
+
* Enable via: SKILLSMITH_USE_HNSW=true
|
|
16
8
|
* @see ADR-009: Embedding Service Fallback Strategy
|
|
17
9
|
*/
|
|
18
10
|
import Database from 'better-sqlite3';
|
|
19
|
-
|
|
20
|
-
//
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
* Tuned for ~10k-100k skills with balanced speed/recall.
|
|
25
|
-
*/
|
|
26
|
-
export const DEFAULT_HNSW_CONFIG = {
|
|
27
|
-
m: 16,
|
|
28
|
-
efConstruction: 200,
|
|
29
|
-
efSearch: 100,
|
|
30
|
-
dimensions: 384, // all-MiniLM-L6-v2
|
|
31
|
-
};
|
|
32
|
-
/**
|
|
33
|
-
* HNSW configuration presets for different use cases
|
|
34
|
-
*/
|
|
35
|
-
export const HNSW_PRESETS = {
|
|
36
|
-
/** Fast search, lower memory, suitable for <10k vectors */
|
|
37
|
-
small: {
|
|
38
|
-
m: 8,
|
|
39
|
-
efConstruction: 100,
|
|
40
|
-
efSearch: 50,
|
|
41
|
-
dimensions: 384,
|
|
42
|
-
},
|
|
43
|
-
/** Balanced performance, suitable for 10k-100k vectors */
|
|
44
|
-
medium: {
|
|
45
|
-
m: 16,
|
|
46
|
-
efConstruction: 200,
|
|
47
|
-
efSearch: 100,
|
|
48
|
-
dimensions: 384,
|
|
49
|
-
},
|
|
50
|
-
/** High recall, suitable for 100k-1M vectors */
|
|
51
|
-
large: {
|
|
52
|
-
m: 32,
|
|
53
|
-
efConstruction: 400,
|
|
54
|
-
efSearch: 150,
|
|
55
|
-
dimensions: 384,
|
|
56
|
-
},
|
|
57
|
-
/** Maximum recall, suitable for >1M vectors or critical accuracy */
|
|
58
|
-
xlarge: {
|
|
59
|
-
m: 48,
|
|
60
|
-
efConstruction: 500,
|
|
61
|
-
efSearch: 200,
|
|
62
|
-
dimensions: 384,
|
|
63
|
-
},
|
|
64
|
-
};
|
|
65
|
-
// ============================================================================
|
|
66
|
-
// Class Skeleton
|
|
67
|
-
// ============================================================================
|
|
11
|
+
export { DEFAULT_HNSW_CONFIG, HNSW_PRESETS } from './hnsw-store.types.js';
|
|
12
|
+
// Re-export factory functions
|
|
13
|
+
export { createHNSWStore, isHNSWAvailable, loadHNSWLib } from './hnsw-store.helpers.js';
|
|
14
|
+
import { DEFAULT_HNSW_CONFIG } from './hnsw-store.types.js';
|
|
15
|
+
import { shouldUseHNSW, validateDimensions, estimateMemoryUsage } from './hnsw-store.helpers.js';
|
|
68
16
|
/**
|
|
69
17
|
* High-performance embedding storage using HNSW index.
|
|
70
|
-
*
|
|
71
|
-
* Provides O(log n) approximate nearest neighbor search while maintaining
|
|
72
|
-
* compatibility with the existing EmbeddingService interface.
|
|
73
|
-
*
|
|
74
|
-
* @example
|
|
75
|
-
* ```typescript
|
|
76
|
-
* // Basic usage
|
|
77
|
-
* const store = new HNSWEmbeddingStore({
|
|
78
|
-
* dbPath: './embeddings.db',
|
|
79
|
-
* indexPath: './embeddings.hnsw',
|
|
80
|
-
* });
|
|
81
|
-
*
|
|
82
|
-
* // Store embeddings
|
|
83
|
-
* store.storeEmbedding('skill-1', embedding1, 'Jest testing framework helper');
|
|
84
|
-
* store.storeEmbedding('skill-2', embedding2, 'Vitest testing utilities');
|
|
85
|
-
*
|
|
86
|
-
* // Find similar
|
|
87
|
-
* const results = store.findSimilar(queryEmbedding, 10);
|
|
88
|
-
* // [{ skillId: 'skill-1', score: 0.95 }, { skillId: 'skill-2', score: 0.87 }, ...]
|
|
89
|
-
*
|
|
90
|
-
* // Clean up
|
|
91
|
-
* store.close();
|
|
92
|
-
* ```
|
|
93
|
-
*
|
|
94
|
-
* @example
|
|
95
|
-
* ```typescript
|
|
96
|
-
* // With custom HNSW config for large dataset
|
|
97
|
-
* const store = new HNSWEmbeddingStore({
|
|
98
|
-
* dbPath: './embeddings.db',
|
|
99
|
-
* indexPath: './embeddings.hnsw',
|
|
100
|
-
* hnswConfig: HNSW_PRESETS.large,
|
|
101
|
-
* maxElements: 500000,
|
|
102
|
-
* });
|
|
103
|
-
* ```
|
|
18
|
+
* Provides O(log n) approximate nearest neighbor search with SQLite persistence.
|
|
104
19
|
*/
|
|
105
20
|
export class HNSWEmbeddingStore {
|
|
106
|
-
// -------------------------------------------------------------------------
|
|
107
|
-
// Private Fields
|
|
108
|
-
// -------------------------------------------------------------------------
|
|
109
|
-
/** SQLite database for metadata */
|
|
110
21
|
db = null;
|
|
111
|
-
/** HNSW index instance (from hnswlib-node) */
|
|
112
22
|
index = null;
|
|
113
|
-
/** Whether HNSW is enabled (false = brute-force fallback) */
|
|
114
23
|
hnswEnabled;
|
|
115
|
-
/** Merged HNSW configuration */
|
|
116
24
|
config;
|
|
117
|
-
/** Maximum index capacity */
|
|
118
25
|
maxElements;
|
|
119
|
-
/** Path to HNSW index file */
|
|
120
26
|
indexPath;
|
|
121
|
-
/** Distance metric */
|
|
122
27
|
distanceMetric;
|
|
123
|
-
/** Auto-save flag */
|
|
124
28
|
autoSave;
|
|
125
|
-
/** Map of skill IDs to internal HNSW labels (for reverse lookup) */
|
|
126
29
|
skillIdToLabel = new Map();
|
|
127
|
-
/** Map of HNSW labels to skill IDs */
|
|
128
30
|
labelToSkillId = new Map();
|
|
129
|
-
/** Next available label for HNSW insertion */
|
|
130
31
|
nextLabel = 0;
|
|
131
|
-
// -------------------------------------------------------------------------
|
|
132
|
-
// Constructor
|
|
133
|
-
// -------------------------------------------------------------------------
|
|
134
|
-
/**
|
|
135
|
-
* Create a new HNSWEmbeddingStore instance.
|
|
136
|
-
*
|
|
137
|
-
* @param options - Configuration options
|
|
138
|
-
*
|
|
139
|
-
* @example
|
|
140
|
-
* ```typescript
|
|
141
|
-
* // Default configuration (auto-detects HNSW availability)
|
|
142
|
-
* const store = new HNSWEmbeddingStore();
|
|
143
|
-
*
|
|
144
|
-
* // With persistence
|
|
145
|
-
* const store = new HNSWEmbeddingStore({
|
|
146
|
-
* dbPath: './embeddings.db',
|
|
147
|
-
* indexPath: './embeddings.hnsw',
|
|
148
|
-
* });
|
|
149
|
-
*
|
|
150
|
-
* // Force brute-force fallback
|
|
151
|
-
* const store = new HNSWEmbeddingStore({ useHNSW: false });
|
|
152
|
-
* ```
|
|
153
|
-
*/
|
|
154
|
-
/** V3 VectorDB instance (if initialized) */
|
|
155
32
|
vectorDB = null;
|
|
156
|
-
/** Promise for async initialization */
|
|
157
33
|
initPromise = null;
|
|
158
34
|
constructor(options = {}) {
|
|
159
|
-
|
|
160
|
-
this.
|
|
161
|
-
// Merge configuration with defaults
|
|
162
|
-
this.config = {
|
|
163
|
-
...DEFAULT_HNSW_CONFIG,
|
|
164
|
-
...options.hnswConfig,
|
|
165
|
-
};
|
|
35
|
+
this.hnswEnabled = shouldUseHNSW(options.useHNSW);
|
|
36
|
+
this.config = { ...DEFAULT_HNSW_CONFIG, ...options.hnswConfig };
|
|
166
37
|
this.maxElements = options.maxElements ?? 100000;
|
|
167
38
|
this.indexPath = options.indexPath;
|
|
168
39
|
this.distanceMetric = options.distanceMetric ?? 'cosine';
|
|
169
40
|
this.autoSave = options.autoSave ?? false;
|
|
170
|
-
// Initialize SQLite database
|
|
171
41
|
if (options.dbPath) {
|
|
172
42
|
this.initDatabase(options.dbPath);
|
|
173
43
|
}
|
|
174
|
-
//
|
|
44
|
+
// IMPORTANT: Keep dynamic import here for V3 lazy loading / graceful degradation
|
|
175
45
|
if (this.hnswEnabled) {
|
|
176
46
|
this.initPromise = this.initHNSWIndex();
|
|
177
47
|
}
|
|
178
48
|
}
|
|
179
|
-
/**
|
|
180
|
-
* Ensure the store is fully initialized.
|
|
181
|
-
* Call this before operations that require the HNSW index.
|
|
182
|
-
*/
|
|
183
49
|
async ensureInitialized() {
|
|
184
|
-
if (this.initPromise)
|
|
50
|
+
if (this.initPromise)
|
|
185
51
|
await this.initPromise;
|
|
186
|
-
}
|
|
187
52
|
}
|
|
188
|
-
// -------------------------------------------------------------------------
|
|
189
|
-
// Public Methods (IEmbeddingStore Interface)
|
|
190
|
-
// -------------------------------------------------------------------------
|
|
191
|
-
/**
|
|
192
|
-
* Store an embedding with its metadata.
|
|
193
|
-
*
|
|
194
|
-
* Inserts the vector into both HNSW index (for fast search) and
|
|
195
|
-
* SQLite (for metadata persistence).
|
|
196
|
-
*
|
|
197
|
-
* @param skillId - Unique identifier for the skill
|
|
198
|
-
* @param embedding - Vector embedding (must match configured dimensions)
|
|
199
|
-
* @param text - Original text that was embedded
|
|
200
|
-
* @throws Error if embedding dimensions don't match configuration
|
|
201
|
-
*/
|
|
202
53
|
storeEmbedding(skillId, embedding, text) {
|
|
203
|
-
|
|
204
|
-
if (embedding.length !== this.config.dimensions) {
|
|
205
|
-
throw new Error(`Embedding dimension mismatch: got ${embedding.length}, expected ${this.config.dimensions}`);
|
|
206
|
-
}
|
|
207
|
-
// Store in SQLite (metadata + embedding blob for brute-force fallback)
|
|
54
|
+
validateDimensions(embedding, this.config.dimensions, 'Embedding');
|
|
208
55
|
if (this.db) {
|
|
209
56
|
const buffer = Buffer.from(embedding.buffer);
|
|
210
57
|
const stmt = this.db.prepare(`
|
|
@@ -213,49 +60,28 @@ export class HNSWEmbeddingStore {
|
|
|
213
60
|
`);
|
|
214
61
|
stmt.run(skillId, buffer, text);
|
|
215
62
|
}
|
|
216
|
-
// Insert into V3 VectorDB (HNSW index)
|
|
217
63
|
if (this.vectorDB) {
|
|
218
|
-
// VectorDB.insert may be sync or async depending on backend
|
|
219
64
|
const result = this.vectorDB.insert(embedding, skillId, { text });
|
|
220
65
|
if (result instanceof Promise) {
|
|
221
|
-
// Fire and forget for sync interface, but log errors
|
|
222
66
|
result.catch((err) => {
|
|
223
67
|
console.warn(`[HNSWEmbeddingStore] Failed to insert into VectorDB: ${err}`);
|
|
224
68
|
});
|
|
225
69
|
}
|
|
226
70
|
}
|
|
227
71
|
}
|
|
228
|
-
/**
|
|
229
|
-
* Retrieve a stored embedding by skill ID.
|
|
230
|
-
*
|
|
231
|
-
* @param skillId - Unique identifier for the skill
|
|
232
|
-
* @returns The embedding if found, null otherwise
|
|
233
|
-
*/
|
|
234
72
|
getEmbedding(skillId) {
|
|
235
73
|
if (!this.db)
|
|
236
74
|
return null;
|
|
237
|
-
const stmt = this.db.prepare(
|
|
238
|
-
SELECT embedding FROM skill_embeddings WHERE skill_id = ?
|
|
239
|
-
`);
|
|
75
|
+
const stmt = this.db.prepare('SELECT embedding FROM skill_embeddings WHERE skill_id = ?');
|
|
240
76
|
const row = stmt.get(skillId);
|
|
241
77
|
if (!row)
|
|
242
78
|
return null;
|
|
243
79
|
return new Float32Array(row.embedding.buffer.slice(row.embedding.byteOffset, row.embedding.byteOffset + row.embedding.byteLength));
|
|
244
80
|
}
|
|
245
|
-
/**
|
|
246
|
-
* Get all stored embeddings.
|
|
247
|
-
*
|
|
248
|
-
* Note: For large datasets, consider using findSimilar() instead
|
|
249
|
-
* to avoid loading all vectors into memory.
|
|
250
|
-
*
|
|
251
|
-
* @returns Map of skill IDs to their embeddings
|
|
252
|
-
*/
|
|
253
81
|
getAllEmbeddings() {
|
|
254
82
|
if (!this.db)
|
|
255
83
|
return new Map();
|
|
256
|
-
const stmt = this.db.prepare(
|
|
257
|
-
SELECT skill_id, embedding FROM skill_embeddings
|
|
258
|
-
`);
|
|
84
|
+
const stmt = this.db.prepare('SELECT skill_id, embedding FROM skill_embeddings');
|
|
259
85
|
const rows = stmt.all();
|
|
260
86
|
const result = new Map();
|
|
261
87
|
for (const row of rows) {
|
|
@@ -264,102 +90,51 @@ export class HNSWEmbeddingStore {
|
|
|
264
90
|
}
|
|
265
91
|
return result;
|
|
266
92
|
}
|
|
267
|
-
/**
|
|
268
|
-
* Find most similar embeddings to a query vector.
|
|
269
|
-
*
|
|
270
|
-
* Uses HNSW for O(log n) approximate search when available,
|
|
271
|
-
* falls back to O(n) brute-force cosine similarity otherwise.
|
|
272
|
-
*
|
|
273
|
-
* @param queryEmbedding - Query vector (must match configured dimensions)
|
|
274
|
-
* @param topK - Number of results to return (default: 10)
|
|
275
|
-
* @returns Array of skill IDs with similarity scores, sorted descending
|
|
276
|
-
*/
|
|
277
93
|
findSimilar(queryEmbedding, topK = 10) {
|
|
278
|
-
|
|
279
|
-
if (queryEmbedding.length !== this.config.dimensions) {
|
|
280
|
-
throw new Error(`Query dimension mismatch: got ${queryEmbedding.length}, expected ${this.config.dimensions}`);
|
|
281
|
-
}
|
|
282
|
-
// Try HNSW search first (if available)
|
|
94
|
+
validateDimensions(queryEmbedding, this.config.dimensions);
|
|
283
95
|
if (this.vectorDB) {
|
|
284
96
|
try {
|
|
285
|
-
// VectorDB.search may be sync or async
|
|
286
97
|
const searchResult = this.vectorDB.search(queryEmbedding, topK);
|
|
287
|
-
// Handle async case by returning empty and logging
|
|
288
|
-
// (sync interface limitation - caller should use findSimilarAsync for async)
|
|
289
98
|
if (searchResult instanceof Promise) {
|
|
290
|
-
|
|
291
|
-
console.warn('[HNSWEmbeddingStore] VectorDB.search returned Promise, using brute-force fallback');
|
|
99
|
+
console.warn('[HNSWEmbeddingStore] VectorDB.search returned Promise, using brute-force');
|
|
292
100
|
}
|
|
293
101
|
else {
|
|
294
|
-
|
|
295
|
-
return searchResult.map((result) => ({
|
|
296
|
-
skillId: result.id,
|
|
297
|
-
score: result.score,
|
|
298
|
-
}));
|
|
102
|
+
return searchResult.map((r) => ({ skillId: r.id, score: r.score }));
|
|
299
103
|
}
|
|
300
104
|
}
|
|
301
105
|
catch (err) {
|
|
302
|
-
console.warn(`[HNSWEmbeddingStore] HNSW search failed, falling back
|
|
106
|
+
console.warn(`[HNSWEmbeddingStore] HNSW search failed, falling back: ${err}`);
|
|
303
107
|
}
|
|
304
108
|
}
|
|
305
|
-
// Brute-force fallback
|
|
109
|
+
// Brute-force fallback
|
|
306
110
|
const allEmbeddings = this.getAllEmbeddings();
|
|
307
111
|
const results = [];
|
|
308
112
|
for (const [skillId, embedding] of allEmbeddings) {
|
|
309
|
-
|
|
310
|
-
results.push({ skillId, score });
|
|
113
|
+
results.push({ skillId, score: this.cosineSimilarity(queryEmbedding, embedding) });
|
|
311
114
|
}
|
|
312
|
-
// Sort by similarity score descending and return topK
|
|
313
115
|
results.sort((a, b) => b.score - a.score);
|
|
314
116
|
return results.slice(0, topK);
|
|
315
117
|
}
|
|
316
|
-
/**
|
|
317
|
-
* Async version of findSimilar for backends that require async search.
|
|
318
|
-
*
|
|
319
|
-
* @param queryEmbedding - Query vector (must match configured dimensions)
|
|
320
|
-
* @param topK - Number of results to return (default: 10)
|
|
321
|
-
* @returns Promise resolving to array of skill IDs with similarity scores
|
|
322
|
-
*/
|
|
323
118
|
async findSimilarAsync(queryEmbedding, topK = 10) {
|
|
324
|
-
// Ensure HNSW is initialized
|
|
325
119
|
await this.ensureInitialized();
|
|
326
|
-
|
|
327
|
-
if (queryEmbedding.length !== this.config.dimensions) {
|
|
328
|
-
throw new Error(`Query dimension mismatch: got ${queryEmbedding.length}, expected ${this.config.dimensions}`);
|
|
329
|
-
}
|
|
330
|
-
// Try HNSW search first (if available)
|
|
120
|
+
validateDimensions(queryEmbedding, this.config.dimensions);
|
|
331
121
|
if (this.vectorDB) {
|
|
332
122
|
try {
|
|
333
123
|
const searchResult = this.vectorDB.search(queryEmbedding, topK);
|
|
334
124
|
const results = searchResult instanceof Promise ? await searchResult : searchResult;
|
|
335
|
-
return results.map((
|
|
336
|
-
skillId: result.id,
|
|
337
|
-
score: result.score,
|
|
338
|
-
}));
|
|
125
|
+
return results.map((r) => ({ skillId: r.id, score: r.score }));
|
|
339
126
|
}
|
|
340
127
|
catch (err) {
|
|
341
|
-
console.warn(`[HNSWEmbeddingStore] HNSW search failed, falling back
|
|
128
|
+
console.warn(`[HNSWEmbeddingStore] HNSW search failed, falling back: ${err}`);
|
|
342
129
|
}
|
|
343
130
|
}
|
|
344
|
-
// Brute-force fallback
|
|
345
131
|
return this.findSimilar(queryEmbedding, topK);
|
|
346
132
|
}
|
|
347
|
-
/**
|
|
348
|
-
* Compute cosine similarity between two embeddings.
|
|
349
|
-
*
|
|
350
|
-
* @param a - First embedding
|
|
351
|
-
* @param b - Second embedding
|
|
352
|
-
* @returns Similarity score between -1 and 1
|
|
353
|
-
* @throws Error if embeddings have different dimensions
|
|
354
|
-
*/
|
|
355
133
|
cosineSimilarity(a, b) {
|
|
356
134
|
if (a.length !== b.length) {
|
|
357
|
-
throw new Error(`Embedding dimension mismatch: ${a.length} vs ${b.length}
|
|
358
|
-
`Expected ${this.config.dimensions}.`);
|
|
135
|
+
throw new Error(`Embedding dimension mismatch: ${a.length} vs ${b.length}`);
|
|
359
136
|
}
|
|
360
|
-
let dotProduct = 0;
|
|
361
|
-
let normA = 0;
|
|
362
|
-
let normB = 0;
|
|
137
|
+
let dotProduct = 0, normA = 0, normB = 0;
|
|
363
138
|
for (let i = 0; i < a.length; i++) {
|
|
364
139
|
dotProduct += a[i] * b[i];
|
|
365
140
|
normA += a[i] * a[i];
|
|
@@ -369,85 +144,44 @@ export class HNSWEmbeddingStore {
|
|
|
369
144
|
return 0;
|
|
370
145
|
return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
371
146
|
}
|
|
372
|
-
/**
|
|
373
|
-
* Check if running in fallback (brute-force) mode.
|
|
374
|
-
*
|
|
375
|
-
* @returns true if using brute-force, false if using HNSW
|
|
376
|
-
*/
|
|
377
147
|
isUsingFallback() {
|
|
378
148
|
return !this.hnswEnabled || this.index === null;
|
|
379
149
|
}
|
|
380
|
-
/**
|
|
381
|
-
* Close database connections and release resources.
|
|
382
|
-
*
|
|
383
|
-
* Saves HNSW index to disk if indexPath was configured.
|
|
384
|
-
* Safe to call multiple times.
|
|
385
|
-
*/
|
|
386
150
|
close() {
|
|
387
|
-
// TODO: SMI-1519 - Implement
|
|
388
|
-
// 1. Save HNSW index if indexPath configured
|
|
389
|
-
// 2. Close SQLite database
|
|
390
|
-
// 3. Clear label mappings
|
|
391
151
|
if (this.db) {
|
|
392
152
|
this.db.close();
|
|
393
153
|
this.db = null;
|
|
394
154
|
}
|
|
395
155
|
}
|
|
396
|
-
// -------------------------------------------------------------------------
|
|
397
|
-
// Extended Public Methods (HNSW-specific)
|
|
398
|
-
// -------------------------------------------------------------------------
|
|
399
|
-
/**
|
|
400
|
-
* Get statistics about the HNSW index.
|
|
401
|
-
*
|
|
402
|
-
* @returns Index statistics including capacity, utilization, and config
|
|
403
|
-
*/
|
|
404
156
|
getStats() {
|
|
405
157
|
let vectorCount = 0;
|
|
406
|
-
// Get count from SQLite
|
|
407
158
|
if (this.db) {
|
|
408
159
|
const stmt = this.db.prepare('SELECT COUNT(*) as count FROM skill_embeddings');
|
|
409
|
-
|
|
410
|
-
vectorCount = row.count;
|
|
160
|
+
vectorCount = stmt.get().count;
|
|
411
161
|
}
|
|
412
|
-
// Get count from VectorDB if available
|
|
413
162
|
if (this.vectorDB) {
|
|
414
163
|
try {
|
|
415
164
|
const size = this.vectorDB.size();
|
|
416
|
-
if (!(size instanceof Promise))
|
|
165
|
+
if (!(size instanceof Promise))
|
|
417
166
|
vectorCount = Math.max(vectorCount, size);
|
|
418
|
-
}
|
|
419
167
|
}
|
|
420
168
|
catch {
|
|
421
|
-
|
|
169
|
+
/* ignore */
|
|
422
170
|
}
|
|
423
171
|
}
|
|
424
|
-
const utilizationPercent = this.maxElements > 0 ? (vectorCount / this.maxElements) * 100 : 0;
|
|
425
|
-
// Estimate memory usage (rough approximation)
|
|
426
|
-
// HNSW uses ~(4 * dimensions + M * 4 * 2) bytes per vector
|
|
427
|
-
const bytesPerVector = 4 * this.config.dimensions + this.config.m * 8;
|
|
428
|
-
const memoryUsageBytes = vectorCount * bytesPerVector;
|
|
429
172
|
return {
|
|
430
173
|
vectorCount,
|
|
431
174
|
maxCapacity: this.maxElements,
|
|
432
|
-
utilizationPercent: Math.round(
|
|
175
|
+
utilizationPercent: Math.round((vectorCount / this.maxElements) * 10000) / 100,
|
|
433
176
|
m: this.config.m,
|
|
434
177
|
efConstruction: this.config.efConstruction,
|
|
435
178
|
efSearch: this.config.efSearch,
|
|
436
179
|
dimensions: this.config.dimensions,
|
|
437
|
-
memoryUsageBytes,
|
|
180
|
+
memoryUsageBytes: estimateMemoryUsage(vectorCount, this.config.dimensions, this.config.m),
|
|
438
181
|
isHNSWEnabled: this.vectorDB !== null,
|
|
439
182
|
indexPath: this.indexPath,
|
|
440
183
|
};
|
|
441
184
|
}
|
|
442
|
-
/**
|
|
443
|
-
* Batch insert multiple embeddings efficiently.
|
|
444
|
-
*
|
|
445
|
-
* More efficient than calling storeEmbedding() in a loop
|
|
446
|
-
* due to batched SQLite transactions and HNSW insertions.
|
|
447
|
-
*
|
|
448
|
-
* @param embeddings - Array of embeddings to insert
|
|
449
|
-
* @returns Batch operation result with counts and timing
|
|
450
|
-
*/
|
|
451
185
|
batchInsert(embeddings) {
|
|
452
186
|
const startTime = Date.now();
|
|
453
187
|
const result = {
|
|
@@ -462,54 +196,37 @@ export class HNSWEmbeddingStore {
|
|
|
462
196
|
result.durationMs = Date.now() - startTime;
|
|
463
197
|
return result;
|
|
464
198
|
}
|
|
465
|
-
// Use a transaction for batch SQLite operations
|
|
466
199
|
const insertStmt = this.db.prepare(`
|
|
467
200
|
INSERT OR REPLACE INTO skill_embeddings (skill_id, embedding, text, created_at)
|
|
468
201
|
VALUES (?, ?, ?, unixepoch())
|
|
469
202
|
`);
|
|
470
|
-
const checkStmt = this.db.prepare(
|
|
471
|
-
SELECT 1 FROM skill_embeddings WHERE skill_id = ?
|
|
472
|
-
`);
|
|
203
|
+
const checkStmt = this.db.prepare('SELECT 1 FROM skill_embeddings WHERE skill_id = ?');
|
|
473
204
|
const transaction = this.db.transaction(() => {
|
|
474
205
|
for (const { skillId, embedding, text } of embeddings) {
|
|
475
206
|
try {
|
|
476
|
-
// Validate dimensions
|
|
477
207
|
if (embedding.length !== this.config.dimensions) {
|
|
478
208
|
result.failed++;
|
|
479
|
-
result.errors.push({
|
|
480
|
-
skillId,
|
|
481
|
-
error: `Dimension mismatch: got ${embedding.length}, expected ${this.config.dimensions}`,
|
|
482
|
-
});
|
|
209
|
+
result.errors.push({ skillId, error: `Dimension mismatch: ${embedding.length}` });
|
|
483
210
|
continue;
|
|
484
211
|
}
|
|
485
|
-
// Check if exists (for updated count)
|
|
486
212
|
const exists = checkStmt.get(skillId);
|
|
487
|
-
|
|
488
|
-
const buffer = Buffer.from(embedding.buffer);
|
|
489
|
-
insertStmt.run(skillId, buffer, text);
|
|
490
|
-
// Insert into VectorDB
|
|
213
|
+
insertStmt.run(skillId, Buffer.from(embedding.buffer), text);
|
|
491
214
|
if (this.vectorDB) {
|
|
492
215
|
try {
|
|
493
216
|
this.vectorDB.insert(embedding, skillId, { text });
|
|
494
217
|
}
|
|
495
218
|
catch (err) {
|
|
496
|
-
|
|
497
|
-
console.warn(`[HNSWEmbeddingStore] VectorDB insert failed for ${skillId}: ${err}`);
|
|
219
|
+
console.warn(`VectorDB insert failed for ${skillId}: ${err}`);
|
|
498
220
|
}
|
|
499
221
|
}
|
|
500
|
-
if (exists)
|
|
222
|
+
if (exists)
|
|
501
223
|
result.updated++;
|
|
502
|
-
|
|
503
|
-
else {
|
|
224
|
+
else
|
|
504
225
|
result.inserted++;
|
|
505
|
-
}
|
|
506
226
|
}
|
|
507
227
|
catch (err) {
|
|
508
228
|
result.failed++;
|
|
509
|
-
result.errors.push({
|
|
510
|
-
skillId,
|
|
511
|
-
error: err instanceof Error ? err.message : String(err),
|
|
512
|
-
});
|
|
229
|
+
result.errors.push({ skillId, error: err instanceof Error ? err.message : String(err) });
|
|
513
230
|
}
|
|
514
231
|
}
|
|
515
232
|
});
|
|
@@ -517,207 +234,106 @@ export class HNSWEmbeddingStore {
|
|
|
517
234
|
result.durationMs = Date.now() - startTime;
|
|
518
235
|
return result;
|
|
519
236
|
}
|
|
520
|
-
/**
|
|
521
|
-
* Remove an embedding from the store.
|
|
522
|
-
*
|
|
523
|
-
* Note: HNSW does not support true deletion. The vector is marked
|
|
524
|
-
* as deleted and excluded from search results, but memory is not
|
|
525
|
-
* reclaimed until the index is rebuilt.
|
|
526
|
-
*
|
|
527
|
-
* @param skillId - Unique identifier for the skill to remove
|
|
528
|
-
* @returns true if removed, false if not found
|
|
529
|
-
*/
|
|
530
237
|
removeEmbedding(skillId) {
|
|
531
238
|
let removed = false;
|
|
532
|
-
// Remove from SQLite
|
|
533
239
|
if (this.db) {
|
|
534
240
|
const stmt = this.db.prepare('DELETE FROM skill_embeddings WHERE skill_id = ?');
|
|
535
|
-
|
|
536
|
-
removed = result.changes > 0;
|
|
241
|
+
removed = stmt.run(skillId).changes > 0;
|
|
537
242
|
}
|
|
538
|
-
// Remove from VectorDB (if supported)
|
|
539
243
|
if (this.vectorDB && removed) {
|
|
540
244
|
try {
|
|
541
245
|
const vdbResult = this.vectorDB.remove(skillId);
|
|
542
|
-
// VectorDB.remove may be sync or async
|
|
543
246
|
if (vdbResult instanceof Promise) {
|
|
544
|
-
vdbResult.catch((err) => {
|
|
545
|
-
console.warn(`[HNSWEmbeddingStore] VectorDB remove failed for ${skillId}: ${err}`);
|
|
546
|
-
});
|
|
247
|
+
vdbResult.catch((err) => console.warn(`VectorDB remove failed: ${err}`));
|
|
547
248
|
}
|
|
548
249
|
}
|
|
549
250
|
catch (err) {
|
|
550
|
-
|
|
551
|
-
console.warn(`[HNSWEmbeddingStore] VectorDB remove failed for ${skillId}: ${err}`);
|
|
251
|
+
console.warn(`VectorDB remove failed: ${err}`);
|
|
552
252
|
}
|
|
553
253
|
}
|
|
554
254
|
return removed;
|
|
555
255
|
}
|
|
556
|
-
/**
|
|
557
|
-
* Save the HNSW index to disk.
|
|
558
|
-
*
|
|
559
|
-
* Note: V3 VectorDB manages its own persistence, so this is a no-op
|
|
560
|
-
* unless using hnswlib-node directly.
|
|
561
|
-
*
|
|
562
|
-
* @throws Error if indexPath was not configured
|
|
563
|
-
*/
|
|
564
256
|
saveIndex() {
|
|
565
|
-
if (!this.indexPath)
|
|
257
|
+
if (!this.indexPath)
|
|
566
258
|
throw new Error('Cannot save index: indexPath not configured');
|
|
567
|
-
|
|
568
|
-
// V3 VectorDB handles its own persistence
|
|
569
|
-
// For hnswlib-node, we would call index.saveIndex(this.indexPath)
|
|
570
|
-
console.log(`[HNSWEmbeddingStore] Index persistence managed by V3 VectorDB backend`);
|
|
259
|
+
console.log('[HNSWEmbeddingStore] Index persistence managed by V3 VectorDB backend');
|
|
571
260
|
}
|
|
572
|
-
/**
|
|
573
|
-
* Load the HNSW index from disk.
|
|
574
|
-
*
|
|
575
|
-
* Note: V3 VectorDB manages its own persistence, so this is a no-op
|
|
576
|
-
* unless using hnswlib-node directly.
|
|
577
|
-
*
|
|
578
|
-
* @throws Error if indexPath was not configured or file doesn't exist
|
|
579
|
-
*/
|
|
580
261
|
loadIndex() {
|
|
581
|
-
if (!this.indexPath)
|
|
262
|
+
if (!this.indexPath)
|
|
582
263
|
throw new Error('Cannot load index: indexPath not configured');
|
|
583
|
-
|
|
584
|
-
// V3 VectorDB handles its own persistence
|
|
585
|
-
// For hnswlib-node, we would call index.loadIndex(this.indexPath)
|
|
586
|
-
console.log(`[HNSWEmbeddingStore] Index persistence managed by V3 VectorDB backend`);
|
|
264
|
+
console.log('[HNSWEmbeddingStore] Index persistence managed by V3 VectorDB backend');
|
|
587
265
|
}
|
|
588
|
-
/**
|
|
589
|
-
* Rebuild the HNSW index from SQLite data.
|
|
590
|
-
*
|
|
591
|
-
* Useful after many deletions to reclaim memory, or to apply
|
|
592
|
-
* new HNSW configuration parameters.
|
|
593
|
-
*
|
|
594
|
-
* @param newConfig - Optional new HNSW configuration
|
|
595
|
-
*/
|
|
596
266
|
async rebuildIndex(newConfig) {
|
|
597
|
-
|
|
598
|
-
if (newConfig) {
|
|
267
|
+
if (newConfig)
|
|
599
268
|
Object.assign(this.config, newConfig);
|
|
600
|
-
}
|
|
601
|
-
// Clear existing VectorDB
|
|
602
269
|
if (this.vectorDB) {
|
|
603
270
|
try {
|
|
604
271
|
const clearResult = this.vectorDB.clear();
|
|
605
|
-
if (clearResult instanceof Promise)
|
|
272
|
+
if (clearResult instanceof Promise)
|
|
606
273
|
await clearResult;
|
|
607
|
-
}
|
|
608
274
|
}
|
|
609
275
|
catch (err) {
|
|
610
|
-
console.warn(`
|
|
276
|
+
console.warn(`Failed to clear VectorDB: ${err}`);
|
|
611
277
|
}
|
|
612
278
|
}
|
|
613
|
-
// Reinitialize VectorDB
|
|
614
279
|
await this.initHNSWIndex();
|
|
615
|
-
// Re-insert all embeddings from SQLite
|
|
616
280
|
if (this.db && this.vectorDB) {
|
|
617
281
|
const allEmbeddings = this.getAllEmbeddings();
|
|
618
282
|
for (const [skillId, embedding] of allEmbeddings) {
|
|
619
283
|
try {
|
|
620
284
|
const result = this.vectorDB.insert(embedding, skillId);
|
|
621
|
-
if (result instanceof Promise)
|
|
285
|
+
if (result instanceof Promise)
|
|
622
286
|
await result;
|
|
623
|
-
}
|
|
624
287
|
}
|
|
625
288
|
catch (err) {
|
|
626
|
-
console.warn(`
|
|
289
|
+
console.warn(`Failed to reinsert ${skillId}: ${err}`);
|
|
627
290
|
}
|
|
628
291
|
}
|
|
629
292
|
}
|
|
630
293
|
}
|
|
631
|
-
/**
|
|
632
|
-
* Update efSearch parameter at runtime.
|
|
633
|
-
*
|
|
634
|
-
* Note: V3 VectorDB does not expose efSearch tuning directly.
|
|
635
|
-
* This method is provided for API compatibility.
|
|
636
|
-
*
|
|
637
|
-
* @param efSearch - New efSearch value (must be > 0)
|
|
638
|
-
*/
|
|
639
294
|
setEfSearch(efSearch) {
|
|
640
|
-
if (efSearch <= 0)
|
|
295
|
+
if (efSearch <= 0)
|
|
641
296
|
throw new Error('efSearch must be > 0');
|
|
642
|
-
}
|
|
643
297
|
this.config.efSearch = efSearch;
|
|
644
|
-
|
|
645
|
-
// For hnswlib-node, we would call index.setEfSearch(efSearch)
|
|
646
|
-
console.log(`[HNSWEmbeddingStore] efSearch updated to ${efSearch} (will apply on next search)`);
|
|
298
|
+
console.log(`[HNSWEmbeddingStore] efSearch updated to ${efSearch}`);
|
|
647
299
|
}
|
|
648
|
-
//
|
|
649
|
-
// Private Methods
|
|
650
|
-
// -------------------------------------------------------------------------
|
|
651
|
-
/**
|
|
652
|
-
* Determine whether to use HNSW based on explicit option or environment.
|
|
653
|
-
*/
|
|
654
|
-
shouldUseHNSW(explicit) {
|
|
655
|
-
if (explicit !== undefined) {
|
|
656
|
-
return explicit;
|
|
657
|
-
}
|
|
658
|
-
// Check environment variable
|
|
659
|
-
const envValue = process.env.SKILLSMITH_USE_HNSW;
|
|
660
|
-
if (envValue !== undefined) {
|
|
661
|
-
return envValue === 'true' || envValue === '1';
|
|
662
|
-
}
|
|
663
|
-
// Default to false (use brute-force) for backward compatibility
|
|
664
|
-
// TODO: Consider changing default to true in future version
|
|
665
|
-
return false;
|
|
666
|
-
}
|
|
667
|
-
/**
|
|
668
|
-
* Initialize SQLite database and create tables.
|
|
669
|
-
*/
|
|
300
|
+
// Private methods
|
|
670
301
|
initDatabase(dbPath) {
|
|
671
302
|
this.db = new Database(dbPath);
|
|
672
|
-
// Create skill_embeddings table
|
|
673
303
|
this.db.exec(`
|
|
674
304
|
CREATE TABLE IF NOT EXISTS skill_embeddings (
|
|
675
305
|
skill_id TEXT PRIMARY KEY,
|
|
676
306
|
embedding BLOB NOT NULL,
|
|
677
307
|
text TEXT NOT NULL,
|
|
678
308
|
created_at INTEGER DEFAULT (unixepoch())
|
|
679
|
-
)
|
|
680
|
-
|
|
681
|
-
// Create index for fast lookups
|
|
682
|
-
this.db.exec(`
|
|
683
|
-
CREATE INDEX IF NOT EXISTS idx_skill_embeddings_id
|
|
684
|
-
ON skill_embeddings(skill_id)
|
|
309
|
+
);
|
|
310
|
+
CREATE INDEX IF NOT EXISTS idx_skill_embeddings_id ON skill_embeddings(skill_id);
|
|
685
311
|
`);
|
|
686
312
|
}
|
|
687
|
-
|
|
688
|
-
* Initialize HNSW index using V3 VectorDB API.
|
|
689
|
-
* Falls back gracefully if V3 is unavailable.
|
|
690
|
-
*/
|
|
313
|
+
// IMPORTANT: Keep dynamic import here for V3 lazy loading / graceful degradation
|
|
691
314
|
async initHNSWIndex() {
|
|
692
315
|
try {
|
|
693
|
-
// Dynamically import V3 VectorDB module
|
|
694
316
|
const vectorDbModule = await import('claude-flow/v3/@claude-flow/cli/dist/src/ruvector/vector-db.js');
|
|
695
|
-
// Load ruvector backend (may use WASM acceleration)
|
|
696
317
|
const loaded = await vectorDbModule.loadRuVector();
|
|
697
|
-
if (!loaded)
|
|
698
|
-
console.warn('[HNSWEmbeddingStore] ruvector not available
|
|
699
|
-
}
|
|
700
|
-
// Create VectorDB instance
|
|
318
|
+
if (!loaded)
|
|
319
|
+
console.warn('[HNSWEmbeddingStore] ruvector not available');
|
|
701
320
|
this.vectorDB = await vectorDbModule.createVectorDB(this.config.dimensions);
|
|
702
|
-
// Log status
|
|
703
321
|
const status = vectorDbModule.getStatus();
|
|
704
|
-
console.log(`[HNSWEmbeddingStore] Initialized
|
|
705
|
-
|
|
706
|
-
// Re-populate VectorDB from SQLite if we have existing data
|
|
322
|
+
console.log(`[HNSWEmbeddingStore] Initialized: ${status.backend}${status.wasmAccelerated ? ' (WASM)' : ''}`);
|
|
323
|
+
// Rebuild from SQLite
|
|
707
324
|
if (this.db) {
|
|
708
325
|
const count = this.db.prepare('SELECT COUNT(*) as c FROM skill_embeddings').get();
|
|
709
326
|
if (count.c > 0) {
|
|
710
|
-
console.log(`[HNSWEmbeddingStore] Rebuilding
|
|
327
|
+
console.log(`[HNSWEmbeddingStore] Rebuilding from ${count.c} embeddings...`);
|
|
711
328
|
const allEmbeddings = this.getAllEmbeddings();
|
|
712
329
|
for (const [skillId, embedding] of allEmbeddings) {
|
|
713
330
|
try {
|
|
714
331
|
const result = this.vectorDB.insert(embedding, skillId);
|
|
715
|
-
if (result instanceof Promise)
|
|
332
|
+
if (result instanceof Promise)
|
|
716
333
|
await result;
|
|
717
|
-
}
|
|
718
334
|
}
|
|
719
335
|
catch (err) {
|
|
720
|
-
console.warn(`
|
|
336
|
+
console.warn(`Failed to insert ${skillId}: ${err}`);
|
|
721
337
|
}
|
|
722
338
|
}
|
|
723
339
|
console.log(`[HNSWEmbeddingStore] Index rebuilt with ${allEmbeddings.size} vectors`);
|
|
@@ -725,81 +341,14 @@ export class HNSWEmbeddingStore {
|
|
|
725
341
|
}
|
|
726
342
|
}
|
|
727
343
|
catch (err) {
|
|
728
|
-
|
|
729
|
-
console.warn(`[HNSWEmbeddingStore] Failed to initialize V3 VectorDB, using brute-force fallback: ${err}`);
|
|
344
|
+
console.warn(`[HNSWEmbeddingStore] V3 VectorDB unavailable, using brute-force: ${err}`);
|
|
730
345
|
this.vectorDB = null;
|
|
731
346
|
}
|
|
732
347
|
}
|
|
733
|
-
/**
|
|
734
|
-
* Convert HNSW distance to similarity score.
|
|
735
|
-
* HNSW returns distances, we need similarities (higher = more similar).
|
|
736
|
-
*/
|
|
737
348
|
distanceToSimilarity(distance) {
|
|
738
|
-
|
|
739
|
-
// So similarity = 1 - distance
|
|
740
|
-
if (this.distanceMetric === 'cosine') {
|
|
349
|
+
if (this.distanceMetric === 'cosine')
|
|
741
350
|
return 1 - distance;
|
|
742
|
-
}
|
|
743
|
-
// For L2/IP, need different conversion
|
|
744
|
-
// TODO: Implement for other metrics
|
|
745
351
|
return 1 / (1 + distance);
|
|
746
352
|
}
|
|
747
353
|
}
|
|
748
|
-
// ============================================================================
|
|
749
|
-
// Factory Functions
|
|
750
|
-
// ============================================================================
|
|
751
|
-
/**
|
|
752
|
-
* Create an HNSWEmbeddingStore with a preset configuration.
|
|
753
|
-
*
|
|
754
|
-
* @param preset - Preset name ('small', 'medium', 'large', 'xlarge')
|
|
755
|
-
* @param options - Additional options (merged with preset)
|
|
756
|
-
* @returns Configured HNSWEmbeddingStore instance
|
|
757
|
-
*
|
|
758
|
-
* @example
|
|
759
|
-
* ```typescript
|
|
760
|
-
* const store = createHNSWStore('large', {
|
|
761
|
-
* dbPath: './embeddings.db',
|
|
762
|
-
* indexPath: './embeddings.hnsw',
|
|
763
|
-
* });
|
|
764
|
-
* ```
|
|
765
|
-
*/
|
|
766
|
-
export function createHNSWStore(preset, options = {}) {
|
|
767
|
-
return new HNSWEmbeddingStore({
|
|
768
|
-
...options,
|
|
769
|
-
hnswConfig: HNSW_PRESETS[preset],
|
|
770
|
-
});
|
|
771
|
-
}
|
|
772
|
-
/**
|
|
773
|
-
* Check if hnswlib-node is available.
|
|
774
|
-
*
|
|
775
|
-
* Useful for conditional logic or graceful degradation.
|
|
776
|
-
*
|
|
777
|
-
* @returns true if hnswlib-node can be loaded
|
|
778
|
-
*/
|
|
779
|
-
export async function isHNSWAvailable() {
|
|
780
|
-
try {
|
|
781
|
-
// Dynamic import to check availability without static analysis errors
|
|
782
|
-
await Function('return import("hnswlib-node")')();
|
|
783
|
-
return true;
|
|
784
|
-
}
|
|
785
|
-
catch {
|
|
786
|
-
return false;
|
|
787
|
-
}
|
|
788
|
-
}
|
|
789
|
-
/**
|
|
790
|
-
* Dynamically load hnswlib-node module.
|
|
791
|
-
*
|
|
792
|
-
* @returns The HierarchicalNSW constructor, or null if unavailable
|
|
793
|
-
* @internal
|
|
794
|
-
*/
|
|
795
|
-
export async function loadHNSWLib() {
|
|
796
|
-
try {
|
|
797
|
-
// Dynamic import to avoid TypeScript static analysis
|
|
798
|
-
const mod = await Function('return import("hnswlib-node")')();
|
|
799
|
-
return mod;
|
|
800
|
-
}
|
|
801
|
-
catch {
|
|
802
|
-
return null;
|
|
803
|
-
}
|
|
804
|
-
}
|
|
805
354
|
//# sourceMappingURL=hnsw-store.js.map
|