@wanshi-kg/wanshi 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +458 -0
- package/dist/__tests__/helpers.js +27 -0
- package/dist/__tests__/helpers.js.map +1 -0
- package/dist/cli/commands/export.command.js +99 -0
- package/dist/cli/commands/export.command.js.map +1 -0
- package/dist/cli/commands/index.js +22 -0
- package/dist/cli/commands/index.js.map +1 -0
- package/dist/cli/commands/inspectMerges.command.js +84 -0
- package/dist/cli/commands/inspectMerges.command.js.map +1 -0
- package/dist/cli/commands/metrics.command.js +196 -0
- package/dist/cli/commands/metrics.command.js.map +1 -0
- package/dist/cli/commands/process.command.js +82 -0
- package/dist/cli/commands/process.command.js.map +1 -0
- package/dist/cli/commands/watch.command.js +91 -0
- package/dist/cli/commands/watch.command.js.map +1 -0
- package/dist/cli/index.js +269 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/optionsToConfig.js +160 -0
- package/dist/cli/optionsToConfig.js.map +1 -0
- package/dist/config/index.js +59 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/legacyHints.js +113 -0
- package/dist/config/legacyHints.js.map +1 -0
- package/dist/config/schema.js +803 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/ui.js +221 -0
- package/dist/config/ui.js.map +1 -0
- package/dist/core/DirectoryProcessor.js +725 -0
- package/dist/core/DirectoryProcessor.js.map +1 -0
- package/dist/core/adapters/IStructuredAdapter.js +3 -0
- package/dist/core/adapters/IStructuredAdapter.js.map +1 -0
- package/dist/core/adapters/SqliteAdapter.js +267 -0
- package/dist/core/adapters/SqliteAdapter.js.map +1 -0
- package/dist/core/adapters/StructuredAdapterRegistry.js +31 -0
- package/dist/core/adapters/StructuredAdapterRegistry.js.map +1 -0
- package/dist/core/adapters/index.js +20 -0
- package/dist/core/adapters/index.js.map +1 -0
- package/dist/core/checkpoint/CheckpointService.js +188 -0
- package/dist/core/checkpoint/CheckpointService.js.map +1 -0
- package/dist/core/checkpoint/index.js +18 -0
- package/dist/core/checkpoint/index.js.map +1 -0
- package/dist/core/corpus/CorpusAnalyzer.js +266 -0
- package/dist/core/corpus/CorpusAnalyzer.js.map +1 -0
- package/dist/core/corpus/CorpusProfileStore.js +92 -0
- package/dist/core/corpus/CorpusProfileStore.js.map +1 -0
- package/dist/core/corpus/index.js +21 -0
- package/dist/core/corpus/index.js.map +1 -0
- package/dist/core/corpus/normalizeGlossary.js +60 -0
- package/dist/core/corpus/normalizeGlossary.js.map +1 -0
- package/dist/core/corpus/relPath.js +52 -0
- package/dist/core/corpus/relPath.js.map +1 -0
- package/dist/core/corpus/termFrequency.js +86 -0
- package/dist/core/corpus/termFrequency.js.map +1 -0
- package/dist/core/cost/CostMeter.js +235 -0
- package/dist/core/cost/CostMeter.js.map +1 -0
- package/dist/core/cost/index.js +19 -0
- package/dist/core/cost/index.js.map +1 -0
- package/dist/core/cost/prices.js +38 -0
- package/dist/core/cost/prices.js.map +1 -0
- package/dist/core/cv/ObjectDetectionService.js +119 -0
- package/dist/core/cv/ObjectDetectionService.js.map +1 -0
- package/dist/core/di/ContainerFactory.js +670 -0
- package/dist/core/di/ContainerFactory.js.map +1 -0
- package/dist/core/di/DIContainer.js +103 -0
- package/dist/core/di/DIContainer.js.map +1 -0
- package/dist/core/di/index.js +19 -0
- package/dist/core/di/index.js.map +1 -0
- package/dist/core/errors/CustomErrors.js +342 -0
- package/dist/core/errors/CustomErrors.js.map +1 -0
- package/dist/core/errors/index.js +18 -0
- package/dist/core/errors/index.js.map +1 -0
- package/dist/core/export/KnowledgeGraphExportService.js +56 -0
- package/dist/core/export/KnowledgeGraphExportService.js.map +1 -0
- package/dist/core/export/index.js +19 -0
- package/dist/core/export/index.js.map +1 -0
- package/dist/core/export/strategies/GraphitiExportStrategy.js +115 -0
- package/dist/core/export/strategies/GraphitiExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/GraphvizDotExportStrategy.js +331 -0
- package/dist/core/export/strategies/GraphvizDotExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/IExportStrategy.js +3 -0
- package/dist/core/export/strategies/IExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/JsonExportStrategy.js +19 -0
- package/dist/core/export/strategies/JsonExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/JsonlExportStrategy.js +69 -0
- package/dist/core/export/strategies/JsonlExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/KblamExportStrategy.js +36 -0
- package/dist/core/export/strategies/KblamExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/LoraExportStrategy.js +46 -0
- package/dist/core/export/strategies/LoraExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/McpExportStrategy.js +67 -0
- package/dist/core/export/strategies/McpExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/index.js +25 -0
- package/dist/core/export/strategies/index.js.map +1 -0
- package/dist/core/export/strategies/kbTriples.js +60 -0
- package/dist/core/export/strategies/kbTriples.js.map +1 -0
- package/dist/core/index.js +22 -0
- package/dist/core/index.js.map +1 -0
- package/dist/core/knowledge/KnowledgeGraphBuilder.js +627 -0
- package/dist/core/knowledge/KnowledgeGraphBuilder.js.map +1 -0
- package/dist/core/knowledge/MergeRecord.js +3 -0
- package/dist/core/knowledge/MergeRecord.js.map +1 -0
- package/dist/core/knowledge/canon/Canonicalizer.js +414 -0
- package/dist/core/knowledge/canon/Canonicalizer.js.map +1 -0
- package/dist/core/knowledge/canon/index.js +18 -0
- package/dist/core/knowledge/canon/index.js.map +1 -0
- package/dist/core/knowledge/contradiction/HeuristicContradictionChecker.js +92 -0
- package/dist/core/knowledge/contradiction/HeuristicContradictionChecker.js.map +1 -0
- package/dist/core/knowledge/contradiction/LlmContradictionChecker.js +52 -0
- package/dist/core/knowledge/contradiction/LlmContradictionChecker.js.map +1 -0
- package/dist/core/knowledge/contradiction/index.js +19 -0
- package/dist/core/knowledge/contradiction/index.js.map +1 -0
- package/dist/core/knowledge/grounding/KeywordGroundingChecker.js +33 -0
- package/dist/core/knowledge/grounding/KeywordGroundingChecker.js.map +1 -0
- package/dist/core/knowledge/grounding/MiniCheckGroundingChecker.js +82 -0
- package/dist/core/knowledge/grounding/MiniCheckGroundingChecker.js.map +1 -0
- package/dist/core/knowledge/grounding/index.js +20 -0
- package/dist/core/knowledge/grounding/index.js.map +1 -0
- package/dist/core/knowledge/grounding/verbalize.js +38 -0
- package/dist/core/knowledge/grounding/verbalize.js.map +1 -0
- package/dist/core/knowledge/images/imageMetaGraph.js +136 -0
- package/dist/core/knowledge/images/imageMetaGraph.js.map +1 -0
- package/dist/core/knowledge/index.js +20 -0
- package/dist/core/knowledge/index.js.map +1 -0
- package/dist/core/knowledge/merging/KnowledgeMerger.js +624 -0
- package/dist/core/knowledge/merging/KnowledgeMerger.js.map +1 -0
- package/dist/core/knowledge/references/ReferenceResolver.js +184 -0
- package/dist/core/knowledge/references/ReferenceResolver.js.map +1 -0
- package/dist/core/knowledge/references/citations/CitationEvidenceProcessor.js +401 -0
- package/dist/core/knowledge/references/citations/CitationEvidenceProcessor.js.map +1 -0
- package/dist/core/knowledge/references/citations/CitationResolver.js +95 -0
- package/dist/core/knowledge/references/citations/CitationResolver.js.map +1 -0
- package/dist/core/knowledge/references/citations/GrobidClient.js +143 -0
- package/dist/core/knowledge/references/citations/GrobidClient.js.map +1 -0
- package/dist/core/knowledge/references/citations/TitleIdResolver.js +101 -0
- package/dist/core/knowledge/references/citations/TitleIdResolver.js.map +1 -0
- package/dist/core/knowledge/references/web/FetchCacheService.js +114 -0
- package/dist/core/knowledge/references/web/FetchCacheService.js.map +1 -0
- package/dist/core/knowledge/references/web/GatedFetcher.js +228 -0
- package/dist/core/knowledge/references/web/GatedFetcher.js.map +1 -0
- package/dist/core/knowledge/references/web/WebReferenceProcessor.js +164 -0
- package/dist/core/knowledge/references/web/WebReferenceProcessor.js.map +1 -0
- package/dist/core/knowledge/search/KnowledgeGraphSearch.js +261 -0
- package/dist/core/knowledge/search/KnowledgeGraphSearch.js.map +1 -0
- package/dist/core/knowledge/vocabulary.js +162 -0
- package/dist/core/knowledge/vocabulary.js.map +1 -0
- package/dist/core/llm/EmbeddingService.js +113 -0
- package/dist/core/llm/EmbeddingService.js.map +1 -0
- package/dist/core/llm/OllamaService.js +146 -0
- package/dist/core/llm/OllamaService.js.map +1 -0
- package/dist/core/llm/OpenAICompatibleService.js +190 -0
- package/dist/core/llm/OpenAICompatibleService.js.map +1 -0
- package/dist/core/llm/OpenAIEmbeddingService.js +129 -0
- package/dist/core/llm/OpenAIEmbeddingService.js.map +1 -0
- package/dist/core/llm/embeddingUtils.js +25 -0
- package/dist/core/llm/embeddingUtils.js.map +1 -0
- package/dist/core/llm/index.js +23 -0
- package/dist/core/llm/index.js.map +1 -0
- package/dist/core/llm/prompts/PromptManager.js +388 -0
- package/dist/core/llm/prompts/PromptManager.js.map +1 -0
- package/dist/core/llm/prompts/PromptTemplateEngine.js +257 -0
- package/dist/core/llm/prompts/PromptTemplateEngine.js.map +1 -0
- package/dist/core/llm/prompts/templates/partials/examples/EXAMPLE_STYLE_GUIDE.md +84 -0
- package/dist/core/llm/prompts/templates/partials/examples/article.md +187 -0
- package/dist/core/llm/prompts/templates/partials/examples/code.md +229 -0
- package/dist/core/llm/prompts/templates/partials/examples/communication.md +205 -0
- package/dist/core/llm/prompts/templates/partials/examples/documentation.md +262 -0
- package/dist/core/llm/prompts/templates/partials/examples/financial.md +157 -0
- package/dist/core/llm/prompts/templates/partials/examples/legal.md +153 -0
- package/dist/core/llm/prompts/templates/partials/examples/logs.md +127 -0
- package/dist/core/llm/prompts/templates/partials/examples/medical.md +218 -0
- package/dist/core/llm/prompts/templates/partials/examples/notes.md +201 -0
- package/dist/core/llm/prompts/templates/partials/examples/research.md +208 -0
- package/dist/core/llm/prompts/templates/partials/examples/tabular.md +178 -0
- package/dist/core/llm/prompts/templates/partials/examples/transcript.md +204 -0
- package/dist/core/llm/prompts/templates/partials/retrieved-context.hbs +18 -0
- package/dist/core/llm/prompts/templates/v1/system.hbs +371 -0
- package/dist/core/llm/prompts/templates/v1/user.hbs +20 -0
- package/dist/core/llm/prompts/templates/v2/system.hbs +573 -0
- package/dist/core/llm/prompts/templates/v2/user.hbs +20 -0
- package/dist/core/llm/prompts/templates/v3/system.hbs +861 -0
- package/dist/core/llm/prompts/templates/v3/user.hbs +16 -0
- package/dist/core/llm/prompts/templates/v4/system.hbs +800 -0
- package/dist/core/llm/prompts/templates/v4/user.hbs +40 -0
- package/dist/core/llm/prompts/templates/v4.5/system.hbs +71 -0
- package/dist/core/llm/prompts/templates/v4.5/user.hbs +46 -0
- package/dist/core/llm/prompts/templates/v5/glossary/system.hbs +40 -0
- package/dist/core/llm/prompts/templates/v5/glossary/user.hbs +11 -0
- package/dist/core/llm/prompts/templates/v5/system.hbs +163 -0
- package/dist/core/llm/prompts/templates/v5/user.hbs +55 -0
- package/dist/core/pipeline/GroundingTransform.js +52 -0
- package/dist/core/pipeline/GroundingTransform.js.map +1 -0
- package/dist/core/pipeline/PipelineRunner.js +51 -0
- package/dist/core/pipeline/PipelineRunner.js.map +1 -0
- package/dist/core/pipeline/RelationFilterTransform.js +72 -0
- package/dist/core/pipeline/RelationFilterTransform.js.map +1 -0
- package/dist/core/pipeline/index.js +20 -0
- package/dist/core/pipeline/index.js.map +1 -0
- package/dist/core/processor/FileProcessor.js +184 -0
- package/dist/core/processor/FileProcessor.js.map +1 -0
- package/dist/core/processor/ProcessedRegistry.js +38 -0
- package/dist/core/processor/ProcessedRegistry.js.map +1 -0
- package/dist/core/processor/ast/AstSeedService.js +0 -0
- package/dist/core/processor/ast/AstSeedService.js.map +1 -0
- package/dist/core/processor/ast/AstSymbolStore.js +110 -0
- package/dist/core/processor/ast/AstSymbolStore.js.map +1 -0
- package/dist/core/processor/ast/index.js +19 -0
- package/dist/core/processor/ast/index.js.map +1 -0
- package/dist/core/processor/chunking/TextChunker.js +98 -0
- package/dist/core/processor/chunking/TextChunker.js.map +1 -0
- package/dist/core/processor/chunking/index.js +18 -0
- package/dist/core/processor/chunking/index.js.map +1 -0
- package/dist/core/processor/classifier/CONTENT_CLASSES.js +294 -0
- package/dist/core/processor/classifier/CONTENT_CLASSES.js.map +1 -0
- package/dist/core/processor/classifier/CascadeContentClassifier.js +107 -0
- package/dist/core/processor/classifier/CascadeContentClassifier.js.map +1 -0
- package/dist/core/processor/classifier/HeuristicContentClassifier.js +113 -0
- package/dist/core/processor/classifier/HeuristicContentClassifier.js.map +1 -0
- package/dist/core/processor/classifier/IContentTypeClassifier.js +3 -0
- package/dist/core/processor/classifier/IContentTypeClassifier.js.map +1 -0
- package/dist/core/processor/classifier/LlmContentClassifier.js +107 -0
- package/dist/core/processor/classifier/LlmContentClassifier.js.map +1 -0
- package/dist/core/processor/classifier/NER_DOMAIN_EXAMPLES.js +498 -0
- package/dist/core/processor/classifier/NER_DOMAIN_EXAMPLES.js.map +1 -0
- package/dist/core/processor/classifier/index.js +21 -0
- package/dist/core/processor/classifier/index.js.map +1 -0
- package/dist/core/processor/classifier/mergeClassifications.js +32 -0
- package/dist/core/processor/classifier/mergeClassifications.js.map +1 -0
- package/dist/core/processor/index.js +20 -0
- package/dist/core/processor/index.js.map +1 -0
- package/dist/core/processor/readers/AudioReader.js +462 -0
- package/dist/core/processor/readers/AudioReader.js.map +1 -0
- package/dist/core/processor/readers/BinaryReader.js +90 -0
- package/dist/core/processor/readers/BinaryReader.js.map +1 -0
- package/dist/core/processor/readers/ChandraPdfReader.js +187 -0
- package/dist/core/processor/readers/ChandraPdfReader.js.map +1 -0
- package/dist/core/processor/readers/ChatExportReader.js +365 -0
- package/dist/core/processor/readers/ChatExportReader.js.map +1 -0
- package/dist/core/processor/readers/DoclingReader.js +445 -0
- package/dist/core/processor/readers/DoclingReader.js.map +1 -0
- package/dist/core/processor/readers/EmailReader.js +259 -0
- package/dist/core/processor/readers/EmailReader.js.map +1 -0
- package/dist/core/processor/readers/EpubReader.js +175 -0
- package/dist/core/processor/readers/EpubReader.js.map +1 -0
- package/dist/core/processor/readers/FileReader.js +90 -0
- package/dist/core/processor/readers/FileReader.js.map +1 -0
- package/dist/core/processor/readers/FileReaderFactory.js +49 -0
- package/dist/core/processor/readers/FileReaderFactory.js.map +1 -0
- package/dist/core/processor/readers/HtmlReader.js +371 -0
- package/dist/core/processor/readers/HtmlReader.js.map +1 -0
- package/dist/core/processor/readers/ImageReader.js +162 -0
- package/dist/core/processor/readers/ImageReader.js.map +1 -0
- package/dist/core/processor/readers/JsonFileReader.js +232 -0
- package/dist/core/processor/readers/JsonFileReader.js.map +1 -0
- package/dist/core/processor/readers/JupyterReader.js +178 -0
- package/dist/core/processor/readers/JupyterReader.js.map +1 -0
- package/dist/core/processor/readers/LatexReader.js +176 -0
- package/dist/core/processor/readers/LatexReader.js.map +1 -0
- package/dist/core/processor/readers/MarkdownReader.js +289 -0
- package/dist/core/processor/readers/MarkdownReader.js.map +1 -0
- package/dist/core/processor/readers/MarkerPdfReader.js +193 -0
- package/dist/core/processor/readers/MarkerPdfReader.js.map +1 -0
- package/dist/core/processor/readers/MistralOcrReader.js +198 -0
- package/dist/core/processor/readers/MistralOcrReader.js.map +1 -0
- package/dist/core/processor/readers/OfficeReader.js +174 -0
- package/dist/core/processor/readers/OfficeReader.js.map +1 -0
- package/dist/core/processor/readers/PdfReader.js +116 -0
- package/dist/core/processor/readers/PdfReader.js.map +1 -0
- package/dist/core/processor/readers/RtfReader.js +107 -0
- package/dist/core/processor/readers/RtfReader.js.map +1 -0
- package/dist/core/processor/readers/SubtitleReader.js +145 -0
- package/dist/core/processor/readers/SubtitleReader.js.map +1 -0
- package/dist/core/processor/readers/TesseractPdfReader.js +183 -0
- package/dist/core/processor/readers/TesseractPdfReader.js.map +1 -0
- package/dist/core/processor/readers/TextReader.js +129 -0
- package/dist/core/processor/readers/TextReader.js.map +1 -0
- package/dist/core/processor/readers/TranscriptReader.js +234 -0
- package/dist/core/processor/readers/TranscriptReader.js.map +1 -0
- package/dist/core/processor/readers/image/imageMetadata.js +155 -0
- package/dist/core/processor/readers/image/imageMetadata.js.map +1 -0
- package/dist/core/processor/readers/index.js +41 -0
- package/dist/core/processor/readers/index.js.map +1 -0
- package/dist/core/processor/readers/referenceExtraction.js +198 -0
- package/dist/core/processor/readers/referenceExtraction.js.map +1 -0
- package/dist/core/processor/readers/stripReferences.js +59 -0
- package/dist/core/processor/readers/stripReferences.js.map +1 -0
- package/dist/core/processor/readers/transcript/turnPacking.js +81 -0
- package/dist/core/processor/readers/transcript/turnPacking.js.map +1 -0
- package/dist/core/progress/NdjsonProgressEmitter.js +30 -0
- package/dist/core/progress/NdjsonProgressEmitter.js.map +1 -0
- package/dist/core/progress/NoopProgressEmitter.js +15 -0
- package/dist/core/progress/NoopProgressEmitter.js.map +1 -0
- package/dist/core/progress/index.js +19 -0
- package/dist/core/progress/index.js.map +1 -0
- package/dist/core/trace/TraceWriter.js +100 -0
- package/dist/core/trace/TraceWriter.js.map +1 -0
- package/dist/core/trace/events.js +13 -0
- package/dist/core/trace/events.js.map +1 -0
- package/dist/core/trace/index.js +20 -0
- package/dist/core/trace/index.js.map +1 -0
- package/dist/core/trace/lineage.js +97 -0
- package/dist/core/trace/lineage.js.map +1 -0
- package/dist/evaluation/BenchmarkRunner.js +171 -0
- package/dist/evaluation/BenchmarkRunner.js.map +1 -0
- package/dist/evaluation/classifier/ClassifierAccuracy.js +185 -0
- package/dist/evaluation/classifier/ClassifierAccuracy.js.map +1 -0
- package/dist/evaluation/classifier/labeledSamples.js +379 -0
- package/dist/evaluation/classifier/labeledSamples.js.map +1 -0
- package/dist/evaluation/compare/goldCompare.js +126 -0
- package/dist/evaluation/compare/goldCompare.js.map +1 -0
- package/dist/evaluation/crossre/compareScoring.js +30 -0
- package/dist/evaluation/crossre/compareScoring.js.map +1 -0
- package/dist/evaluation/datasets/CrossREDataset.js +170 -0
- package/dist/evaluation/datasets/CrossREDataset.js.map +1 -0
- package/dist/evaluation/datasets/IDataset.js +3 -0
- package/dist/evaluation/datasets/IDataset.js.map +1 -0
- package/dist/evaluation/datasets/RebelDataset.js +117 -0
- package/dist/evaluation/datasets/RebelDataset.js.map +1 -0
- package/dist/evaluation/datasets/RedocredDataset.js +218 -0
- package/dist/evaluation/datasets/RedocredDataset.js.map +1 -0
- package/dist/evaluation/datasets/SemEval2010Dataset.js +150 -0
- package/dist/evaluation/datasets/SemEval2010Dataset.js.map +1 -0
- package/dist/evaluation/index.js +33 -0
- package/dist/evaluation/index.js.map +1 -0
- package/dist/evaluation/matching/ExactMatcher.js +75 -0
- package/dist/evaluation/matching/ExactMatcher.js.map +1 -0
- package/dist/evaluation/matching/SemanticMatcher.js +143 -0
- package/dist/evaluation/matching/SemanticMatcher.js.map +1 -0
- package/dist/evaluation/metrics/TripleMetrics.js +64 -0
- package/dist/evaluation/metrics/TripleMetrics.js.map +1 -0
- package/dist/evaluation/mine/MineCheckpoint.js +114 -0
- package/dist/evaluation/mine/MineCheckpoint.js.map +1 -0
- package/dist/evaluation/mine/MineDataset.js +208 -0
- package/dist/evaluation/mine/MineDataset.js.map +1 -0
- package/dist/evaluation/mine/MineReporter.js +98 -0
- package/dist/evaluation/mine/MineReporter.js.map +1 -0
- package/dist/evaluation/mine/MineRunner.js +148 -0
- package/dist/evaluation/mine/MineRunner.js.map +1 -0
- package/dist/evaluation/mine/MineScorer.js +127 -0
- package/dist/evaluation/mine/MineScorer.js.map +1 -0
- package/dist/evaluation/mine/types.js +12 -0
- package/dist/evaluation/mine/types.js.map +1 -0
- package/dist/evaluation/reporters/ConsoleReporter.js +55 -0
- package/dist/evaluation/reporters/ConsoleReporter.js.map +1 -0
- package/dist/evaluation/reporters/JsonReporter.js +50 -0
- package/dist/evaluation/reporters/JsonReporter.js.map +1 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/quality/CompositeScore.js +61 -0
- package/dist/quality/CompositeScore.js.map +1 -0
- package/dist/quality/ConsistencyMetrics.js +70 -0
- package/dist/quality/ConsistencyMetrics.js.map +1 -0
- package/dist/quality/FactualMetrics.js +76 -0
- package/dist/quality/FactualMetrics.js.map +1 -0
- package/dist/quality/GraphHealthMetrics.js +68 -0
- package/dist/quality/GraphHealthMetrics.js.map +1 -0
- package/dist/quality/SemanticMetrics.js +102 -0
- package/dist/quality/SemanticMetrics.js.map +1 -0
- package/dist/quality/StructuralMetrics.js +60 -0
- package/dist/quality/StructuralMetrics.js.map +1 -0
- package/dist/quality/index.js +23 -0
- package/dist/quality/index.js.map +1 -0
- package/dist/shared/index.js +20 -0
- package/dist/shared/index.js.map +1 -0
- package/dist/shared/logger/Logger.js +3 -0
- package/dist/shared/logger/Logger.js.map +1 -0
- package/dist/shared/logger/LoggerFactory.js +75 -0
- package/dist/shared/logger/LoggerFactory.js.map +1 -0
- package/dist/shared/logger/index.js +19 -0
- package/dist/shared/logger/index.js.map +1 -0
- package/dist/shared/shutdown.js +30 -0
- package/dist/shared/shutdown.js.map +1 -0
- package/dist/shared/utils/agglomerativeCluster.js +269 -0
- package/dist/shared/utils/agglomerativeCluster.js.map +1 -0
- package/dist/shared/utils/astSymbols.js +69 -0
- package/dist/shared/utils/astSymbols.js.map +1 -0
- package/dist/shared/utils/cosineSimilarity.js +18 -0
- package/dist/shared/utils/cosineSimilarity.js.map +1 -0
- package/dist/shared/utils/directoryTree.js +184 -0
- package/dist/shared/utils/directoryTree.js.map +1 -0
- package/dist/shared/utils/documentOutline.js +74 -0
- package/dist/shared/utils/documentOutline.js.map +1 -0
- package/dist/shared/utils/index.js +24 -0
- package/dist/shared/utils/index.js.map +1 -0
- package/dist/shared/utils/jaroWinklerSimilarity.js +60 -0
- package/dist/shared/utils/jaroWinklerSimilarity.js.map +1 -0
- package/dist/shared/utils/parseJsonLenient.js +27 -0
- package/dist/shared/utils/parseJsonLenient.js.map +1 -0
- package/dist/shared/utils/readConfig.js +42 -0
- package/dist/shared/utils/readConfig.js.map +1 -0
- package/dist/shared/utils/readRtf.js +216 -0
- package/dist/shared/utils/readRtf.js.map +1 -0
- package/dist/shared/utils/softmax.js +26 -0
- package/dist/shared/utils/softmax.js.map +1 -0
- package/dist/types/ContentClass.js +3 -0
- package/dist/types/ContentClass.js.map +1 -0
- package/dist/types/CorpusProfile.js +3 -0
- package/dist/types/CorpusProfile.js.map +1 -0
- package/dist/types/IContradictionChecker.js +3 -0
- package/dist/types/IContradictionChecker.js.map +1 -0
- package/dist/types/ICorpusAnalyzer.js +3 -0
- package/dist/types/ICorpusAnalyzer.js.map +1 -0
- package/dist/types/IDirectoryProcessor.js +3 -0
- package/dist/types/IDirectoryProcessor.js.map +1 -0
- package/dist/types/IEmbeddingProvider.js +3 -0
- package/dist/types/IEmbeddingProvider.js.map +1 -0
- package/dist/types/IEmbeddingService.js +6 -0
- package/dist/types/IEmbeddingService.js.map +1 -0
- package/dist/types/IFileProcessor.js +3 -0
- package/dist/types/IFileProcessor.js.map +1 -0
- package/dist/types/IGroundingChecker.js +3 -0
- package/dist/types/IGroundingChecker.js.map +1 -0
- package/dist/types/IKnowledgeGraphBuilder.js +3 -0
- package/dist/types/IKnowledgeGraphBuilder.js.map +1 -0
- package/dist/types/IKnowledgeGraphExporter.js +3 -0
- package/dist/types/IKnowledgeGraphExporter.js.map +1 -0
- package/dist/types/IKnowledgeGraphMerger.js +3 -0
- package/dist/types/IKnowledgeGraphMerger.js.map +1 -0
- package/dist/types/IKnowledgeGraphSearch.js +3 -0
- package/dist/types/IKnowledgeGraphSearch.js.map +1 -0
- package/dist/types/ILLMProvider.js +3 -0
- package/dist/types/ILLMProvider.js.map +1 -0
- package/dist/types/ILLMService.js +3 -0
- package/dist/types/ILLMService.js.map +1 -0
- package/dist/types/IObjectDetector.js +3 -0
- package/dist/types/IObjectDetector.js.map +1 -0
- package/dist/types/IProcessingService.js +3 -0
- package/dist/types/IProcessingService.js.map +1 -0
- package/dist/types/IProgressEmitter.js +3 -0
- package/dist/types/IProgressEmitter.js.map +1 -0
- package/dist/types/IPromptManager.js +3 -0
- package/dist/types/IPromptManager.js.map +1 -0
- package/dist/types/KnowledgeGraph.js +3 -0
- package/dist/types/KnowledgeGraph.js.map +1 -0
- package/dist/types/MCPKnowledgeGraph.js +3 -0
- package/dist/types/MCPKnowledgeGraph.js.map +1 -0
- package/dist/types/Observation.js +21 -0
- package/dist/types/Observation.js.map +1 -0
- package/dist/types/ProcessingOptions.js +3 -0
- package/dist/types/ProcessingOptions.js.map +1 -0
- package/dist/types/index.js +40 -0
- package/dist/types/index.js.map +1 -0
- package/package.json +122 -0
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CONTENT_CLASSES = void 0;
|
|
4
|
+
exports.CONTENT_CLASSES = {
|
|
5
|
+
code: {
|
|
6
|
+
name: "code",
|
|
7
|
+
filePatterns: [
|
|
8
|
+
// Tier 1: Smoking Guns (15.0-10.0)
|
|
9
|
+
{ pattern: /\.(ts|js|jsx|tsx|mjs)$/i, weight: 15.0 },
|
|
10
|
+
{ pattern: /\.(py|rb|php|java|cpp|c|h|go|rs|scala|kt|swift)$/i, weight: 15.0 },
|
|
11
|
+
{ pattern: /\.(sql|sh|bat|ps1|dockerfile)$/i, weight: 12.0 },
|
|
12
|
+
{ pattern: /^.*\/(src|lib|app|components|utils|services|api|tests?)\//i, weight: 8.0 },
|
|
13
|
+
],
|
|
14
|
+
contentPatterns: [
|
|
15
|
+
// Tier 1: Smoking Guns (15.0-10.0) - Structural code patterns
|
|
16
|
+
{ pattern: /\b(function|fun|type|class|enum|interface|struct)\s+[\w\d_]+[\s\n\r]*\{/igm, weight: 12.0 },
|
|
17
|
+
{ pattern: /\b(async|await|def|let|var|const)\s+[\w\d_]+/igm, weight: 10.0 },
|
|
18
|
+
{ pattern: /\b[\w\d_]+\s+(implements|extends)\s+[\w\d_]+[\s\n\r]*\{/igm, weight: 12.0 },
|
|
19
|
+
// Tier 2: Strong Contextual (7.0-4.0) - Code-specific constructs
|
|
20
|
+
{ pattern: /\b(?:try|catch|finally|throw)\s*[\(\{]/igm, weight: 5.0 },
|
|
21
|
+
{ pattern: /\bconsole\.(?:log|error|warn|info|debug|trace|write(?:line|ln)?)\s*\(/igm, weight: 5.0 },
|
|
22
|
+
{ pattern: /\b[\w\d_]{2,}(\.[\w\d_]{2,})*\s*\(/igm, weight: 5.0 },
|
|
23
|
+
// Tier 3: Supporting Evidence (3.0-1.0) - Control flow and keywords
|
|
24
|
+
{ pattern: /\b(?:if|for|while|switch|else|elif)\s*\(/igm, weight: 2.0 },
|
|
25
|
+
{ pattern: /\b(?:return|break|continue|yield)\s+/igm, weight: 2.0 },
|
|
26
|
+
{ pattern: /\b(?:public|private|protected|static|final)\s+[\w\d_]+/igm, weight: 2.0 },
|
|
27
|
+
// Tier 4: Noise (0.5-0.1) - Code punctuation
|
|
28
|
+
{ pattern: /[{}();]/g, weight: 0.1 },
|
|
29
|
+
],
|
|
30
|
+
},
|
|
31
|
+
financial: {
|
|
32
|
+
name: "financial",
|
|
33
|
+
filePatterns: [
|
|
34
|
+
{ pattern: /\b(?:financial|revenue|earnings|trading|investment|quarterly|annual).*\.(?:md|txt|pdf|xlsx?)$/i, weight: 10.0 },
|
|
35
|
+
{ pattern: /^.*\/(finance|accounting|trading|investment|revenue|earnings)\//i, weight: 8.0 },
|
|
36
|
+
{ pattern: /^(?:10-K|10-Q|8-K|annual|quarterly).*report/i, weight: 12.0 },
|
|
37
|
+
],
|
|
38
|
+
contentPatterns: [
|
|
39
|
+
// Tier 1: Smoking Guns - Formatted financial data
|
|
40
|
+
{ pattern: /\$\d{1,3}(?:,\d{3})*(?:\.\d{2})?(?:\s*(?:million|billion|trillion|[KMB]))?/g, weight: 10.0 },
|
|
41
|
+
{ pattern: /\b(?:NYSE|NASDAQ|S&P\s*500|Dow\s+Jones|ticker\s+symbol):\s*[A-Z]{2,5}/g, weight: 8.0 },
|
|
42
|
+
// Tier 2: Strong Contextual - Financial metrics and reporting
|
|
43
|
+
{ pattern: /\b(?:revenue|profit|earnings|EBITDA|ROI|P\/E\s+ratio|market\s+cap)\s*[:=]\s*\$?\d/ig, weight: 6.0 },
|
|
44
|
+
{ pattern: /\b(?:Q[1-4]|quarterly|annual)\s+(?:revenue|earnings|report|results|guidance)\b/ig, weight: 5.0 },
|
|
45
|
+
{ pattern: /\b(?:beat|miss|exceed)(?:s|ed)?\s+(?:estimates?|expectations?|consensus)/ig, weight: 5.0 },
|
|
46
|
+
// Tier 3: Supporting Evidence - Financial vocabulary
|
|
47
|
+
{ pattern: /\b(?:shares?|stock|equity|portfolio|investment|dividend|yield|valuation)\b/ig, weight: 2.0 },
|
|
48
|
+
{ pattern: /\b(?:SEC|filing|10-K|10-Q|8-K|proxy|statement)\b/ig, weight: 2.0 },
|
|
49
|
+
{ pattern: /\b(?:analyst|investor|shareholder|stakeholder)\b/ig, weight: 1.5 },
|
|
50
|
+
// Tier 4: Noise - Percentage and numbers in financial context
|
|
51
|
+
{ pattern: /\d+(?:\.\d+)?%/g, weight: 0.2 },
|
|
52
|
+
],
|
|
53
|
+
},
|
|
54
|
+
medical: {
|
|
55
|
+
name: "medical",
|
|
56
|
+
filePatterns: [
|
|
57
|
+
{ pattern: /\b(?:medical|health|clinical|patient|drug|treatment|pharma).*\.(?:md|txt|pdf)$/i, weight: 10.0 },
|
|
58
|
+
{ pattern: /^.*\/(medical|health|clinical|patient|drug|treatment)\//i, weight: 8.0 },
|
|
59
|
+
{ pattern: /^(?:chart|record|protocol|trial|study).*\.(txt|md)$/i, weight: 8.0 },
|
|
60
|
+
],
|
|
61
|
+
contentPatterns: [
|
|
62
|
+
// Tier 1: Smoking Guns - Medical dosages and procedures
|
|
63
|
+
{ pattern: /\b\d+\s*(?:mg|ml|mcg|µg|IU|units?)\b/g, weight: 10.0 },
|
|
64
|
+
{ pattern: /\b(?:BID|TID|QID|PRN|PO|IV|IM|SC|QHS|AC|PC)\b/g, weight: 8.0 },
|
|
65
|
+
// Tier 2: Strong Contextual - Clinical and regulatory
|
|
66
|
+
{ pattern: /\b(?:clinical\s+trial|randomized\s+controlled|double-blind|placebo-controlled|efficacy|adverse\s+(?:events?|effects?))\b/ig, weight: 6.0 },
|
|
67
|
+
{ pattern: /\b(?:FDA|EMA)\s+(?:approval|clearance|indication|contraindication|black\s+box\s+warning)/ig, weight: 6.0 },
|
|
68
|
+
{ pattern: /\b(?:phase\s+[I-IV]|IRB|informed\s+consent|protocol\s+amendment)\b/ig, weight: 5.0 },
|
|
69
|
+
// Tier 3: Supporting Evidence - Medical terminology
|
|
70
|
+
{ pattern: /\b(?:patient|diagnosis|treatment|medication|dosage|prescription|therapeutic)\b/ig, weight: 2.5 },
|
|
71
|
+
{ pattern: /\b(?:symptoms?|condition|disease|disorder|syndrome|pathology)\b/ig, weight: 2.0 },
|
|
72
|
+
{ pattern: /\b(?:blood\s+pressure|heart\s+rate|temperature|vital\s+signs|laboratory\s+values)\b/ig, weight: 2.0 },
|
|
73
|
+
// Tier 4: Noise - Medical abbreviations
|
|
74
|
+
{ pattern: /\b[A-Z]{2,5}\b/g, weight: 0.1 },
|
|
75
|
+
],
|
|
76
|
+
},
|
|
77
|
+
legal: {
|
|
78
|
+
name: "legal",
|
|
79
|
+
filePatterns: [
|
|
80
|
+
{ pattern: /^.*\/(legal|contracts|compliance|regulatory|terms)\//i, weight: 8.0 },
|
|
81
|
+
{ pattern: /\b(?:contract|agreement|terms|legal|conditions|compliance)/i, weight: 8.0 },
|
|
82
|
+
{ pattern: /^(?:NDA|MSA|SLA|ToS|privacy.?policy)/i, weight: 12.0 },
|
|
83
|
+
{ pattern: /\bLICENSE\.(txt|md|pdf)?$/i, weight: 10.0 },
|
|
84
|
+
],
|
|
85
|
+
contentPatterns: [
|
|
86
|
+
// Tier 1: Smoking Guns - Legal document structure
|
|
87
|
+
{ pattern: /\b(?:whereas|hereby|party|parties)\s+(?:\w+\s+){0,3}(?:agree|shall|will)\b/ig, weight: 12.0 },
|
|
88
|
+
{ pattern: /\b(?:effective\s+date|term|termination|renewal)\s*[:=]/ig, weight: 10.0 },
|
|
89
|
+
// Tier 2: Strong Contextual - Legal language and procedures
|
|
90
|
+
{ pattern: /\b(?:jurisdiction|court|statute|regulation|compliance|precedent)\b/ig, weight: 6.0 },
|
|
91
|
+
{ pattern: /\b(?:liable|liability|damages|indemnify|breach)\s+(?:of|for|to)\b/ig, weight: 8.0 },
|
|
92
|
+
{ pattern: /\b(?:attorney|counsel|legal)\s+(?:\w+\s+){0,2}(?:opinion|ruling|advice)\b/ig, weight: 6.0 },
|
|
93
|
+
// Tier 3: Supporting Evidence - Legal vocabulary
|
|
94
|
+
{ pattern: /\b(?:contract|agreement|clause|provision|section|subsection)\b/ig, weight: 2.0 },
|
|
95
|
+
{ pattern: /\b(?:plaintiff|defendant|appellant|respondent)\b/ig, weight: 2.0 },
|
|
96
|
+
{ pattern: /\b(?:constitute|deem|execute|enforce|waive)\b/ig, weight: 1.5 },
|
|
97
|
+
// Tier 4: Noise - Legal formatting
|
|
98
|
+
{ pattern: /\b(?:section|sec\.?)\s+\d+(?:\.\d+)*/ig, weight: 0.2 },
|
|
99
|
+
],
|
|
100
|
+
},
|
|
101
|
+
research: {
|
|
102
|
+
name: "research",
|
|
103
|
+
filePatterns: [
|
|
104
|
+
// Tier 1: Smoking Guns - Academic paper patterns
|
|
105
|
+
{ pattern: /\d{4}\.\d{4,5}(?:v\d)?\.(md|txt|pdf)$/i, weight: 15.0 },
|
|
106
|
+
{ pattern: /(findings|research|experiments|studies|analysis|thesis)(.*?)\.(md|doc|docx|rtf|txt|pdf)$/i, weight: 12.0 },
|
|
107
|
+
{ pattern: /^.*\/(papers?|research|experiments|studies|analysis|thesis)\//i, weight: 8.0 },
|
|
108
|
+
],
|
|
109
|
+
contentPatterns: [
|
|
110
|
+
// Tier 1: Smoking Guns - Academic structure and citations
|
|
111
|
+
{ pattern: /\b(?:doi|arxiv|pubmed|citation)[:\/]\s*\S+/ig, weight: 15.0 },
|
|
112
|
+
{ pattern: /^#{1,6}\s*(?:\S+\s+)?(?:abstract|related\s+work|literature\s+review|background|methodology|evaluation|experiments?|results?|discussion|conclusion|future\s+work|acknowledgements?|references?|ethical\s+considerations)\s*$/gim, weight: 13.0 },
|
|
113
|
+
{ pattern: /\b\w+\s+et\s+al\.?\s+\((?:19|20)\d{2}\)/ig, weight: 12.0 },
|
|
114
|
+
{ pattern: /\[[^\]]*(?:19|20)\d{2}[^\]]*\](?!\()/ig, weight: 12.0 },
|
|
115
|
+
// Tier 2: Strong Contextual - Academic language
|
|
116
|
+
{ pattern: /\b(?:we\s+(?:adopt|proposed?|present(ed)?|introduced?|demonstrated?|evaluated?|investigated?|compare|found|explore|observe))\b/ig, weight: 10.0 },
|
|
117
|
+
{ pattern: /\b(?:our\s+(?:analysis|hypothesis|method|approach|model|algorithm|framework|contributions?|findings?|results?|corpus))\b/ig, weight: 9.0 },
|
|
118
|
+
{ pattern: /\b(?:experimental\s+(?:results|evaluation|setup|design|validation))\b/ig, weight: 9.0 },
|
|
119
|
+
{ pattern: /\b(?:state-of-the-art|sota|baseline\s+(?:comparison|method|model))\b/ig, weight: 6.0 },
|
|
120
|
+
// Tier 3: Supporting Evidence - Research methodology
|
|
121
|
+
{ pattern: /\b(?:accuracy|precision|recall|f1[-\s]?score|bleu|rouge|meteor)\s*[:=]\s*(?:0?\.\d+|\d+\.?\d*%)/ig, weight: 3.0 },
|
|
122
|
+
{ pattern: /\b(?:p-value|p)\s*[<>=]\s*0\.\d+/ig, weight: 3.0 },
|
|
123
|
+
{ pattern: /\b(?:significance|confidence\s+interval|standard\s+deviation)\b/ig, weight: 2.0 },
|
|
124
|
+
{ pattern: /\b(?:dataset|corpus|benchmark|ground\s+truth)\b/ig, weight: 2.0 },
|
|
125
|
+
// Tier 4: Noise - Academic formatting
|
|
126
|
+
{ pattern: /\$[^$\n]{3,}\$/g, weight: 0.5 },
|
|
127
|
+
],
|
|
128
|
+
},
|
|
129
|
+
transcript: {
|
|
130
|
+
name: "transcript",
|
|
131
|
+
filePatterns: [
|
|
132
|
+
{ pattern: /^.*\/(meetings|transcripts|minutes|calls|interviews)\//i, weight: 10.0 },
|
|
133
|
+
{ pattern: /\b(?:meeting|transcript|minutes|call|interview).*\.(txt|md)$/i, weight: 10.0 },
|
|
134
|
+
{ pattern: /^(?:standup|sync|planning|review).*\.(txt|md)$/i, weight: 8.0 },
|
|
135
|
+
],
|
|
136
|
+
contentPatterns: [
|
|
137
|
+
// Tier 1: Smoking Guns - Transcript structure
|
|
138
|
+
{ pattern: /^\[\d\d:\d\d:\d\d\.\d\d\d\s-->\s\d\d:\d\d:\d\d\.\d\d\d\]\s+/ig, weight: 15.0 },
|
|
139
|
+
{ pattern: /^\w+\s*:\s*(?![A-Z]{2,})/gm, weight: 8.0 },
|
|
140
|
+
{ pattern: /\b(?:attendees?|participants?):\s*\w/ig, weight: 8.0 },
|
|
141
|
+
{ pattern: /\b(?:action\s+items?|next\s+steps?|follow.?up)\s*:?\s*\w/ig, weight: 8.0 },
|
|
142
|
+
// Tier 2: Strong Contextual - Meeting language
|
|
143
|
+
{ pattern: /\b(?:meeting|transcript|minutes)\s+(?:for|of|-)?\s*\w/ig, weight: 6.0 },
|
|
144
|
+
{ pattern: /\b(?:decision|agreed|resolved|discussed)\s+(?:to|on|that)\b/ig, weight: 5.0 },
|
|
145
|
+
{ pattern: /\b(?:speaker|moderator|presenter)\s*\d*:\s*/ig, weight: 5.0 },
|
|
146
|
+
// Tier 3: Supporting Evidence - Temporal and organizational
|
|
147
|
+
{ pattern: /\b(?:today|yesterday|tomorrow|this\s+week|next\s+week|by\s+(?:friday|monday))\b/ig, weight: 2.0 },
|
|
148
|
+
{ pattern: /\b(?:agenda|schedule|timeline|deadline)\b/ig, weight: 2.0 },
|
|
149
|
+
{ pattern: /\b(?:team|group|department|stakeholder)\b/ig, weight: 1.5 },
|
|
150
|
+
// Tier 4: Noise - Time stamps
|
|
151
|
+
{ pattern: /\d{1,2}:\d{2}(?:\s*[AP]M)?/g, weight: 0.2 },
|
|
152
|
+
],
|
|
153
|
+
},
|
|
154
|
+
tabular: {
|
|
155
|
+
name: "tabular",
|
|
156
|
+
filePatterns: [
|
|
157
|
+
// Tier 1: Smoking Guns - Data file extensions
|
|
158
|
+
{ pattern: /\.(csv|tsv|xlsx?|xls)$/i, weight: 20.0 },
|
|
159
|
+
{ pattern: /^.*\/(data|datasets?|exports?|reports?|metrics|table)\//i, weight: 8.0 },
|
|
160
|
+
],
|
|
161
|
+
contentPatterns: [
|
|
162
|
+
// Tier 1: Smoking Guns - Structured tabular data (4+ columns)
|
|
163
|
+
{ pattern: /^([^,\n]*,){3,}[^,\n]*$/gm, weight: 2.75 },
|
|
164
|
+
{ pattern: /^([^|\n]*\|){3,}[^|\n]*$/gm, weight: 2.75 },
|
|
165
|
+
{ pattern: /^([^;\n]*\;){3,}[^;\n]*$/gm, weight: 2.75 },
|
|
166
|
+
{ pattern: /^([^:\n]*\:){3,}[^:\n]*$/gm, weight: 2.75 },
|
|
167
|
+
{ pattern: /^([^\t\n]*\t){3,}[^\t\n]*$/gm, weight: 2.75 },
|
|
168
|
+
// Tier 2: Strong Contextual - Table headers and structure
|
|
169
|
+
{ pattern: /^(?:\w+[,|\t]){3,}\w+$/gm, weight: 2.5 },
|
|
170
|
+
{ pattern: /^[^,\n]*,\s*\d+(?:\.\d+)?\s*,/gm, weight: 2.5 },
|
|
171
|
+
// Tier 3: Supporting Evidence - Data patterns
|
|
172
|
+
{ pattern: /\b(?:column|row|field|record|entry)\b/ig, weight: 2.0 },
|
|
173
|
+
{ pattern: /\b(?:total|sum|average|count|min|max)\b/ig, weight: 1.0 },
|
|
174
|
+
// Tier 4: Noise - Basic separators (2-3 columns only)
|
|
175
|
+
{ pattern: /^[^,\n]*,[^,\n]*$/gm, weight: 0.1 },
|
|
176
|
+
],
|
|
177
|
+
},
|
|
178
|
+
communication: {
|
|
179
|
+
name: "communication",
|
|
180
|
+
filePatterns: [
|
|
181
|
+
{ pattern: /^.*\/(email|messages?|chat|correspondence|communications?)\//i, weight: 10.0 },
|
|
182
|
+
{ pattern: /\b(?:email|message|chat|correspondence).*\.(txt|md|eml)$/i, weight: 10.0 },
|
|
183
|
+
{ pattern: /^(?:inbox|sent|draft).*\.(txt|md)$/i, weight: 8.0 },
|
|
184
|
+
],
|
|
185
|
+
contentPatterns: [
|
|
186
|
+
// Tier 1: Smoking Guns - Email structure
|
|
187
|
+
{ pattern: /^(?:From|To|Subject|Cc|Bcc|Date|Reply-To):\s*\S/gim, weight: 12.0 },
|
|
188
|
+
{ pattern: /^On\s+.+\s+wrote:\s*$/gim, weight: 10.0 },
|
|
189
|
+
{ pattern: /@[\w.-]+\.\w+/g, weight: 8.0 },
|
|
190
|
+
// Tier 2: Strong Contextual - Communication patterns
|
|
191
|
+
{ pattern: /\b(?:dear|hi|hello|sincerely|regards|best)\s+\w/ig, weight: 6.0 },
|
|
192
|
+
{ pattern: /^>\s+/gm, weight: 5.0 },
|
|
193
|
+
{ pattern: /\b(?:email|message|correspondence|communication)\b/ig, weight: 4.0 },
|
|
194
|
+
// Tier 3: Supporting Evidence - Communication vocabulary
|
|
195
|
+
{ pattern: /\b(?:sent|received|replied|forwarded|attached)\b/ig, weight: 2.0 },
|
|
196
|
+
{ pattern: /\b(?:thread|conversation|discussion|reply)\b/ig, weight: 2.0 },
|
|
197
|
+
{ pattern: /\b(?:cc|bcc|fwd|re):/ig, weight: 1.5 },
|
|
198
|
+
// Tier 4: Noise - Common greetings
|
|
199
|
+
{ pattern: /\b(?:thanks?|please|regards)\b/ig, weight: 0.1 },
|
|
200
|
+
],
|
|
201
|
+
},
|
|
202
|
+
documentation: {
|
|
203
|
+
name: "documentation",
|
|
204
|
+
filePatterns: [
|
|
205
|
+
// Tier 1: Smoking Guns - Standard documentation files
|
|
206
|
+
{ pattern: /^(?:README|CODE_OF_CONDUCT|CONTRIBUTING|SECURITY|ROADMAP|LICENSE|CHANGELOG|TODO|INSTALL)(?:\.(md|txt|rst))?$/i, weight: 20.0 },
|
|
207
|
+
{ pattern: /^.*\/(?:docs?|documentation|guides?|tutorials?|help|wiki)\//i, weight: 8.0 },
|
|
208
|
+
],
|
|
209
|
+
contentPatterns: [
|
|
210
|
+
// Tier 1: Smoking Guns - Installation and setup
|
|
211
|
+
{ pattern: /\b(?:npm\s+install|pip\s+install|yarn\s+add|git\s+clone|composer\s+install|brew\s+install)\b/ig, weight: 12.0 },
|
|
212
|
+
{ pattern: /\b(?:installation|getting\s+started|quick\s+start|setup\s+guide|development\s+setup)\b/ig, weight: 10.0 },
|
|
213
|
+
{ pattern: /\b(?:gpl-[\w\d.]+|mit)\s+license/ig, weight: 10.0 },
|
|
214
|
+
{ pattern: /^```(?:json|java|\w+script|cpp|bash|shell)\n[\s\S]+```/g, weight: 10.0 },
|
|
215
|
+
// Tier 2: Strong Contextual - User guidance and API docs
|
|
216
|
+
{ pattern: /\b(?:you\s+(?:can|should|need\s+to|must)|to\s+(?:do\s+this|get\s+started)|follow\s+these\s+steps|here's\s+how)\b/ig, weight: 6.0 },
|
|
217
|
+
{ pattern: /^#{1,6}\s*(?:installation|usage|api\s+reference|getting\s+started|tutorial|examples?|configuration)\s*$/gim, weight: 6.0 },
|
|
218
|
+
{ pattern: /\b(?:GET|POST|PUT|DELETE|PATCH)\s+\/\w+/g, weight: 5.0 },
|
|
219
|
+
// Tier 3: Supporting Evidence - Documentation vocabulary
|
|
220
|
+
{ pattern: /\b(?:tutorial|guide|walkthrough|step-by-step|how-to|quick\s+reference)\b/ig, weight: 2.5 },
|
|
221
|
+
{ pattern: /\b(?:endpoint|route|parameter|response|request|authentication)\b/ig, weight: 2.0 },
|
|
222
|
+
{ pattern: /\b(?:note|warning|tip|important|caution):\s*\w/ig, weight: 2.0 },
|
|
223
|
+
// Tier 4: Noise - Code examples in docs
|
|
224
|
+
{ pattern: /```[\s\S]*?```/g, weight: 0.1 },
|
|
225
|
+
],
|
|
226
|
+
},
|
|
227
|
+
technical: {
|
|
228
|
+
name: "technical",
|
|
229
|
+
filePatterns: [
|
|
230
|
+
{ pattern: /\.(log|conf|cfg|config|ini|toml|yaml|yml|xml)$/i, weight: 12.0 },
|
|
231
|
+
{ pattern: /^.*\/(logs?|config|infrastructure|system|server)\//i, weight: 8.0 },
|
|
232
|
+
{ pattern: /^(?:docker|k8s|terraform|ansible).*\.(yml|yaml|json)$/i, weight: 10.0 },
|
|
233
|
+
],
|
|
234
|
+
contentPatterns: [
|
|
235
|
+
// Tier 1: Smoking Guns - Log patterns and timestamps
|
|
236
|
+
{ pattern: /^\d{4}-\d{2}-\d{2}[\sT]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?/gm, weight: 10.0 },
|
|
237
|
+
{ pattern: /\b(?:ERROR|WARN|INFO|DEBUG|FATAL|TRACE)\s*[:|\[]/ig, weight: 8.0 },
|
|
238
|
+
// Tier 2: Strong Contextual - System and infrastructure
|
|
239
|
+
{ pattern: /\b(?:server|service|daemon|process|thread)\b/ig, weight: 5.0 },
|
|
240
|
+
{ pattern: /\b(?:configuration|config|setting|parameter)\b/ig, weight: 4.0 },
|
|
241
|
+
{ pattern: /^\s*(?:host|port|url|path|timeout)[:=]/gim, weight: 4.0 },
|
|
242
|
+
// Tier 3: Supporting Evidence - Technical vocabulary
|
|
243
|
+
{ pattern: /\b(?:timeout|connection|network|database|cache)\b/ig, weight: 2.0 },
|
|
244
|
+
{ pattern: /\b(?:deploy|deployment|build|release|version)\b/ig, weight: 2.0 },
|
|
245
|
+
{ pattern: /\b(?:cpu|memory|disk|load|performance)\b/ig, weight: 1.5 },
|
|
246
|
+
// Tier 4: Noise - Generic tech terms
|
|
247
|
+
{ pattern: /\b(?:system|data|user|file)\b/ig, weight: 0.1 },
|
|
248
|
+
],
|
|
249
|
+
},
|
|
250
|
+
narrative: {
|
|
251
|
+
name: "narrative",
|
|
252
|
+
filePatterns: [
|
|
253
|
+
{ pattern: /\b(?:article|story|blog|news|opinion|essay|editorial).*\.(?:md|txt)$/i, weight: 8.0 },
|
|
254
|
+
{ pattern: /^.*\/(articles?|posts?|stories?|content|editorial)\//i, weight: 6.0 },
|
|
255
|
+
],
|
|
256
|
+
contentPatterns: [
|
|
257
|
+
// Tier 1: Smoking Guns - Narrative structure
|
|
258
|
+
{ pattern: /^The\s+\w+\s+of\s+/gim, weight: 8.0 },
|
|
259
|
+
// Tier 2: Strong Contextual - Narrative discourse
|
|
260
|
+
{ pattern: /\b(?:furthermore|however|therefore|moreover|nevertheless|meanwhile|subsequently|consequently)\b/ig, weight: 5.0 },
|
|
261
|
+
{ pattern: /\b(?:discusses|explores|examines|investigates|analyzes|argues|contends|suggests)\b/ig, weight: 4.0 },
|
|
262
|
+
{ pattern: /\b(?:according\s+to|research\s+shows|studies\s+indicate|experts\s+believe)\b/ig, weight: 4.0 },
|
|
263
|
+
// Tier 3: Supporting Evidence - Narrative vocabulary
|
|
264
|
+
{ pattern: /\b(?:article|story|report|analysis|opinion|essay)\b/ig, weight: 2.0 },
|
|
265
|
+
{ pattern: /\b(?:conclusion|summary|findings|impact)\b/ig, weight: 2.0 },
|
|
266
|
+
{ pattern: /\b(?:industry|society|community|population|public)\b/ig, weight: 1.5 },
|
|
267
|
+
// Tier 4: Noise - Common connectors
|
|
268
|
+
{ pattern: /\b(?:and|but|or|so|then|also)\b/ig, weight: 0.05 },
|
|
269
|
+
],
|
|
270
|
+
},
|
|
271
|
+
reference: {
|
|
272
|
+
name: "reference",
|
|
273
|
+
filePatterns: [
|
|
274
|
+
{ pattern: /^.*\/(reference|glossary|catalog|index|specs?|standards?)\//i, weight: 10.0 },
|
|
275
|
+
{ pattern: /\b(?:glossary|dictionary|catalog|index|reference).*\.(md|txt)$/i, weight: 10.0 },
|
|
276
|
+
{ pattern: /^(?:spec|standard|rfc|definition).*\.(md|txt)$/i, weight: 8.0 },
|
|
277
|
+
],
|
|
278
|
+
contentPatterns: [
|
|
279
|
+
// Tier 1: Smoking Guns - Reference structure
|
|
280
|
+
{ pattern: /^[A-Z]{2,}:\s+/gm, weight: 9.0 },
|
|
281
|
+
{ pattern: /^[\w\s]+:\s+[\w\s]/gm, weight: 7.0 },
|
|
282
|
+
// Tier 2: Strong Contextual - Reference language
|
|
283
|
+
{ pattern: /\b(?:glossary|dictionary|catalog|index|reference)\b/ig, weight: 5.0 },
|
|
284
|
+
{ pattern: /\b(?:definition|term|entry|specification|standard)\b/ig, weight: 4.0 },
|
|
285
|
+
{ pattern: /\b(?:see\s+also|cross.?reference|related|synonym)\b/ig, weight: 4.0 },
|
|
286
|
+
// Tier 3: Supporting Evidence - Reference vocabulary
|
|
287
|
+
{ pattern: /\b(?:alphabetically|categorized|indexed|listed)\b/ig, weight: 2.0 },
|
|
288
|
+
{ pattern: /\b(?:abbreviation|acronym|symbol|notation)\b/ig, weight: 2.0 },
|
|
289
|
+
// Tier 4: Noise - List markers
|
|
290
|
+
{ pattern: /^\*\s+\w+/gm, weight: 0.2 },
|
|
291
|
+
],
|
|
292
|
+
},
|
|
293
|
+
};
|
|
294
|
+
//# sourceMappingURL=CONTENT_CLASSES.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CONTENT_CLASSES.js","sourceRoot":"","sources":["../../../../src/core/processor/classifier/CONTENT_CLASSES.ts"],"names":[],"mappings":";;;AAGa,QAAA,eAAe,GAA6C;IACvE,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,YAAY,EAAE;YACZ,mCAAmC;YACnC,EAAE,OAAO,EAAE,yBAAyB,EAAE,MAAM,EAAE,IAAI,EAAE;YACpD,EAAE,OAAO,EAAE,mDAAmD,EAAE,MAAM,EAAE,IAAI,EAAE;YAC9E,EAAE,OAAO,EAAE,iCAAiC,EAAE,MAAM,EAAE,IAAI,EAAE;YAC5D,EAAE,OAAO,EAAE,4DAA4D,EAAE,MAAM,EAAE,GAAG,EAAE;SACvF;QACD,eAAe,EAAE;YACf,8DAA8D;YAC9D,EAAE,OAAO,EAAE,4EAA4E,EAAE,MAAM,EAAE,IAAI,EAAE;YACvG,EAAE,OAAO,EAAE,iDAAiD,EAAE,MAAM,EAAE,IAAI,EAAE;YAC5E,EAAE,OAAO,EAAE,4DAA4D,EAAE,MAAM,EAAE,IAAI,EAAE;YAEvF,iEAAiE;YACjE,EAAE,OAAO,EAAE,2CAA2C,EAAE,MAAM,EAAE,GAAG,EAAE;YACrE,EAAE,OAAO,EAAE,0EAA0E,EAAE,MAAM,EAAE,GAAG,EAAE;YACpG,EAAE,OAAO,EAAE,uCAAuC,EAAE,MAAM,EAAE,GAAG,EAAE;YAEjE,oEAAoE;YACpE,EAAE,OAAO,EAAE,6CAA6C,EAAE,MAAM,EAAE,GAAG,EAAE;YACvE,EAAE,OAAO,EAAE,yCAAyC,EAAE,MAAM,EAAE,GAAG,EAAE;YACnE,EAAE,OAAO,EAAE,2DAA2D,EAAE,MAAM,EAAE,GAAG,EAAE;YAErF,6CAA6C;YAC7C,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,EAAE;SACrC;KACF;IAED,SAAS,EAAE;QACT,IAAI,EAAE,WAAW;QACjB,YAAY,EAAE;YACZ,EAAE,OAAO,EAAE,gGAAgG,EAAE,MAAM,EAAE,IAAI,EAAE;YAC3H,EAAE,OAAO,EAAE,kEAAkE,EAAE,MAAM,EAAE,GAAG,EAAE;YAC5F,EAAE,OAAO,EAAE,8CAA8C,EAAE,MAAM,EAAE,IAAI,EAAE;SAC1E;QACD,eAAe,EAAE;YACf,kDAAkD;YAClD,EAAE,OAAO,EAAE,6EAA6E,EAAE,MAAM,EAAE,IAAI,EAAE;YACxG,EAAE,OAAO,EAAE,wEAAwE,EAAE,MAAM,EAAE,GAAG,EAAE;YAElG,8DAA8D;YAC9D,EAAE,OAAO,EAAE,qFAAqF,EAAE,MAAM,EAAE,GAAG,EAAE;YAC/G,EAAE,OAAO,EAAE,kFAAkF,EAAE,MAAM,EAAE,GAAG,EAAE;YAC5G,EAAE,OAAO,EAAE,4EAA4E,EAAE,MAAM,EAAE,GAAG,EAAE;YAEtG,qDAAqD;YACrD,EAAE,OAAO,EAAE,8EAA8E,EAAE,MAAM,EAAE,GAAG,EAAE;YACxG,EAAE,OAAO,EAAE,oDAAoD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC9E,EAAE,OAAO,EAAE,oDAAoD,EAAE,MAAM,EAAE,GAAG,EAAE;YAE9E,8DAA8D;YAC9D,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,EAAE;SAC5C;KACF;IAED,OAAO,EAAE;QACP,IAAI,EAAE,SAAS;QACf,YAAY,EAAE;YACZ,EAAE,OAAO,EAAE,iFAAiF,EAAE,MAAM,EAAE,IAAI,EAAE;YAC5G,EAAE,OAAO,EAAE,0DAA0D,EAAE,MAAM,EAAE,GAAG,EAAE;YACpF,EAAE,OAAO,EAAE,sDAAsD,EAAE,MAAM,EAAE,GAAG,EAAE;SACjF;QACD,eAAe,EAAE;YACf,wDAAwD;YACxD,EAAE,OAAO,EAAE,uCAAuC,EAAE,MAAM,EAAE,IAAI,EAAE;YAClE,EAAE,OAAO,EAAE,gDAAgD,EAAE,MAAM,EAAE,GAAG,EAAE;YAE1E,sDAAsD;YACtD,EAAE,OAAO,EAAE,4HAA4H,EAAE,MAAM,EAAE,GAAG,EAAE;YACtJ,EAAE,OAAO,EAAE,4FAA4F,EAAE,MAAM,EAAE,GAAG,EAAE;YACtH,EAAE,OAAO,EAAE,sEAAsE,EAAE,MAAM,EAAE,GAAG,EAAE;YAEhG,oDAAoD;YACpD,EAAE,OAAO,EAAE,kFAAkF,EAAE,MAAM,EAAE,GAAG,EAAE;YAC5G,EAAE,OAAO,EAAE,mEAAmE,EAAE,MAAM,EAAE,GAAG,EAAE;YAC7F,EAAE,OAAO,EAAE,uFAAuF,EAAE,MAAM,EAAE,GAAG,EAAE;YAEjH,wCAAwC;YACxC,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,EAAE;SAC5C;KACF;IAED,KAAK,EAAE;QACL,IAAI,EAAE,OAAO;QACb,YAAY,EAAE;YACZ,EAAE,OAAO,EAAE,uDAAuD,EAAE,MAAM,EAAE,GAAG,EAAE;YACjF,EAAE,OAAO,EAAE,6DAA6D,EAAE,MAAM,EAAE,GAAG,EAAE;YACvF,EAAE,OAAO,EAAE,uCAAuC,EAAE,MAAM,EAAE,IAAI,EAAE;YAClE,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,IAAI,EAAE;SACxD;QACD,eAAe,EAAE;YACf,kDAAkD;YAClD,EAAE,OAAO,EAAE,8EAA8E,EAAE,MAAM,EAAE,IAAI,EAAE;YACzG,EAAE,OAAO,EAAE,0DAA0D,EAAE,MAAM,EAAE,IAAI,EAAE;YAErF,4DAA4D;YAC5D,EAAE,OAAO,EAAE,sEAAsE,EAAE,MAAM,EAAE,GAAG,EAAE;YAChG,EAAE,OAAO,EAAE,qEAAqE,EAAE,MAAM,EAAE,GAAG,EAAE;YAC/F,EAAE,OAAO,EAAE,6EAA6E,EAAE,MAAM,EAAE,GAAG,EAAE;YAEvG,iDAAiD;YACjD,EAAE,OAAO,EAAE,kEAAkE,EAAE,MAAM,EAAE,GAAG,EAAE;YAC5F,EAAE,OAAO,EAAE,oDAAoD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC9E,EAAE,OAAO,EAAE,iDAAiD,EAAE,MAAM,EAAE,GAAG,EAAE;YAE3E,mCAAmC;YACnC,EAAE,OAAO,EAAE,wCAAwC,EAAE,MAAM,EAAE,GAAG,EAAE;SACnE;KACF;IAED,QAAQ,EAAE;QACR,IAAI,EAAE,UAAU;QAChB,YAAY,EAAE;YACZ,iDAAiD;YACjD,EAAE,OAAO,EAAE,wCAAwC,EAAE,MAAM,EAAE,IAAI,EAAE;YACnE,EAAE,OAAO,EAAE,2FAA2F,EAAE,MAAM,EAAE,IAAI,EAAE;YACtH,EAAE,OAAO,EAAE,gEAAgE,EAAE,MAAM,EAAE,GAAG,EAAE;SAC3F;QACD,eAAe,EAAE;YACf,0DAA0D;YAC1D,EAAE,OAAO,EAAE,8CAA8C,EAAE,MAAM,EAAE,IAAI,EAAE;YACzE,EAAE,OAAO,EAAE,gOAAgO,EAAE,MAAM,EAAE,IAAI,EAAE;YAC3P,EAAE,OAAO,EAAE,2CAA2C,EAAE,MAAM,EAAE,IAAI,EAAE;YACtE,EAAE,OAAO,EAAE,wCAAwC,EAAE,MAAM,EAAE,IAAI,EAAE;YAEnE,gDAAgD;YAChD,EAAE,OAAO,EAAE,kIAAkI,EAAE,MAAM,EAAE,IAAI,EAAE;YAC7J,EAAE,OAAO,EAAE,4HAA4H,EAAE,MAAM,EAAE,GAAG,EAAE;YACtJ,EAAE,OAAO,EAAE,yEAAyE,EAAE,MAAM,EAAE,GAAG,EAAE;YACnG,EAAE,OAAO,EAAE,wEAAwE,EAAE,MAAM,EAAE,GAAG,EAAE;YAElG,qDAAqD;YACrD,EAAE,OAAO,EAAE,mGAAmG,EAAE,MAAM,EAAE,GAAG,EAAE;YAC7H,EAAE,OAAO,EAAE,oCAAoC,EAAE,MAAM,EAAE,GAAG,EAAE;YAC9D,EAAE,OAAO,EAAE,mEAAmE,EAAE,MAAM,EAAE,GAAG,EAAE;YAC7F,EAAE,OAAO,EAAE,mDAAmD,EAAE,MAAM,EAAE,GAAG,EAAE;YAE7E,sCAAsC;YACtC,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,EAAE;SAC5C;KACF;IAED,UAAU,EAAE;QACV,IAAI,EAAE,YAAY;QAClB,YAAY,EAAE;YACZ,EAAE,OAAO,EAAE,yDAAyD,EAAE,MAAM,EAAE,IAAI,EAAE;YACpF,EAAE,OAAO,EAAE,+DAA+D,EAAE,MAAM,EAAE,IAAI,EAAE;YAC1F,EAAE,OAAO,EAAE,iDAAiD,EAAE,MAAM,EAAE,GAAG,EAAE;SAC5E;QACD,eAAe,EAAE;YACf,8CAA8C;YAC9C,EAAE,OAAO,EAAE,+DAA+D,EAAE,MAAM,EAAE,IAAI,EAAE;YAC1F,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,GAAG,EAAE;YACtD,EAAE,OAAO,EAAE,wCAAwC,EAAE,MAAM,EAAE,GAAG,EAAE;YAClE,EAAE,OAAO,EAAE,4DAA4D,EAAE,MAAM,EAAE,GAAG,EAAE;YAEtF,+CAA+C;YAC/C,EAAE,OAAO,EAAE,yDAAyD,EAAE,MAAM,EAAE,GAAG,EAAE;YACnF,EAAE,OAAO,EAAE,+DAA+D,EAAE,MAAM,EAAE,GAAG,EAAE;YACzF,EAAE,OAAO,EAAE,+CAA+C,EAAE,MAAM,EAAE,GAAG,EAAE;YAEzE,4DAA4D;YAC5D,EAAE,OAAO,EAAE,mFAAmF,EAAE,MAAM,EAAE,GAAG,EAAE;YAC7G,EAAE,OAAO,EAAE,6CAA6C,EAAE,MAAM,EAAE,GAAG,EAAE;YACvE,EAAE,OAAO,EAAE,6CAA6C,EAAE,MAAM,EAAE,GAAG,EAAE;YAEvE,8BAA8B;YAC9B,EAAE,OAAO,EAAE,6BAA6B,EAAE,MAAM,EAAE,GAAG,EAAE;SACxD;KACF;IAED,OAAO,EAAE;QACP,IAAI,EAAE,SAAS;QACf,YAAY,EAAE;YACZ,8CAA8C;YAC9C,EAAE,OAAO,EAAE,yBAAyB,EAAE,MAAM,EAAE,IAAI,EAAE;YACpD,EAAE,OAAO,EAAE,0DAA0D,EAAE,MAAM,EAAE,GAAG,EAAE;SACrF;QACD,eAAe,EAAE;YACf,8DAA8D;YAC9D,EAAE,OAAO,EAAE,2BAA2B,EAAE,MAAM,EAAE,IAAI,EAAE;YACtD,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,IAAI,EAAE;YACvD,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,IAAI,EAAE;YACvD,EAAE,OAAO,EAAE,4BAA4B,EAAE,MAAM,EAAE,IAAI,EAAE;YACvD,EAAE,OAAO,EAAE,8BAA8B,EAAE,MAAM,EAAE,IAAI,EAAE;YAEzD,0DAA0D;YAC1D,EAAE,OAAO,EAAE,0BAA0B,EAAE,MAAM,EAAE,GAAG,EAAE;YACpD,EAAE,OAAO,EAAE,iCAAiC,EAAE,MAAM,EAAE,GAAG,EAAE;YAE3D,8CAA8C;YAC9C,EAAE,OAAO,EAAE,yCAAyC,EAAE,MAAM,EAAE,GAAG,EAAE;YACnE,EAAE,OAAO,EAAE,2CAA2C,EAAE,MAAM,EAAE,GAAG,EAAE;YAErE,sDAAsD;YACtD,EAAE,OAAO,EAAE,qBAAqB,EAAE,MAAM,EAAE,GAAG,EAAE;SAChD;KACF;IAED,aAAa,EAAE;QACb,IAAI,EAAE,eAAe;QACrB,YAAY,EAAE;YACZ,EAAE,OAAO,EAAE,+DAA+D,EAAE,MAAM,EAAE,IAAI,EAAE;YAC1F,EAAE,OAAO,EAAE,2DAA2D,EAAE,MAAM,EAAE,IAAI,EAAE;YACtF,EAAE,OAAO,EAAE,qCAAqC,EAAE,MAAM,EAAE,GAAG,EAAE;SAChE;QACD,eAAe,EAAE;YACf,yCAAyC;YACzC,EAAE,OAAO,EAAE,oDAAoD,EAAE,MAAM,EAAE,IAAI,EAAE;YAC/E,EAAE,OAAO,EAAE,0BAA0B,EAAE,MAAM,EAAE,IAAI,EAAE;YACrD,EAAE,OAAO,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,EAAE;YAE1C,qDAAqD;YACrD,EAAE,OAAO,EAAE,mDAAmD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC7E,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE;YACnC,EAAE,OAAO,EAAE,sDAAsD,EAAE,MAAM,EAAE,GAAG,EAAE;YAEhF,yDAAyD;YACzD,EAAE,OAAO,EAAE,oDAAoD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC9E,EAAE,OAAO,EAAE,gDAAgD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC1E,EAAE,OAAO,EAAE,wBAAwB,EAAE,MAAM,EAAE,GAAG,EAAE;YAElD,mCAAmC;YACnC,EAAE,OAAO,EAAE,kCAAkC,EAAE,MAAM,EAAE,GAAG,EAAE;SAC7D;KACF;IAED,aAAa,EAAE;QACb,IAAI,EAAE,eAAe;QACrB,YAAY,EAAE;YACZ,sDAAsD;YACtD,EAAE,OAAO,EAAE,+GAA+G,EAAE,MAAM,EAAE,IAAI,EAAE;YAC1I,EAAE,OAAO,EAAE,8DAA8D,EAAE,MAAM,EAAE,GAAG,EAAE;SACzF;QACD,eAAe,EAAE;YACf,gDAAgD;YAChD,EAAE,OAAO,EAAE,gGAAgG,EAAE,MAAM,EAAE,IAAI,EAAE;YAC3H,EAAE,OAAO,EAAE,0FAA0F,EAAE,MAAM,EAAE,IAAI,EAAE;YACrH,EAAE,OAAO,EAAE,oCAAoC,EAAE,MAAM,EAAE,IAAI,EAAE;YAE/D,EAAE,OAAO,EAAE,yDAAyD,EAAE,MAAM,EAAE,IAAI,EAAE;YAEpF,yDAAyD;YACzD,EAAE,OAAO,EAAE,oHAAoH,EAAE,MAAM,EAAE,GAAG,EAAE;YAC9I,EAAE,OAAO,EAAE,4GAA4G,EAAE,MAAM,EAAE,GAAG,EAAE;YACtI,EAAE,OAAO,EAAE,0CAA0C,EAAE,MAAM,EAAE,GAAG,EAAE;YAEpE,yDAAyD;YACzD,EAAE,OAAO,EAAE,4EAA4E,EAAE,MAAM,EAAE,GAAG,EAAE;YACtG,EAAE,OAAO,EAAE,oEAAoE,EAAE,MAAM,EAAE,GAAG,EAAE;YAC9F,EAAE,OAAO,EAAE,kDAAkD,EAAE,MAAM,EAAE,GAAG,EAAE;YAE5E,wCAAwC;YACxC,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,GAAG,EAAE;SAC5C;KACF;IAED,SAAS,EAAE;QACT,IAAI,EAAE,WAAW;QACjB,YAAY,EAAE;YACZ,EAAE,OAAO,EAAE,iDAAiD,EAAE,MAAM,EAAE,IAAI,EAAE;YAC5E,EAAE,OAAO,EAAE,qDAAqD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC/E,EAAE,OAAO,EAAE,wDAAwD,EAAE,MAAM,EAAE,IAAI,EAAE;SACpF;QACD,eAAe,EAAE;YACf,qDAAqD;YACrD,EAAE,OAAO,EAAE,4EAA4E,EAAE,MAAM,EAAE,IAAI,EAAE;YACvG,EAAE,OAAO,EAAE,oDAAoD,EAAE,MAAM,EAAE,GAAG,EAAE;YAE9E,wDAAwD;YACxD,EAAE,OAAO,EAAE,gDAAgD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC1E,EAAE,OAAO,EAAE,kDAAkD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC5E,EAAE,OAAO,EAAE,2CAA2C,EAAE,MAAM,EAAE,GAAG,EAAE;YAErE,qDAAqD;YACrD,EAAE,OAAO,EAAE,qDAAqD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC/E,EAAE,OAAO,EAAE,mDAAmD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC7E,EAAE,OAAO,EAAE,4CAA4C,EAAE,MAAM,EAAE,GAAG,EAAE;YAEtE,qCAAqC;YACrC,EAAE,OAAO,EAAE,iCAAiC,EAAE,MAAM,EAAE,GAAG,EAAE;SAC5D;KACF;IAED,SAAS,EAAE;QACT,IAAI,EAAE,WAAW;QACjB,YAAY,EAAE;YACZ,EAAE,OAAO,EAAE,uEAAuE,EAAE,MAAM,EAAE,GAAG,EAAE;YACjG,EAAE,OAAO,EAAE,uDAAuD,EAAE,MAAM,EAAE,GAAG,EAAE;SAClF;QACD,eAAe,EAAE;YACf,6CAA6C;YAC7C,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,EAAE,GAAG,EAAE;YAEjD,kDAAkD;YAClD,EAAE,OAAO,EAAE,mGAAmG,EAAE,MAAM,EAAE,GAAG,EAAE;YAC7H,EAAE,OAAO,EAAE,sFAAsF,EAAE,MAAM,EAAE,GAAG,EAAE;YAChH,EAAE,OAAO,EAAE,gFAAgF,EAAE,MAAM,EAAE,GAAG,EAAE;YAE1G,qDAAqD;YACrD,EAAE,OAAO,EAAE,uDAAuD,EAAE,MAAM,EAAE,GAAG,EAAE;YACjF,EAAE,OAAO,EAAE,8CAA8C,EAAE,MAAM,EAAE,GAAG,EAAE;YACxE,EAAE,OAAO,EAAE,wDAAwD,EAAE,MAAM,EAAE,GAAG,EAAE;YAElF,oCAAoC;YACpC,EAAE,OAAO,EAAE,mCAAmC,EAAE,MAAM,EAAE,IAAI,EAAE;SAC/D;KACF;IAED,SAAS,EAAE;QACT,IAAI,EAAE,WAAW;QACjB,YAAY,EAAE;YACZ,EAAE,OAAO,EAAE,8DAA8D,EAAE,MAAM,EAAE,IAAI,EAAE;YACzF,EAAE,OAAO,EAAE,iEAAiE,EAAE,MAAM,EAAE,IAAI,EAAE;YAC5F,EAAE,OAAO,EAAE,iDAAiD,EAAE,MAAM,EAAE,GAAG,EAAE;SAC5E;QACD,eAAe,EAAE;YACf,6CAA6C;YAC7C,EAAE,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAE,GAAG,EAAE;YAC5C,EAAE,OAAO,EAAE,sBAAsB,EAAE,MAAM,EAAE,GAAG,EAAE;YAEhD,iDAAiD;YACjD,EAAE,OAAO,EAAE,uDAAuD,EAAE,MAAM,EAAE,GAAG,EAAE;YACjF,EAAE,OAAO,EAAE,wDAAwD,EAAE,MAAM,EAAE,GAAG,EAAE;YAClF,EAAE,OAAO,EAAE,uDAAuD,EAAE,MAAM,EAAE,GAAG,EAAE;YAEjF,qDAAqD;YACrD,EAAE,OAAO,EAAE,qDAAqD,EAAE,MAAM,EAAE,GAAG,EAAE;YAC/E,EAAE,OAAO,EAAE,gDAAgD,EAAE,MAAM,EAAE,GAAG,EAAE;YAE1E,+BAA+B;YAC/B,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE;SACxC;KACF;CACF,CAAC"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.CascadeContentClassifier = exports.DEFAULT_MAX_ESCALATIONS = void 0;
|
|
13
|
+
const vocabulary_1 = require("../../knowledge/vocabulary");
|
|
14
|
+
const trace_1 = require("../../trace");
|
|
15
|
+
/**
|
|
16
|
+
* Per-run cap on LLM tie-break escalations — a cost guard so an ambiguous corpus
|
|
17
|
+
* can't fan out into unbounded LLM calls. (A1 will lift this to config.)
|
|
18
|
+
*/
|
|
19
|
+
exports.DEFAULT_MAX_ESCALATIONS = 50;
|
|
20
|
+
/**
|
|
21
|
+
* Phase-B confidence cascade: the cheap heuristic decides the easy majority; only
|
|
22
|
+
* a genuine *tie* is escalated to the LLM, which disambiguates among the two
|
|
23
|
+
* candidates the heuristic couldn't separate.
|
|
24
|
+
*
|
|
25
|
+
* The escalation trigger reuses the exact Phase-A gate: `activeDomainClasses`
|
|
26
|
+
* returns **two** classes precisely in the "close" branch (both clear the floor and
|
|
27
|
+
* are within the mixed-domain margin). So the cascade fires where the deterministic
|
|
28
|
+
* path would otherwise route *multi* — trying to resolve it to a single domain
|
|
29
|
+
* before falling back to multi. Decisive (1) and abstain (0) pass through untouched,
|
|
30
|
+
* and so does every case once the per-run budget is spent or no LLM is wired, so a
|
|
31
|
+
* `cascade` run degrades cleanly to the deterministic Phase-A behavior.
|
|
32
|
+
*
|
|
33
|
+
* The LLM supplies only the *class* pick — its (miscalibrated) confidence is
|
|
34
|
+
* ignored; the winner inherits the heuristic's combined mass for the tied pair.
|
|
35
|
+
*/
|
|
36
|
+
class CascadeContentClassifier {
|
|
37
|
+
constructor(heuristic, llm, logger, maxEscalations = exports.DEFAULT_MAX_ESCALATIONS) {
|
|
38
|
+
this.heuristic = heuristic;
|
|
39
|
+
this.llm = llm;
|
|
40
|
+
this.logger = logger;
|
|
41
|
+
this.maxEscalations = maxEscalations;
|
|
42
|
+
this.escalationsUsed = 0;
|
|
43
|
+
}
|
|
44
|
+
classify(content, path) {
|
|
45
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
46
|
+
var _a;
|
|
47
|
+
const results = yield this.heuristic.classify(content, path);
|
|
48
|
+
// Only a top-2 tie (the gate's "close" branch) is worth an LLM call.
|
|
49
|
+
const active = (0, vocabulary_1.activeDomainClasses)(results);
|
|
50
|
+
if (active.length < 2)
|
|
51
|
+
return results;
|
|
52
|
+
if (!this.llm || this.escalationsUsed >= this.maxEscalations) {
|
|
53
|
+
// No tie-break available → keep the calibrated-multi result (Phase-A behavior).
|
|
54
|
+
return results;
|
|
55
|
+
}
|
|
56
|
+
this.escalationsUsed++;
|
|
57
|
+
let pick;
|
|
58
|
+
try {
|
|
59
|
+
const llmResults = yield this.llm.classify(content, path);
|
|
60
|
+
pick = (_a = llmResults[0]) === null || _a === void 0 ? void 0 : _a.class;
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
this.logger.warn(`Cascade LLM tie-break failed; keeping heuristic multi: ${err}`);
|
|
64
|
+
this.traceEscalation(path, results, active, null);
|
|
65
|
+
return results;
|
|
66
|
+
}
|
|
67
|
+
this.traceEscalation(path, results, active, pick !== null && pick !== void 0 ? pick : null);
|
|
68
|
+
// Honor the LLM only when it picks one of the two tied candidates — a single
|
|
69
|
+
// call must not promote a class the heuristic ranked far down.
|
|
70
|
+
if (!pick || !active.includes(pick)) {
|
|
71
|
+
this.logger.debug(`Cascade tie-break pick "${pick}" not in [${active.join(", ")}]; keeping heuristic multi`);
|
|
72
|
+
return results;
|
|
73
|
+
}
|
|
74
|
+
return this.collapseTie(results, active[0], active[1], pick);
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
/** Debug trace: record a cascade tie-break escalation (observe-only). */
|
|
78
|
+
traceEscalation(path, results, active, pick) {
|
|
79
|
+
if (!trace_1.trace.enabled)
|
|
80
|
+
return;
|
|
81
|
+
trace_1.trace.emit({
|
|
82
|
+
stage: "classify", type: "classification", file: path,
|
|
83
|
+
distribution: results.map((r) => ({ class: r.class, confidence: r.confidence })),
|
|
84
|
+
gate: "multi", activeClasses: active, escalated: true,
|
|
85
|
+
tieBreak: { tied: [active[0], active[1]], pick },
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Resolve a top-2 tie to `winner`: it absorbs the pair's combined confidence and
|
|
90
|
+
* the other tied class is demoted to 0, so the downstream gate now routes a
|
|
91
|
+
* single domain (`winner`).
|
|
92
|
+
*/
|
|
93
|
+
collapseTie(results, a, b, winner) {
|
|
94
|
+
var _a, _b, _c, _d;
|
|
95
|
+
const loser = winner === a ? b : a;
|
|
96
|
+
const confA = (_b = (_a = results.find((r) => r.class === a)) === null || _a === void 0 ? void 0 : _a.confidence) !== null && _b !== void 0 ? _b : 0;
|
|
97
|
+
const confB = (_d = (_c = results.find((r) => r.class === b)) === null || _c === void 0 ? void 0 : _c.confidence) !== null && _d !== void 0 ? _d : 0;
|
|
98
|
+
this.logger.debug(`Cascade tie-break: ${a}/${b} → ${winner}`);
|
|
99
|
+
return results
|
|
100
|
+
.map((r) => r.class === winner
|
|
101
|
+
? Object.assign(Object.assign({}, r), { confidence: confA + confB }) : r.class === loser
|
|
102
|
+
? Object.assign(Object.assign({}, r), { confidence: 0 }) : r)
|
|
103
|
+
.sort((x, y) => y.confidence - x.confidence);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
exports.CascadeContentClassifier = CascadeContentClassifier;
|
|
107
|
+
//# sourceMappingURL=CascadeContentClassifier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"CascadeContentClassifier.js","sourceRoot":"","sources":["../../../../src/core/processor/classifier/CascadeContentClassifier.ts"],"names":[],"mappings":";;;;;;;;;;;;AAEA,2DAAiE;AAEjE,uCAAoC;AAEpC;;;GAGG;AACU,QAAA,uBAAuB,GAAG,EAAE,CAAC;AAE1C;;;;;;;;;;;;;;;GAeG;AACH,MAAa,wBAAwB;IAGnC,YACmB,SAA6B,EAC7B,GAAmC,EACnC,MAAc,EACd,iBAAyB,+BAAuB;QAHhD,cAAS,GAAT,SAAS,CAAoB;QAC7B,QAAG,GAAH,GAAG,CAAgC;QACnC,WAAM,GAAN,MAAM,CAAQ;QACd,mBAAc,GAAd,cAAc,CAAkC;QAN3D,oBAAe,GAAG,CAAC,CAAC;IAOzB,CAAC;IAEE,QAAQ,CACZ,OAAe,EACf,IAAY;;;YAEZ,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAE7D,qEAAqE;YACrE,MAAM,MAAM,GAAG,IAAA,gCAAmB,EAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,OAAO,CAAC;YAEtC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC7D,gFAAgF;gBAChF,OAAO,OAAO,CAAC;YACjB,CAAC;YAED,IAAI,CAAC,eAAe,EAAE,CAAC;YACvB,IAAI,IAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;gBAC1D,IAAI,GAAG,MAAA,UAAU,CAAC,CAAC,CAAC,0CAAE,KAAK,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,0DAA0D,GAAG,EAAE,CAChE,CAAC;gBACF,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;gBAClD,OAAO,OAAO,CAAC;YACjB,CAAC;YAED,IAAI,CAAC,eAAe,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,IAAI,CAAC,CAAC;YAE1D,6EAA6E;YAC7E,+DAA+D;YAC/D,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,2BAA2B,IAAI,aAAa,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,4BAA4B,CAC1F,CAAC;gBACF,OAAO,OAAO,CAAC;YACjB,CAAC;YAED,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/D,CAAC;KAAA;IAED,yEAAyE;IACjE,eAAe,CACrB,IAAY,EACZ,OAA+B,EAC/B,MAAsB,EACtB,IAAyB;QAEzB,IAAI,CAAC,aAAK,CAAC,OAAO;YAAE,OAAO;QAC3B,aAAK,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,IAAI;YACrD,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;YAChF,IAAI,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI;YACrD,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;SACjD,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACK,WAAW,CACjB,OAA+B,EAC/B,CAAe,EACf,CAAe,EACf,MAAoB;;QAEpB,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnC,MAAM,KAAK,GAAG,MAAA,MAAA,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,0CAAE,UAAU,mCAAI,CAAC,CAAC;QAClE,MAAM,KAAK,GAAG,MAAA,MAAA,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,0CAAE,UAAU,mCAAI,CAAC,CAAC;QAClE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,IAAI,CAAC,MAAM,MAAM,EAAE,CAAC,CAAC;QAE9D,OAAO,OAAO;aACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACT,CAAC,CAAC,KAAK,KAAK,MAAM;YAChB,CAAC,iCAAM,CAAC,KAAE,UAAU,EAAE,KAAK,GAAG,KAAK,IACnC,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,KAAK;YACnB,CAAC,iCAAM,CAAC,KAAE,UAAU,EAAE,CAAC,IACvB,CAAC,CAAC,CAAC,CACN;aACA,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IACjD,CAAC;CACF;AA9FD,4DA8FC"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.HeuristicContentClassifier = exports.DEFAULT_CROSS_VALIDATION_FACTOR = exports.DEFAULT_SOFTMAX_TEMPERATURE = void 0;
|
|
13
|
+
const CONTENT_CLASSES_1 = require("./CONTENT_CLASSES");
|
|
14
|
+
const shared_1 = require("../../../shared");
|
|
15
|
+
/**
|
|
16
|
+
* Softmax temperature applied to the raw per-class scores to turn them into a
|
|
17
|
+
* comparable distribution (S2). Lower = sharper (decisive single-domain); higher
|
|
18
|
+
* = flatter (more ties → the cascade's "close" branch in `activeDomainClasses`).
|
|
19
|
+
* Tuned on the A3 harness; overridable via `classifier.temperature` (A1).
|
|
20
|
+
*/
|
|
21
|
+
exports.DEFAULT_SOFTMAX_TEMPERATURE = 2.0;
|
|
22
|
+
/**
|
|
23
|
+
* Magnitude of the cross-validation penalty: each class subtracts every *other*
|
|
24
|
+
* class's positive patterns at this fraction of their weight. Overridable via
|
|
25
|
+
* `classifier.crossValidationFactor` (A1).
|
|
26
|
+
*/
|
|
27
|
+
exports.DEFAULT_CROSS_VALIDATION_FACTOR = 0.15;
|
|
28
|
+
class HeuristicContentClassifier {
|
|
29
|
+
constructor(logger, temperature = exports.DEFAULT_SOFTMAX_TEMPERATURE, crossValidationFactor = exports.DEFAULT_CROSS_VALIDATION_FACTOR) {
|
|
30
|
+
this.logger = logger;
|
|
31
|
+
this.temperature = temperature;
|
|
32
|
+
this.crossValidationFactor = crossValidationFactor;
|
|
33
|
+
this.crossValidatedClasses = {};
|
|
34
|
+
this.initializeCrossValidation();
|
|
35
|
+
}
|
|
36
|
+
initializeCrossValidation() {
|
|
37
|
+
// For each class, collect ALL other classes positive patterns as negatives
|
|
38
|
+
for (const [className, config] of Object.entries(CONTENT_CLASSES_1.CONTENT_CLASSES)) {
|
|
39
|
+
const negativePatterns = [];
|
|
40
|
+
// Collect positive patterns from ALL other classes
|
|
41
|
+
for (const [otherClassName, otherConfig] of Object.entries(CONTENT_CLASSES_1.CONTENT_CLASSES)) {
|
|
42
|
+
if (otherClassName !== className) {
|
|
43
|
+
// Add other classes positive patterns as negatives with reduced weight
|
|
44
|
+
for (const pattern of otherConfig.contentPatterns) {
|
|
45
|
+
negativePatterns.push({
|
|
46
|
+
pattern: pattern.pattern,
|
|
47
|
+
weight: -pattern.weight * this.crossValidationFactor, // negative, reduced magnitude
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
this.crossValidatedClasses[className] = Object.assign(Object.assign({}, config), { negativePatterns });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
classify(content, path) {
|
|
56
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
57
|
+
const raw = Object.values(this.crossValidatedClasses).map((config) => this.rawScore(content, path, config));
|
|
58
|
+
// Softmax over the raw scores → a comparable distribution across classes (S2),
|
|
59
|
+
// not 12 independent squashes. We return *all* classes ranked and let the single
|
|
60
|
+
// downstream gate (`activeDomainClasses`) decide abstain/single/multi — the old
|
|
61
|
+
// `> 0.7` self-filter is gone (S3), so the classifier ranks and the consumer gates.
|
|
62
|
+
const confidences = (0, shared_1.softmax)(raw.map((r) => r.total), this.temperature);
|
|
63
|
+
const results = raw.map((r, i) => ({
|
|
64
|
+
class: r.class,
|
|
65
|
+
confidence: confidences[i],
|
|
66
|
+
}));
|
|
67
|
+
for (let i = 0; i < raw.length; i++) {
|
|
68
|
+
const r = raw[i];
|
|
69
|
+
this.logger.debug(`[heuristic] ${r.class}: file=${r.fileScore.toFixed(2)} content=${r.contentScore.toFixed(2)} ` +
|
|
70
|
+
`neg=${r.negativeScore.toFixed(2)} total=${r.total.toFixed(2)} p=${confidences[i].toFixed(3)}`);
|
|
71
|
+
}
|
|
72
|
+
return results.sort((a, b) => b.confidence - a.confidence);
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
rawScore(content, path, config) {
|
|
76
|
+
// Track the three contributions separately so a debug run can explain *why*
|
|
77
|
+
// a file scored the way it did (the seam for tuning weights/thresholds).
|
|
78
|
+
let fileScore = 0;
|
|
79
|
+
let contentScore = 0;
|
|
80
|
+
let negativeScore = 0;
|
|
81
|
+
// File pattern scoring
|
|
82
|
+
for (const { pattern, weight } of config.filePatterns) {
|
|
83
|
+
if (pattern.test(path)) {
|
|
84
|
+
fileScore += weight;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Positive pattern scoring
|
|
88
|
+
for (const { pattern, weight } of config.contentPatterns) {
|
|
89
|
+
const matches = content.match(pattern) || [];
|
|
90
|
+
const score = Math.log(matches.length + 1) * weight;
|
|
91
|
+
if (score > 0) {
|
|
92
|
+
contentScore += score;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
// Negative pattern scoring
|
|
96
|
+
for (const { pattern, weight } of config.negativePatterns) {
|
|
97
|
+
const matches = content.match(pattern) || [];
|
|
98
|
+
const score = Math.log(matches.length + 1) * weight; // weight is negative
|
|
99
|
+
if (score < 0) {
|
|
100
|
+
negativeScore += score;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return {
|
|
104
|
+
class: config.name,
|
|
105
|
+
fileScore,
|
|
106
|
+
contentScore,
|
|
107
|
+
negativeScore,
|
|
108
|
+
total: fileScore + contentScore + negativeScore,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
exports.HeuristicContentClassifier = HeuristicContentClassifier;
|
|
113
|
+
//# sourceMappingURL=HeuristicContentClassifier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"HeuristicContentClassifier.js","sourceRoot":"","sources":["../../../../src/core/processor/classifier/HeuristicContentClassifier.ts"],"names":[],"mappings":";;;;;;;;;;;;AAOA,uDAAoD;AACpD,4CAAkD;AAgBlD;;;;;GAKG;AACU,QAAA,2BAA2B,GAAG,GAAG,CAAC;AAE/C;;;;GAIG;AACU,QAAA,+BAA+B,GAAG,IAAI,CAAC;AAEpD,MAAa,0BAA0B;IAGrC,YACU,MAAc,EACL,cAAsB,mCAA2B,EACjD,wBAAgC,uCAA+B;QAFxE,WAAM,GAAN,MAAM,CAAQ;QACL,gBAAW,GAAX,WAAW,CAAsC;QACjD,0BAAqB,GAArB,qBAAqB,CAA0C;QAL1E,0BAAqB,GAA6C,EAAE,CAAC;QAO3E,IAAI,CAAC,yBAAyB,EAAE,CAAC;IACnC,CAAC;IAEO,yBAAyB;QAC/B,2EAA2E;QAC3E,KAAK,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iCAAe,CAAC,EAAE,CAAC;YAClE,MAAM,gBAAgB,GAAqB,EAAE,CAAC;YAE9C,mDAAmD;YACnD,KAAK,MAAM,CAAC,cAAc,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,iCAAe,CAAC,EAAE,CAAC;gBAC5E,IAAI,cAAc,KAAK,SAAS,EAAE,CAAC;oBACjC,uEAAuE;oBACvE,KAAK,MAAM,OAAO,IAAI,WAAW,CAAC,eAAe,EAAE,CAAC;wBAClD,gBAAgB,CAAC,IAAI,CAAC;4BACpB,OAAO,EAAE,OAAO,CAAC,OAAO;4BACxB,MAAM,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,qBAAqB,EAAE,8BAA8B;yBACrF,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,mCAChC,MAAM,KACT,gBAAgB,GACjB,CAAC;QACJ,CAAC;IACH,CAAC;IAEK,QAAQ,CAAC,OAAe,EAAE,IAAY;;YAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACnE,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CACrC,CAAC;YAEF,+EAA+E;YAC/E,iFAAiF;YACjF,gFAAgF;YAChF,oFAAoF;YACpF,MAAM,WAAW,GAAG,IAAA,gBAAO,EACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EACvB,IAAI,CAAC,WAAW,CACjB,CAAC;YAEF,MAAM,OAAO,GAA2B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACzD,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC;aAC3B,CAAC,CAAC,CAAC;YAEJ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;gBACjB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,eAAe,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;oBAC5F,OAAO,CAAC,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACjG,CAAC;YACJ,CAAC;YAED,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAC7D,CAAC;KAAA;IAEO,QAAQ,CACd,OAAe,EACf,IAAY,EACZ,MAAgC;QAEhC,4EAA4E;QAC5E,yEAAyE;QACzE,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,uBAAuB;QACvB,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,MAAM,CAAC,YAAY,EAAE,CAAC;YACtD,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,SAAS,IAAI,MAAM,CAAC;YACtB,CAAC;QACH,CAAC;QAED,2BAA2B;QAC3B,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,MAAM,CAAC,eAAe,EAAE,CAAC;YACzD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC;YACpD,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,YAAY,IAAI,KAAK,CAAC;YACxB,CAAC;QACH,CAAC;QAED,2BAA2B;QAC3B,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;YAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,qBAAqB;YAC1E,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;gBACd,aAAa,IAAI,KAAK,CAAC;YACzB,CAAC;QACH,CAAC;QAED,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,IAAI;YAClB,SAAS;YACT,YAAY;YACZ,aAAa;YACb,KAAK,EAAE,SAAS,GAAG,YAAY,GAAG,aAAa;SAChD,CAAC;IACJ,CAAC;CACF;AA9GD,gEA8GC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IContentTypeClassifier.js","sourceRoot":"","sources":["../../../../src/core/processor/classifier/IContentTypeClassifier.ts"],"names":[],"mappings":""}
|