@wanshi-kg/wanshi 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +458 -0
- package/dist/__tests__/helpers.js +27 -0
- package/dist/__tests__/helpers.js.map +1 -0
- package/dist/cli/commands/export.command.js +99 -0
- package/dist/cli/commands/export.command.js.map +1 -0
- package/dist/cli/commands/index.js +22 -0
- package/dist/cli/commands/index.js.map +1 -0
- package/dist/cli/commands/inspectMerges.command.js +84 -0
- package/dist/cli/commands/inspectMerges.command.js.map +1 -0
- package/dist/cli/commands/metrics.command.js +196 -0
- package/dist/cli/commands/metrics.command.js.map +1 -0
- package/dist/cli/commands/process.command.js +82 -0
- package/dist/cli/commands/process.command.js.map +1 -0
- package/dist/cli/commands/watch.command.js +91 -0
- package/dist/cli/commands/watch.command.js.map +1 -0
- package/dist/cli/index.js +269 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/optionsToConfig.js +160 -0
- package/dist/cli/optionsToConfig.js.map +1 -0
- package/dist/config/index.js +59 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/legacyHints.js +113 -0
- package/dist/config/legacyHints.js.map +1 -0
- package/dist/config/schema.js +803 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/ui.js +221 -0
- package/dist/config/ui.js.map +1 -0
- package/dist/core/DirectoryProcessor.js +725 -0
- package/dist/core/DirectoryProcessor.js.map +1 -0
- package/dist/core/adapters/IStructuredAdapter.js +3 -0
- package/dist/core/adapters/IStructuredAdapter.js.map +1 -0
- package/dist/core/adapters/SqliteAdapter.js +267 -0
- package/dist/core/adapters/SqliteAdapter.js.map +1 -0
- package/dist/core/adapters/StructuredAdapterRegistry.js +31 -0
- package/dist/core/adapters/StructuredAdapterRegistry.js.map +1 -0
- package/dist/core/adapters/index.js +20 -0
- package/dist/core/adapters/index.js.map +1 -0
- package/dist/core/checkpoint/CheckpointService.js +188 -0
- package/dist/core/checkpoint/CheckpointService.js.map +1 -0
- package/dist/core/checkpoint/index.js +18 -0
- package/dist/core/checkpoint/index.js.map +1 -0
- package/dist/core/corpus/CorpusAnalyzer.js +266 -0
- package/dist/core/corpus/CorpusAnalyzer.js.map +1 -0
- package/dist/core/corpus/CorpusProfileStore.js +92 -0
- package/dist/core/corpus/CorpusProfileStore.js.map +1 -0
- package/dist/core/corpus/index.js +21 -0
- package/dist/core/corpus/index.js.map +1 -0
- package/dist/core/corpus/normalizeGlossary.js +60 -0
- package/dist/core/corpus/normalizeGlossary.js.map +1 -0
- package/dist/core/corpus/relPath.js +52 -0
- package/dist/core/corpus/relPath.js.map +1 -0
- package/dist/core/corpus/termFrequency.js +86 -0
- package/dist/core/corpus/termFrequency.js.map +1 -0
- package/dist/core/cost/CostMeter.js +235 -0
- package/dist/core/cost/CostMeter.js.map +1 -0
- package/dist/core/cost/index.js +19 -0
- package/dist/core/cost/index.js.map +1 -0
- package/dist/core/cost/prices.js +38 -0
- package/dist/core/cost/prices.js.map +1 -0
- package/dist/core/cv/ObjectDetectionService.js +119 -0
- package/dist/core/cv/ObjectDetectionService.js.map +1 -0
- package/dist/core/di/ContainerFactory.js +670 -0
- package/dist/core/di/ContainerFactory.js.map +1 -0
- package/dist/core/di/DIContainer.js +103 -0
- package/dist/core/di/DIContainer.js.map +1 -0
- package/dist/core/di/index.js +19 -0
- package/dist/core/di/index.js.map +1 -0
- package/dist/core/errors/CustomErrors.js +342 -0
- package/dist/core/errors/CustomErrors.js.map +1 -0
- package/dist/core/errors/index.js +18 -0
- package/dist/core/errors/index.js.map +1 -0
- package/dist/core/export/KnowledgeGraphExportService.js +56 -0
- package/dist/core/export/KnowledgeGraphExportService.js.map +1 -0
- package/dist/core/export/index.js +19 -0
- package/dist/core/export/index.js.map +1 -0
- package/dist/core/export/strategies/GraphitiExportStrategy.js +115 -0
- package/dist/core/export/strategies/GraphitiExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/GraphvizDotExportStrategy.js +331 -0
- package/dist/core/export/strategies/GraphvizDotExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/IExportStrategy.js +3 -0
- package/dist/core/export/strategies/IExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/JsonExportStrategy.js +19 -0
- package/dist/core/export/strategies/JsonExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/JsonlExportStrategy.js +69 -0
- package/dist/core/export/strategies/JsonlExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/KblamExportStrategy.js +36 -0
- package/dist/core/export/strategies/KblamExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/LoraExportStrategy.js +46 -0
- package/dist/core/export/strategies/LoraExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/McpExportStrategy.js +67 -0
- package/dist/core/export/strategies/McpExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/index.js +25 -0
- package/dist/core/export/strategies/index.js.map +1 -0
- package/dist/core/export/strategies/kbTriples.js +60 -0
- package/dist/core/export/strategies/kbTriples.js.map +1 -0
- package/dist/core/index.js +22 -0
- package/dist/core/index.js.map +1 -0
- package/dist/core/knowledge/KnowledgeGraphBuilder.js +627 -0
- package/dist/core/knowledge/KnowledgeGraphBuilder.js.map +1 -0
- package/dist/core/knowledge/MergeRecord.js +3 -0
- package/dist/core/knowledge/MergeRecord.js.map +1 -0
- package/dist/core/knowledge/canon/Canonicalizer.js +414 -0
- package/dist/core/knowledge/canon/Canonicalizer.js.map +1 -0
- package/dist/core/knowledge/canon/index.js +18 -0
- package/dist/core/knowledge/canon/index.js.map +1 -0
- package/dist/core/knowledge/contradiction/HeuristicContradictionChecker.js +92 -0
- package/dist/core/knowledge/contradiction/HeuristicContradictionChecker.js.map +1 -0
- package/dist/core/knowledge/contradiction/LlmContradictionChecker.js +52 -0
- package/dist/core/knowledge/contradiction/LlmContradictionChecker.js.map +1 -0
- package/dist/core/knowledge/contradiction/index.js +19 -0
- package/dist/core/knowledge/contradiction/index.js.map +1 -0
- package/dist/core/knowledge/grounding/KeywordGroundingChecker.js +33 -0
- package/dist/core/knowledge/grounding/KeywordGroundingChecker.js.map +1 -0
- package/dist/core/knowledge/grounding/MiniCheckGroundingChecker.js +82 -0
- package/dist/core/knowledge/grounding/MiniCheckGroundingChecker.js.map +1 -0
- package/dist/core/knowledge/grounding/index.js +20 -0
- package/dist/core/knowledge/grounding/index.js.map +1 -0
- package/dist/core/knowledge/grounding/verbalize.js +38 -0
- package/dist/core/knowledge/grounding/verbalize.js.map +1 -0
- package/dist/core/knowledge/images/imageMetaGraph.js +136 -0
- package/dist/core/knowledge/images/imageMetaGraph.js.map +1 -0
- package/dist/core/knowledge/index.js +20 -0
- package/dist/core/knowledge/index.js.map +1 -0
- package/dist/core/knowledge/merging/KnowledgeMerger.js +624 -0
- package/dist/core/knowledge/merging/KnowledgeMerger.js.map +1 -0
- package/dist/core/knowledge/references/ReferenceResolver.js +184 -0
- package/dist/core/knowledge/references/ReferenceResolver.js.map +1 -0
- package/dist/core/knowledge/references/citations/CitationEvidenceProcessor.js +401 -0
- package/dist/core/knowledge/references/citations/CitationEvidenceProcessor.js.map +1 -0
- package/dist/core/knowledge/references/citations/CitationResolver.js +95 -0
- package/dist/core/knowledge/references/citations/CitationResolver.js.map +1 -0
- package/dist/core/knowledge/references/citations/GrobidClient.js +143 -0
- package/dist/core/knowledge/references/citations/GrobidClient.js.map +1 -0
- package/dist/core/knowledge/references/citations/TitleIdResolver.js +101 -0
- package/dist/core/knowledge/references/citations/TitleIdResolver.js.map +1 -0
- package/dist/core/knowledge/references/web/FetchCacheService.js +114 -0
- package/dist/core/knowledge/references/web/FetchCacheService.js.map +1 -0
- package/dist/core/knowledge/references/web/GatedFetcher.js +228 -0
- package/dist/core/knowledge/references/web/GatedFetcher.js.map +1 -0
- package/dist/core/knowledge/references/web/WebReferenceProcessor.js +164 -0
- package/dist/core/knowledge/references/web/WebReferenceProcessor.js.map +1 -0
- package/dist/core/knowledge/search/KnowledgeGraphSearch.js +261 -0
- package/dist/core/knowledge/search/KnowledgeGraphSearch.js.map +1 -0
- package/dist/core/knowledge/vocabulary.js +162 -0
- package/dist/core/knowledge/vocabulary.js.map +1 -0
- package/dist/core/llm/EmbeddingService.js +113 -0
- package/dist/core/llm/EmbeddingService.js.map +1 -0
- package/dist/core/llm/OllamaService.js +146 -0
- package/dist/core/llm/OllamaService.js.map +1 -0
- package/dist/core/llm/OpenAICompatibleService.js +190 -0
- package/dist/core/llm/OpenAICompatibleService.js.map +1 -0
- package/dist/core/llm/OpenAIEmbeddingService.js +129 -0
- package/dist/core/llm/OpenAIEmbeddingService.js.map +1 -0
- package/dist/core/llm/embeddingUtils.js +25 -0
- package/dist/core/llm/embeddingUtils.js.map +1 -0
- package/dist/core/llm/index.js +23 -0
- package/dist/core/llm/index.js.map +1 -0
- package/dist/core/llm/prompts/PromptManager.js +388 -0
- package/dist/core/llm/prompts/PromptManager.js.map +1 -0
- package/dist/core/llm/prompts/PromptTemplateEngine.js +257 -0
- package/dist/core/llm/prompts/PromptTemplateEngine.js.map +1 -0
- package/dist/core/llm/prompts/templates/partials/examples/EXAMPLE_STYLE_GUIDE.md +84 -0
- package/dist/core/llm/prompts/templates/partials/examples/article.md +187 -0
- package/dist/core/llm/prompts/templates/partials/examples/code.md +229 -0
- package/dist/core/llm/prompts/templates/partials/examples/communication.md +205 -0
- package/dist/core/llm/prompts/templates/partials/examples/documentation.md +262 -0
- package/dist/core/llm/prompts/templates/partials/examples/financial.md +157 -0
- package/dist/core/llm/prompts/templates/partials/examples/legal.md +153 -0
- package/dist/core/llm/prompts/templates/partials/examples/logs.md +127 -0
- package/dist/core/llm/prompts/templates/partials/examples/medical.md +218 -0
- package/dist/core/llm/prompts/templates/partials/examples/notes.md +201 -0
- package/dist/core/llm/prompts/templates/partials/examples/research.md +208 -0
- package/dist/core/llm/prompts/templates/partials/examples/tabular.md +178 -0
- package/dist/core/llm/prompts/templates/partials/examples/transcript.md +204 -0
- package/dist/core/llm/prompts/templates/partials/retrieved-context.hbs +18 -0
- package/dist/core/llm/prompts/templates/v1/system.hbs +371 -0
- package/dist/core/llm/prompts/templates/v1/user.hbs +20 -0
- package/dist/core/llm/prompts/templates/v2/system.hbs +573 -0
- package/dist/core/llm/prompts/templates/v2/user.hbs +20 -0
- package/dist/core/llm/prompts/templates/v3/system.hbs +861 -0
- package/dist/core/llm/prompts/templates/v3/user.hbs +16 -0
- package/dist/core/llm/prompts/templates/v4/system.hbs +800 -0
- package/dist/core/llm/prompts/templates/v4/user.hbs +40 -0
- package/dist/core/llm/prompts/templates/v4.5/system.hbs +71 -0
- package/dist/core/llm/prompts/templates/v4.5/user.hbs +46 -0
- package/dist/core/llm/prompts/templates/v5/glossary/system.hbs +40 -0
- package/dist/core/llm/prompts/templates/v5/glossary/user.hbs +11 -0
- package/dist/core/llm/prompts/templates/v5/system.hbs +163 -0
- package/dist/core/llm/prompts/templates/v5/user.hbs +55 -0
- package/dist/core/pipeline/GroundingTransform.js +52 -0
- package/dist/core/pipeline/GroundingTransform.js.map +1 -0
- package/dist/core/pipeline/PipelineRunner.js +51 -0
- package/dist/core/pipeline/PipelineRunner.js.map +1 -0
- package/dist/core/pipeline/RelationFilterTransform.js +72 -0
- package/dist/core/pipeline/RelationFilterTransform.js.map +1 -0
- package/dist/core/pipeline/index.js +20 -0
- package/dist/core/pipeline/index.js.map +1 -0
- package/dist/core/processor/FileProcessor.js +184 -0
- package/dist/core/processor/FileProcessor.js.map +1 -0
- package/dist/core/processor/ProcessedRegistry.js +38 -0
- package/dist/core/processor/ProcessedRegistry.js.map +1 -0
- package/dist/core/processor/ast/AstSeedService.js +0 -0
- package/dist/core/processor/ast/AstSeedService.js.map +1 -0
- package/dist/core/processor/ast/AstSymbolStore.js +110 -0
- package/dist/core/processor/ast/AstSymbolStore.js.map +1 -0
- package/dist/core/processor/ast/index.js +19 -0
- package/dist/core/processor/ast/index.js.map +1 -0
- package/dist/core/processor/chunking/TextChunker.js +98 -0
- package/dist/core/processor/chunking/TextChunker.js.map +1 -0
- package/dist/core/processor/chunking/index.js +18 -0
- package/dist/core/processor/chunking/index.js.map +1 -0
- package/dist/core/processor/classifier/CONTENT_CLASSES.js +294 -0
- package/dist/core/processor/classifier/CONTENT_CLASSES.js.map +1 -0
- package/dist/core/processor/classifier/CascadeContentClassifier.js +107 -0
- package/dist/core/processor/classifier/CascadeContentClassifier.js.map +1 -0
- package/dist/core/processor/classifier/HeuristicContentClassifier.js +113 -0
- package/dist/core/processor/classifier/HeuristicContentClassifier.js.map +1 -0
- package/dist/core/processor/classifier/IContentTypeClassifier.js +3 -0
- package/dist/core/processor/classifier/IContentTypeClassifier.js.map +1 -0
- package/dist/core/processor/classifier/LlmContentClassifier.js +107 -0
- package/dist/core/processor/classifier/LlmContentClassifier.js.map +1 -0
- package/dist/core/processor/classifier/NER_DOMAIN_EXAMPLES.js +498 -0
- package/dist/core/processor/classifier/NER_DOMAIN_EXAMPLES.js.map +1 -0
- package/dist/core/processor/classifier/index.js +21 -0
- package/dist/core/processor/classifier/index.js.map +1 -0
- package/dist/core/processor/classifier/mergeClassifications.js +32 -0
- package/dist/core/processor/classifier/mergeClassifications.js.map +1 -0
- package/dist/core/processor/index.js +20 -0
- package/dist/core/processor/index.js.map +1 -0
- package/dist/core/processor/readers/AudioReader.js +462 -0
- package/dist/core/processor/readers/AudioReader.js.map +1 -0
- package/dist/core/processor/readers/BinaryReader.js +90 -0
- package/dist/core/processor/readers/BinaryReader.js.map +1 -0
- package/dist/core/processor/readers/ChandraPdfReader.js +187 -0
- package/dist/core/processor/readers/ChandraPdfReader.js.map +1 -0
- package/dist/core/processor/readers/ChatExportReader.js +365 -0
- package/dist/core/processor/readers/ChatExportReader.js.map +1 -0
- package/dist/core/processor/readers/DoclingReader.js +445 -0
- package/dist/core/processor/readers/DoclingReader.js.map +1 -0
- package/dist/core/processor/readers/EmailReader.js +259 -0
- package/dist/core/processor/readers/EmailReader.js.map +1 -0
- package/dist/core/processor/readers/EpubReader.js +175 -0
- package/dist/core/processor/readers/EpubReader.js.map +1 -0
- package/dist/core/processor/readers/FileReader.js +90 -0
- package/dist/core/processor/readers/FileReader.js.map +1 -0
- package/dist/core/processor/readers/FileReaderFactory.js +49 -0
- package/dist/core/processor/readers/FileReaderFactory.js.map +1 -0
- package/dist/core/processor/readers/HtmlReader.js +371 -0
- package/dist/core/processor/readers/HtmlReader.js.map +1 -0
- package/dist/core/processor/readers/ImageReader.js +162 -0
- package/dist/core/processor/readers/ImageReader.js.map +1 -0
- package/dist/core/processor/readers/JsonFileReader.js +232 -0
- package/dist/core/processor/readers/JsonFileReader.js.map +1 -0
- package/dist/core/processor/readers/JupyterReader.js +178 -0
- package/dist/core/processor/readers/JupyterReader.js.map +1 -0
- package/dist/core/processor/readers/LatexReader.js +176 -0
- package/dist/core/processor/readers/LatexReader.js.map +1 -0
- package/dist/core/processor/readers/MarkdownReader.js +289 -0
- package/dist/core/processor/readers/MarkdownReader.js.map +1 -0
- package/dist/core/processor/readers/MarkerPdfReader.js +193 -0
- package/dist/core/processor/readers/MarkerPdfReader.js.map +1 -0
- package/dist/core/processor/readers/MistralOcrReader.js +198 -0
- package/dist/core/processor/readers/MistralOcrReader.js.map +1 -0
- package/dist/core/processor/readers/OfficeReader.js +174 -0
- package/dist/core/processor/readers/OfficeReader.js.map +1 -0
- package/dist/core/processor/readers/PdfReader.js +116 -0
- package/dist/core/processor/readers/PdfReader.js.map +1 -0
- package/dist/core/processor/readers/RtfReader.js +107 -0
- package/dist/core/processor/readers/RtfReader.js.map +1 -0
- package/dist/core/processor/readers/SubtitleReader.js +145 -0
- package/dist/core/processor/readers/SubtitleReader.js.map +1 -0
- package/dist/core/processor/readers/TesseractPdfReader.js +183 -0
- package/dist/core/processor/readers/TesseractPdfReader.js.map +1 -0
- package/dist/core/processor/readers/TextReader.js +129 -0
- package/dist/core/processor/readers/TextReader.js.map +1 -0
- package/dist/core/processor/readers/TranscriptReader.js +234 -0
- package/dist/core/processor/readers/TranscriptReader.js.map +1 -0
- package/dist/core/processor/readers/image/imageMetadata.js +155 -0
- package/dist/core/processor/readers/image/imageMetadata.js.map +1 -0
- package/dist/core/processor/readers/index.js +41 -0
- package/dist/core/processor/readers/index.js.map +1 -0
- package/dist/core/processor/readers/referenceExtraction.js +198 -0
- package/dist/core/processor/readers/referenceExtraction.js.map +1 -0
- package/dist/core/processor/readers/stripReferences.js +59 -0
- package/dist/core/processor/readers/stripReferences.js.map +1 -0
- package/dist/core/processor/readers/transcript/turnPacking.js +81 -0
- package/dist/core/processor/readers/transcript/turnPacking.js.map +1 -0
- package/dist/core/progress/NdjsonProgressEmitter.js +30 -0
- package/dist/core/progress/NdjsonProgressEmitter.js.map +1 -0
- package/dist/core/progress/NoopProgressEmitter.js +15 -0
- package/dist/core/progress/NoopProgressEmitter.js.map +1 -0
- package/dist/core/progress/index.js +19 -0
- package/dist/core/progress/index.js.map +1 -0
- package/dist/core/trace/TraceWriter.js +100 -0
- package/dist/core/trace/TraceWriter.js.map +1 -0
- package/dist/core/trace/events.js +13 -0
- package/dist/core/trace/events.js.map +1 -0
- package/dist/core/trace/index.js +20 -0
- package/dist/core/trace/index.js.map +1 -0
- package/dist/core/trace/lineage.js +97 -0
- package/dist/core/trace/lineage.js.map +1 -0
- package/dist/evaluation/BenchmarkRunner.js +171 -0
- package/dist/evaluation/BenchmarkRunner.js.map +1 -0
- package/dist/evaluation/classifier/ClassifierAccuracy.js +185 -0
- package/dist/evaluation/classifier/ClassifierAccuracy.js.map +1 -0
- package/dist/evaluation/classifier/labeledSamples.js +379 -0
- package/dist/evaluation/classifier/labeledSamples.js.map +1 -0
- package/dist/evaluation/compare/goldCompare.js +126 -0
- package/dist/evaluation/compare/goldCompare.js.map +1 -0
- package/dist/evaluation/crossre/compareScoring.js +30 -0
- package/dist/evaluation/crossre/compareScoring.js.map +1 -0
- package/dist/evaluation/datasets/CrossREDataset.js +170 -0
- package/dist/evaluation/datasets/CrossREDataset.js.map +1 -0
- package/dist/evaluation/datasets/IDataset.js +3 -0
- package/dist/evaluation/datasets/IDataset.js.map +1 -0
- package/dist/evaluation/datasets/RebelDataset.js +117 -0
- package/dist/evaluation/datasets/RebelDataset.js.map +1 -0
- package/dist/evaluation/datasets/RedocredDataset.js +218 -0
- package/dist/evaluation/datasets/RedocredDataset.js.map +1 -0
- package/dist/evaluation/datasets/SemEval2010Dataset.js +150 -0
- package/dist/evaluation/datasets/SemEval2010Dataset.js.map +1 -0
- package/dist/evaluation/index.js +33 -0
- package/dist/evaluation/index.js.map +1 -0
- package/dist/evaluation/matching/ExactMatcher.js +75 -0
- package/dist/evaluation/matching/ExactMatcher.js.map +1 -0
- package/dist/evaluation/matching/SemanticMatcher.js +143 -0
- package/dist/evaluation/matching/SemanticMatcher.js.map +1 -0
- package/dist/evaluation/metrics/TripleMetrics.js +64 -0
- package/dist/evaluation/metrics/TripleMetrics.js.map +1 -0
- package/dist/evaluation/mine/MineCheckpoint.js +114 -0
- package/dist/evaluation/mine/MineCheckpoint.js.map +1 -0
- package/dist/evaluation/mine/MineDataset.js +208 -0
- package/dist/evaluation/mine/MineDataset.js.map +1 -0
- package/dist/evaluation/mine/MineReporter.js +98 -0
- package/dist/evaluation/mine/MineReporter.js.map +1 -0
- package/dist/evaluation/mine/MineRunner.js +148 -0
- package/dist/evaluation/mine/MineRunner.js.map +1 -0
- package/dist/evaluation/mine/MineScorer.js +127 -0
- package/dist/evaluation/mine/MineScorer.js.map +1 -0
- package/dist/evaluation/mine/types.js +12 -0
- package/dist/evaluation/mine/types.js.map +1 -0
- package/dist/evaluation/reporters/ConsoleReporter.js +55 -0
- package/dist/evaluation/reporters/ConsoleReporter.js.map +1 -0
- package/dist/evaluation/reporters/JsonReporter.js +50 -0
- package/dist/evaluation/reporters/JsonReporter.js.map +1 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/quality/CompositeScore.js +61 -0
- package/dist/quality/CompositeScore.js.map +1 -0
- package/dist/quality/ConsistencyMetrics.js +70 -0
- package/dist/quality/ConsistencyMetrics.js.map +1 -0
- package/dist/quality/FactualMetrics.js +76 -0
- package/dist/quality/FactualMetrics.js.map +1 -0
- package/dist/quality/GraphHealthMetrics.js +68 -0
- package/dist/quality/GraphHealthMetrics.js.map +1 -0
- package/dist/quality/SemanticMetrics.js +102 -0
- package/dist/quality/SemanticMetrics.js.map +1 -0
- package/dist/quality/StructuralMetrics.js +60 -0
- package/dist/quality/StructuralMetrics.js.map +1 -0
- package/dist/quality/index.js +23 -0
- package/dist/quality/index.js.map +1 -0
- package/dist/shared/index.js +20 -0
- package/dist/shared/index.js.map +1 -0
- package/dist/shared/logger/Logger.js +3 -0
- package/dist/shared/logger/Logger.js.map +1 -0
- package/dist/shared/logger/LoggerFactory.js +75 -0
- package/dist/shared/logger/LoggerFactory.js.map +1 -0
- package/dist/shared/logger/index.js +19 -0
- package/dist/shared/logger/index.js.map +1 -0
- package/dist/shared/shutdown.js +30 -0
- package/dist/shared/shutdown.js.map +1 -0
- package/dist/shared/utils/agglomerativeCluster.js +269 -0
- package/dist/shared/utils/agglomerativeCluster.js.map +1 -0
- package/dist/shared/utils/astSymbols.js +69 -0
- package/dist/shared/utils/astSymbols.js.map +1 -0
- package/dist/shared/utils/cosineSimilarity.js +18 -0
- package/dist/shared/utils/cosineSimilarity.js.map +1 -0
- package/dist/shared/utils/directoryTree.js +184 -0
- package/dist/shared/utils/directoryTree.js.map +1 -0
- package/dist/shared/utils/documentOutline.js +74 -0
- package/dist/shared/utils/documentOutline.js.map +1 -0
- package/dist/shared/utils/index.js +24 -0
- package/dist/shared/utils/index.js.map +1 -0
- package/dist/shared/utils/jaroWinklerSimilarity.js +60 -0
- package/dist/shared/utils/jaroWinklerSimilarity.js.map +1 -0
- package/dist/shared/utils/parseJsonLenient.js +27 -0
- package/dist/shared/utils/parseJsonLenient.js.map +1 -0
- package/dist/shared/utils/readConfig.js +42 -0
- package/dist/shared/utils/readConfig.js.map +1 -0
- package/dist/shared/utils/readRtf.js +216 -0
- package/dist/shared/utils/readRtf.js.map +1 -0
- package/dist/shared/utils/softmax.js +26 -0
- package/dist/shared/utils/softmax.js.map +1 -0
- package/dist/types/ContentClass.js +3 -0
- package/dist/types/ContentClass.js.map +1 -0
- package/dist/types/CorpusProfile.js +3 -0
- package/dist/types/CorpusProfile.js.map +1 -0
- package/dist/types/IContradictionChecker.js +3 -0
- package/dist/types/IContradictionChecker.js.map +1 -0
- package/dist/types/ICorpusAnalyzer.js +3 -0
- package/dist/types/ICorpusAnalyzer.js.map +1 -0
- package/dist/types/IDirectoryProcessor.js +3 -0
- package/dist/types/IDirectoryProcessor.js.map +1 -0
- package/dist/types/IEmbeddingProvider.js +3 -0
- package/dist/types/IEmbeddingProvider.js.map +1 -0
- package/dist/types/IEmbeddingService.js +6 -0
- package/dist/types/IEmbeddingService.js.map +1 -0
- package/dist/types/IFileProcessor.js +3 -0
- package/dist/types/IFileProcessor.js.map +1 -0
- package/dist/types/IGroundingChecker.js +3 -0
- package/dist/types/IGroundingChecker.js.map +1 -0
- package/dist/types/IKnowledgeGraphBuilder.js +3 -0
- package/dist/types/IKnowledgeGraphBuilder.js.map +1 -0
- package/dist/types/IKnowledgeGraphExporter.js +3 -0
- package/dist/types/IKnowledgeGraphExporter.js.map +1 -0
- package/dist/types/IKnowledgeGraphMerger.js +3 -0
- package/dist/types/IKnowledgeGraphMerger.js.map +1 -0
- package/dist/types/IKnowledgeGraphSearch.js +3 -0
- package/dist/types/IKnowledgeGraphSearch.js.map +1 -0
- package/dist/types/ILLMProvider.js +3 -0
- package/dist/types/ILLMProvider.js.map +1 -0
- package/dist/types/ILLMService.js +3 -0
- package/dist/types/ILLMService.js.map +1 -0
- package/dist/types/IObjectDetector.js +3 -0
- package/dist/types/IObjectDetector.js.map +1 -0
- package/dist/types/IProcessingService.js +3 -0
- package/dist/types/IProcessingService.js.map +1 -0
- package/dist/types/IProgressEmitter.js +3 -0
- package/dist/types/IProgressEmitter.js.map +1 -0
- package/dist/types/IPromptManager.js +3 -0
- package/dist/types/IPromptManager.js.map +1 -0
- package/dist/types/KnowledgeGraph.js +3 -0
- package/dist/types/KnowledgeGraph.js.map +1 -0
- package/dist/types/MCPKnowledgeGraph.js +3 -0
- package/dist/types/MCPKnowledgeGraph.js.map +1 -0
- package/dist/types/Observation.js +21 -0
- package/dist/types/Observation.js.map +1 -0
- package/dist/types/ProcessingOptions.js +3 -0
- package/dist/types/ProcessingOptions.js.map +1 -0
- package/dist/types/index.js +40 -0
- package/dist/types/index.js.map +1 -0
- package/package.json +122 -0
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
45
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
46
|
+
};
|
|
47
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
48
|
+
exports.HtmlReader = void 0;
|
|
49
|
+
const fs = __importStar(require("fs"));
|
|
50
|
+
const path = __importStar(require("path"));
|
|
51
|
+
const FileReader_1 = require("./FileReader");
|
|
52
|
+
const detect_file_encoding_and_language_1 = __importDefault(require("detect-file-encoding-and-language"));
|
|
53
|
+
const iconv_1 = require("iconv");
|
|
54
|
+
const referenceExtraction_1 = require("./referenceExtraction");
|
|
55
|
+
/**
|
|
56
|
+
* Features:
|
|
57
|
+
* - Smart content extraction (separates main content from navigation/ads)
|
|
58
|
+
* - Document outline generation with heading hierarchy
|
|
59
|
+
* - Metadata extraction (title, description, keywords, etc.)
|
|
60
|
+
* - Image extraction with alt text and captions
|
|
61
|
+
* - Link analysis and extraction
|
|
62
|
+
* - Text formatting preservation where appropriate
|
|
63
|
+
*
|
|
64
|
+
* Supported formats: .html, .htm, .xhtml
|
|
65
|
+
*/
|
|
66
|
+
class HtmlReader extends FileReader_1.FileReader {
|
|
67
|
+
constructor(chunker, logger, extractLinks = false) {
|
|
68
|
+
super(['.html', '.htm', '.xhtml', '.php'], chunker, logger);
|
|
69
|
+
this.extractLinks = extractLinks;
|
|
70
|
+
}
|
|
71
|
+
getName() {
|
|
72
|
+
return 'HtmlReader';
|
|
73
|
+
}
|
|
74
|
+
read(filePath) {
|
|
75
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
76
|
+
yield this.validateFile(filePath);
|
|
77
|
+
try {
|
|
78
|
+
this.logger.debug(`Extracting content from HTML file: ${filePath}`);
|
|
79
|
+
const startTime = Date.now();
|
|
80
|
+
const encoding = yield (0, detect_file_encoding_and_language_1.default)(filePath);
|
|
81
|
+
const conv = new iconv_1.Iconv(encoding.encoding || 'utf-8', 'utf-8');
|
|
82
|
+
const rawBuffer = yield fs.promises.readFile(filePath);
|
|
83
|
+
const rawHtml = conv.convert(rawBuffer).toString('utf-8');
|
|
84
|
+
const stats = yield fs.promises.stat(filePath);
|
|
85
|
+
// Parse with multiple approaches for comprehensive extraction
|
|
86
|
+
const cheerioResult = yield this.parseWithCheerio(rawHtml);
|
|
87
|
+
const htmlToTextResult = yield this.parseWithHtmlToText(rawHtml);
|
|
88
|
+
const content = htmlToTextResult.cleanText || htmlToTextResult.text;
|
|
89
|
+
// Extract images
|
|
90
|
+
const images = this.extractImages(cheerioResult.$ || null);
|
|
91
|
+
const processingTime = Date.now() - startTime;
|
|
92
|
+
// Build comprehensive metadata
|
|
93
|
+
const metadata = {
|
|
94
|
+
type: 'html_document',
|
|
95
|
+
fileName: path.basename(filePath),
|
|
96
|
+
filePath: filePath,
|
|
97
|
+
fileSize: stats.size,
|
|
98
|
+
createdAt: stats.birthtime.toISOString(),
|
|
99
|
+
modifiedAt: stats.mtime.toISOString(),
|
|
100
|
+
extension: path.extname(filePath).toLowerCase(),
|
|
101
|
+
encoding: 'utf-8',
|
|
102
|
+
// Document content analysis
|
|
103
|
+
contentLength: content.length,
|
|
104
|
+
htmlSize: rawHtml.length,
|
|
105
|
+
processingTimeMs: processingTime,
|
|
106
|
+
processorUsed: 'cheerio+unfluff+html-to-text',
|
|
107
|
+
// HTML metadata
|
|
108
|
+
title: cheerioResult.title || '',
|
|
109
|
+
description: cheerioResult.description || '',
|
|
110
|
+
keywords: cheerioResult.keywords || [],
|
|
111
|
+
author: cheerioResult.author || cheerioResult.author || '',
|
|
112
|
+
language: cheerioResult.language || 'en',
|
|
113
|
+
// Content structure
|
|
114
|
+
linkCount: cheerioResult.linkCount || 0,
|
|
115
|
+
imageCount: images.length,
|
|
116
|
+
paragraphCount: cheerioResult.paragraphCount || 0,
|
|
117
|
+
// Content quality indicators
|
|
118
|
+
wordCount: this.countWords(content),
|
|
119
|
+
readingTime: this.estimateReadingTime(content),
|
|
120
|
+
// Advanced metadata
|
|
121
|
+
socialMedia: {},
|
|
122
|
+
structuredData: cheerioResult.structuredData || [],
|
|
123
|
+
canonicalUrl: cheerioResult.canonicalUrl || '',
|
|
124
|
+
favicon: cheerioResult.favicon || '',
|
|
125
|
+
// Content classification
|
|
126
|
+
hasMainContent: !!htmlToTextResult.text && htmlToTextResult.text.length > 100,
|
|
127
|
+
contentType: this.classifyContent(cheerioResult.$),
|
|
128
|
+
status: 'success'
|
|
129
|
+
};
|
|
130
|
+
// Phase 0 reference extraction (network-free), gated by config.
|
|
131
|
+
if (this.extractLinks) {
|
|
132
|
+
const links = (0, referenceExtraction_1.extractHtmlLinks)(rawHtml);
|
|
133
|
+
if (links.length) {
|
|
134
|
+
const references = { links };
|
|
135
|
+
metadata.references = references;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
this.logger.debug(`Successfully processed HTML file ${filePath} in ${processingTime}ms`);
|
|
139
|
+
return {
|
|
140
|
+
chunks: [
|
|
141
|
+
{
|
|
142
|
+
content: content,
|
|
143
|
+
index: 1,
|
|
144
|
+
totalChunks: 1,
|
|
145
|
+
startOffset: 0,
|
|
146
|
+
endOffset: content.length,
|
|
147
|
+
images: images.length > 0 ? images : undefined,
|
|
148
|
+
},
|
|
149
|
+
],
|
|
150
|
+
metadata: metadata
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
catch (error) {
|
|
154
|
+
this.logger.error(`Failed to read HTML file ${filePath}: ${error.message}`);
|
|
155
|
+
return {
|
|
156
|
+
chunks: [
|
|
157
|
+
{
|
|
158
|
+
content: '',
|
|
159
|
+
index: 1,
|
|
160
|
+
totalChunks: 1,
|
|
161
|
+
startOffset: 0,
|
|
162
|
+
endOffset: 0,
|
|
163
|
+
},
|
|
164
|
+
],
|
|
165
|
+
metadata: {
|
|
166
|
+
type: 'html_document',
|
|
167
|
+
description: 'HTML Document',
|
|
168
|
+
fileName: path.basename(filePath),
|
|
169
|
+
filePath: filePath,
|
|
170
|
+
status: 'error',
|
|
171
|
+
error: error.message,
|
|
172
|
+
errorType: error.name,
|
|
173
|
+
processorUsed: 'html_reader_legacy'
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Parse HTML using Cheerio for fast DOM manipulation and metadata extraction
|
|
181
|
+
*/
|
|
182
|
+
parseWithCheerio(html) {
|
|
183
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
184
|
+
try {
|
|
185
|
+
const cheerio = yield Promise.resolve().then(() => __importStar(require('cheerio')));
|
|
186
|
+
const $ = cheerio.load(html);
|
|
187
|
+
// Extract basic metadata
|
|
188
|
+
const title = $('title').first().text().trim() ||
|
|
189
|
+
$('meta[property="og:title"]').attr('content') ||
|
|
190
|
+
$('h1').first().text().trim();
|
|
191
|
+
const description = $('meta[name="description"]').attr('content') ||
|
|
192
|
+
$('meta[property="og:description"]').attr('content') ||
|
|
193
|
+
$('meta[name="twitter:description"]').attr('content') || '';
|
|
194
|
+
const keywords = ($('meta[name="keywords"]').attr('content') || '')
|
|
195
|
+
.split(',')
|
|
196
|
+
.map(k => k.trim())
|
|
197
|
+
.filter(k => k.length > 0);
|
|
198
|
+
const author = $('meta[name="author"]').attr('content') ||
|
|
199
|
+
$('meta[property="article:author"]').attr('content') || '';
|
|
200
|
+
const language = $('html').attr('lang') ||
|
|
201
|
+
$('meta[http-equiv="content-language"]').attr('content') || 'en';
|
|
202
|
+
// Content analysis
|
|
203
|
+
const linkCount = $('a[href]').length;
|
|
204
|
+
const paragraphCount = $('p').length;
|
|
205
|
+
const imageCount = $('img').length;
|
|
206
|
+
// Extract structured data
|
|
207
|
+
const structuredData = [];
|
|
208
|
+
$('script[type="application/ld+json"]').each((_, elem) => {
|
|
209
|
+
try {
|
|
210
|
+
const jsonData = JSON.parse($(elem).html() || '{}');
|
|
211
|
+
structuredData.push(jsonData);
|
|
212
|
+
}
|
|
213
|
+
catch (e) {
|
|
214
|
+
// Ignore malformed JSON-LD
|
|
215
|
+
}
|
|
216
|
+
});
|
|
217
|
+
// Social media metadata
|
|
218
|
+
const socialMedia = {
|
|
219
|
+
ogTitle: $('meta[property="og:title"]').attr('content') || '',
|
|
220
|
+
ogDescription: $('meta[property="og:description"]').attr('content') || '',
|
|
221
|
+
ogImage: $('meta[property="og:image"]').attr('content') || '',
|
|
222
|
+
ogUrl: $('meta[property="og:url"]').attr('content') || '',
|
|
223
|
+
twitterCard: $('meta[name="twitter:card"]').attr('content') || '',
|
|
224
|
+
twitterTitle: $('meta[name="twitter:title"]').attr('content') || '',
|
|
225
|
+
twitterDescription: $('meta[name="twitter:description"]').attr('content') || '',
|
|
226
|
+
twitterImage: $('meta[name="twitter:image"]').attr('content') || ''
|
|
227
|
+
};
|
|
228
|
+
const canonicalUrl = $('link[rel="canonical"]').attr('href') || '';
|
|
229
|
+
const favicon = $('link[rel="icon"]').attr('href') ||
|
|
230
|
+
$('link[rel="shortcut icon"]').attr('href') || '';
|
|
231
|
+
return {
|
|
232
|
+
$,
|
|
233
|
+
title,
|
|
234
|
+
description,
|
|
235
|
+
keywords,
|
|
236
|
+
author,
|
|
237
|
+
language,
|
|
238
|
+
linkCount,
|
|
239
|
+
paragraphCount,
|
|
240
|
+
imageCount,
|
|
241
|
+
structuredData,
|
|
242
|
+
socialMedia,
|
|
243
|
+
canonicalUrl,
|
|
244
|
+
favicon
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
catch (error) {
|
|
248
|
+
this.logger.warn(`Cheerio parsing failed: ${error.message}`);
|
|
249
|
+
return {
|
|
250
|
+
title: '',
|
|
251
|
+
description: '',
|
|
252
|
+
keywords: [],
|
|
253
|
+
author: '',
|
|
254
|
+
language: 'en',
|
|
255
|
+
linkCount: 0,
|
|
256
|
+
paragraphCount: 0,
|
|
257
|
+
imageCount: 0,
|
|
258
|
+
structuredData: [],
|
|
259
|
+
socialMedia: {},
|
|
260
|
+
canonicalUrl: '',
|
|
261
|
+
favicon: ''
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Parse HTML using html-to-text for clean text extraction
|
|
268
|
+
*/
|
|
269
|
+
parseWithHtmlToText(html) {
|
|
270
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
271
|
+
try {
|
|
272
|
+
const { convert } = yield Promise.resolve().then(() => __importStar(require('html-to-text')));
|
|
273
|
+
const options = {
|
|
274
|
+
wordwrap: false,
|
|
275
|
+
selectors: [
|
|
276
|
+
// Preserve important structure
|
|
277
|
+
{ selector: 'h1', format: 'heading', options: { leadingLineBreaks: 2, trailingLineBreaks: 1 } },
|
|
278
|
+
{ selector: 'h2', format: 'heading', options: { leadingLineBreaks: 2, trailingLineBreaks: 1 } },
|
|
279
|
+
{ selector: 'h3', format: 'heading', options: { leadingLineBreaks: 2, trailingLineBreaks: 1 } },
|
|
280
|
+
{ selector: 'h4', format: 'heading', options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
|
|
281
|
+
{ selector: 'h5', format: 'heading', options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
|
|
282
|
+
{ selector: 'h6', format: 'heading', options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
|
|
283
|
+
{ selector: 'p', format: 'paragraph', options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
|
|
284
|
+
{ selector: 'br', format: 'lineBreak' },
|
|
285
|
+
{ selector: 'ul', format: 'unorderedList', options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
|
|
286
|
+
{ selector: 'ol', format: 'orderedList', options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
|
|
287
|
+
{ selector: 'li', format: 'listItem', options: { leadingLineBreaks: 1 } },
|
|
288
|
+
{ selector: 'blockquote', format: 'blockquote', options: { leadingLineBreaks: 1, trailingLineBreaks: 1 } },
|
|
289
|
+
{ selector: 'table', format: 'table' },
|
|
290
|
+
// Remove unwanted elements
|
|
291
|
+
{ selector: 'script', format: 'skip' },
|
|
292
|
+
{ selector: 'style', format: 'skip' },
|
|
293
|
+
{ selector: 'nav', format: 'skip' },
|
|
294
|
+
{ selector: '.navigation', format: 'skip' },
|
|
295
|
+
{ selector: '.sidebar', format: 'skip' },
|
|
296
|
+
{ selector: '.footer', format: 'skip' },
|
|
297
|
+
{ selector: '.header', format: 'skip' },
|
|
298
|
+
{ selector: '.menu', format: 'skip' },
|
|
299
|
+
{ selector: '.advertisement', format: 'skip' },
|
|
300
|
+
{ selector: '.ads', format: 'skip' }
|
|
301
|
+
]
|
|
302
|
+
};
|
|
303
|
+
const text = convert(html, options);
|
|
304
|
+
return {
|
|
305
|
+
text: text.trim(),
|
|
306
|
+
cleanText: this.cleanText(text)
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
catch (error) {
|
|
310
|
+
this.logger.warn(`html-to-text parsing failed: ${error.message}`);
|
|
311
|
+
return {
|
|
312
|
+
text: '',
|
|
313
|
+
cleanText: ''
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Extract images with metadata
|
|
320
|
+
*/
|
|
321
|
+
extractImages($) {
|
|
322
|
+
// For this implementation, we're not extracting actual image data
|
|
323
|
+
// In a real scenario, you'd fetch the images and convert to buffers
|
|
324
|
+
// This is a placeholder that shows how to extract image metadata
|
|
325
|
+
return [];
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Count words in text
|
|
329
|
+
*/
|
|
330
|
+
countWords(text) {
|
|
331
|
+
return text.split(/\s+/).filter(word => word.length > 0).length;
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Estimate reading time in minutes
|
|
335
|
+
*/
|
|
336
|
+
estimateReadingTime(text) {
|
|
337
|
+
const wordsPerMinute = 200; // Average reading speed
|
|
338
|
+
const wordCount = this.countWords(text);
|
|
339
|
+
return Math.ceil(wordCount / wordsPerMinute);
|
|
340
|
+
}
|
|
341
|
+
/**
|
|
342
|
+
* Classify content type
|
|
343
|
+
*/
|
|
344
|
+
classifyContent($) {
|
|
345
|
+
if (!$)
|
|
346
|
+
return 'unknown';
|
|
347
|
+
if ($('article').length > 0)
|
|
348
|
+
return 'article';
|
|
349
|
+
if ($('nav').length > 2 || $('.navigation').length > 0)
|
|
350
|
+
return 'navigation';
|
|
351
|
+
if ($('form').length > 2)
|
|
352
|
+
return 'form';
|
|
353
|
+
if ($('table').length > 1)
|
|
354
|
+
return 'data';
|
|
355
|
+
if ($('img').length > 5)
|
|
356
|
+
return 'gallery';
|
|
357
|
+
return 'webpage';
|
|
358
|
+
}
|
|
359
|
+
/**
|
|
360
|
+
* Clean extracted text
|
|
361
|
+
*/
|
|
362
|
+
cleanText(text) {
|
|
363
|
+
return text
|
|
364
|
+
.replace(/\n{3,}/g, '\n\n') // Remove excessive line breaks
|
|
365
|
+
.replace(/\s+/g, ' ') // Normalize whitespace
|
|
366
|
+
.replace(/^\s+|\s+$/g, '') // Trim
|
|
367
|
+
.replace(/\t/g, ' '); // Replace tabs with spaces
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
exports.HtmlReader = HtmlReader;
|
|
371
|
+
//# sourceMappingURL=HtmlReader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"HtmlReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/HtmlReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,6CAAuE;AAIvE,0GAAiE;AACjE,iCAA8B;AAC9B,+DAAwE;AAExE;;;;;;;;;;GAUG;AACH,MAAa,UAAW,SAAQ,uBAAU;IACxC,YACE,OAAoB,EACpB,MAAc,EACG,eAAwB,KAAK;QAE9C,KAAK,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QAF3C,iBAAY,GAAZ,YAAY,CAAiB;IAGhD,CAAC;IAED,OAAO;QACL,OAAO,YAAY,CAAC;IACtB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAElC,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;gBAEpE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,MAAM,QAAQ,GAAG,MAAM,IAAA,2CAAgB,EAAC,QAAQ,CAAC,CAAC;gBAClD,MAAM,IAAI,GAAG,IAAI,aAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,OAAO,EAAE,OAAO,CAAC,CAAC;gBAC9D,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACvD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAC1D,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE/C,8DAA8D;gBAC9D,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;gBAC3D,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;gBAEjE,MAAM,OAAO,GAAG,gBAAgB,CAAC,SAAS,IAAI,gBAAgB,CAAC,IAAI,CAAC;gBAEpE,iBAAiB;gBACjB,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;gBAE3D,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;gBAE9C,+BAA+B;gBAC/B,MAAM,QAAQ,GAAG;oBACf,IAAI,EAAE,eAAe;oBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBACjC,QAAQ,EAAE,QAAQ;oBAClB,QAAQ,EAAE,KAAK,CAAC,IAAI;oBACpB,SAAS,EAAE,KAAK,CAAC,SAAS,CAAC,WAAW,EAAE;oBACxC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE;oBACrC,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE;oBAC/C,QAAQ,EAAE,OAAO;oBAEjB,4BAA4B;oBAC5B,aAAa,EAAE,OAAO,CAAC,MAAM;oBAC7B,QAAQ,EAAE,OAAO,CAAC,MAAM;oBACxB,gBAAgB,EAAE,cAAc;oBAChC,aAAa,EAAE,8BAA8B;oBAE7C,gBAAgB;oBAChB,KAAK,EAAE,aAAa,CAAC,KAAK,IAAI,EAAE;oBAChC,WAAW,EAAE,aAAa,CAAC,WAAW,IAAI,EAAE;oBAC5C,QAAQ,EAAE,aAAa,CAAC,QAAQ,IAAI,EAAE;oBACtC,MAAM,EAAE,aAAa,CAAC,MAAM,IAAI,aAAa,CAAC,MAAM,IAAI,EAAE;oBAC1D,QAAQ,EAAE,aAAa,CAAC,QAAQ,IAAI,IAAI;oBAExC,oBAAoB;oBACpB,SAAS,EAAE,aAAa,CAAC,SAAS,IAAI,CAAC;oBACvC,UAAU,EAAE,MAAM,CAAC,MAAM;oBACzB,cAAc,EAAE,aAAa,CAAC,cAAc,IAAI,CAAC;oBAEjD,6BAA6B;oBAC7B,SAAS,EAAE,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;oBACnC,WAAW,EAAE,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC;oBAE9C,oBAAoB;oBACpB,WAAW,EAAE,EAAE;oBACf,cAAc,EAAE,aAAa,CAAC,cAAc,IAAI,EAAE;oBAClD,YAAY,EAAE,aAAa,CAAC,YAAY,IAAI,EAAE;oBAC9C,OAAO,EAAE,aAAa,CAAC,OAAO,IAAI,EAAE;oBAEpC,yBAAyB;oBACzB,cAAc,EAAE,CAAC,CAAC,gBAAgB,CAAC,IAAI,IAAI,gBAAgB,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG;oBAC7E,WAAW,EAAE,IAAI,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC;oBAElD,MAAM,EAAE,SAAS;iBAClB,CAAC;gBAEF,gEAAgE;gBAChE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBACtB,MAAM,KAAK,GAAG,IAAA,sCAAgB,EAAC,OAAO,CAAC,CAAC;oBACxC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;wBACjB,MAAM,UAAU,GAAkB,EAAE,KAAK,EAAE,CAAC;wBAC3C,QAAgC,CAAC,UAAU,GAAG,UAAU,CAAC;oBAC5D,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,oCAAoC,QAAQ,OAAO,cAAc,IAAI,CAAC,CAAC;gBAEzF,OAAO;oBACL,MAAM,EAAE;wBACN;4BACE,OAAO,EAAE,OAAO;4BAChB,KAAK,EAAE,CAAC;4BACR,WAAW,EAAE,CAAC;4BACd,WAAW,EAAE,CAAC;4BACd,SAAS,EAAE,OAAO,CAAC,MAAM;4BACzB,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;yBAC/C;qBACF;oBACD,QAAQ,EAAE,QAAQ;iBACnB,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,4BAA4B,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAE5E,OAAO;oBACL,MAAM,EAAE;wBACN;4BACE,OAAO,EAAE,EAAE;4BACX,KAAK,EAAE,CAAC;4BACR,WAAW,EAAE,CAAC;4BACd,WAAW,EAAE,CAAC;4BACd,SAAS,EAAE,CAAC;yBACb;qBACF;oBACD,QAAQ,EAAE;wBACR,IAAI,EAAE,eAAe;wBACrB,WAAW,EAAE,eAAe;wBAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;wBACjC,QAAQ,EAAE,QAAQ;wBAClB,MAAM,EAAE,OAAO;wBACf,KAAK,EAAE,KAAK,CAAC,OAAO;wBACpB,SAAS,EAAE,KAAK,CAAC,IAAI;wBACrB,aAAa,EAAE,oBAAoB;qBACpC;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;IAED;;OAEG;IACW,gBAAgB,CAAC,IAAY;;YACzC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,wDAAa,SAAS,GAAC,CAAC;gBACxC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAE7B,yBAAyB;gBACzB,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;oBACjC,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;oBAC9C,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBAE3C,MAAM,WAAW,GAAG,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;oBAC9C,CAAC,CAAC,iCAAiC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;oBACpD,CAAC,CAAC,kCAAkC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBAE/E,MAAM,QAAQ,GAAG,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;qBAChE,KAAK,CAAC,GAAG,CAAC;qBACV,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;qBAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAE7B,MAAM,MAAM,GAAG,CAAC,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;oBACzC,CAAC,CAAC,iCAAiC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;gBAEzE,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;oBACvB,CAAC,CAAC,qCAAqC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC;gBAEjF,mBAAmB;gBACnB,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;gBACtC,MAAM,cAAc,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;gBACrC,MAAM,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;gBAEnC,0BAA0B;gBAC1B,MAAM,cAAc,GAAU,EAAE,CAAC;gBACjC,CAAC,CAAC,oCAAoC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;oBACvD,IAAI,CAAC;wBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,IAAI,CAAC,CAAC;wBACpD,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAChC,CAAC;oBAAC,OAAO,CAAC,EAAE,CAAC;wBACX,2BAA2B;oBAC7B,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,wBAAwB;gBACxB,MAAM,WAAW,GAAG;oBAClB,OAAO,EAAE,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;oBAC7D,aAAa,EAAE,CAAC,CAAC,iCAAiC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;oBACzE,OAAO,EAAE,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;oBAC7D,KAAK,EAAE,CAAC,CAAC,yBAAyB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;oBACzD,WAAW,EAAE,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;oBACjE,YAAY,EAAE,CAAC,CAAC,4BAA4B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;oBACnE,kBAAkB,EAAE,CAAC,CAAC,kCAAkC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;oBAC/E,YAAY,EAAE,CAAC,CAAC,4BAA4B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE;iBACpE,CAAC;gBAEF,MAAM,YAAY,GAAG,CAAC,CAAC,uBAAuB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACnE,MAAM,OAAO,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;oBACnC,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBAEjE,OAAO;oBACL,CAAC;oBACD,KAAK;oBACL,WAAW;oBACX,QAAQ;oBACR,MAAM;oBACN,QAAQ;oBACR,SAAS;oBACT,cAAc;oBACd,UAAU;oBACV,cAAc;oBACd,WAAW;oBACX,YAAY;oBACZ,OAAO;iBACR,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC7D,OAAO;oBACL,KAAK,EAAE,EAAE;oBACT,WAAW,EAAE,EAAE;oBACf,QAAQ,EAAE,EAAE;oBACZ,MAAM,EAAE,EAAE;oBACV,QAAQ,EAAE,IAAI;oBACd,SAAS,EAAE,CAAC;oBACZ,cAAc,EAAE,CAAC;oBACjB,UAAU,EAAE,CAAC;oBACb,cAAc,EAAE,EAAE;oBAClB,WAAW,EAAE,EAAE;oBACf,YAAY,EAAE,EAAE;oBAChB,OAAO,EAAE,EAAE;iBACZ,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;IAED;;OAEG;IACW,mBAAmB,CAAC,IAAY;;YAC5C,IAAI,CAAC;gBACH,MAAM,EAAE,OAAO,EAAE,GAAG,wDAAa,cAAc,GAAC,CAAC;gBAEjD,MAAM,OAAO,GAAsB;oBACjC,QAAQ,EAAE,KAAK;oBACf,SAAS,EAAE;wBACT,+BAA+B;wBAC/B,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBAC/F,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBAC/F,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBAC/F,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBAC/F,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBAC/F,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBAC/F,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBAChG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE;wBACvC,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBACrG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBACnG,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,EAAE;wBACzE,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,EAAE,iBAAiB,EAAE,CAAC,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE;wBAC1G,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE;wBACtC,2BAA2B;wBAC3B,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE;wBACtC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;wBACrC,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE;wBACnC,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE;wBAC3C,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE;wBACxC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE;wBACvC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE;wBACvC,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;wBACrC,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,EAAE;wBAC9C,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;qBACrC;iBACF,CAAC;gBAEF,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;gBAEpC,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;oBACjB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;iBAChC,CAAC;YAEJ,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBACpB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gCAAgC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;gBAClE,OAAO;oBACL,IAAI,EAAE,EAAE;oBACR,SAAS,EAAE,EAAE;iBACd,CAAC;YACJ,CAAC;QACH,CAAC;KAAA;IAED;;OAEG;IACK,aAAa,CAAC,CAAM;QAC1B,kEAAkE;QAClE,oEAAoE;QACpE,iEAAiE;QACjE,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,IAAY;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAClE,CAAC;IAED;;OAEG;IACK,mBAAmB,CAAC,IAAY;QACtC,MAAM,cAAc,GAAG,GAAG,CAAC,CAAC,wBAAwB;QACpD,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACxC,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC,CAAC;IAC/C,CAAC;IAED;;OAEG;IACK,eAAe,CAAC,CAAM;QAC5B,IAAI,CAAC,CAAC;YAAE,OAAO,SAAS,CAAC;QAEzB,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,SAAS,CAAC;QAC9C,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,aAAa,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,YAAY,CAAC;QAC5E,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,MAAM,CAAC;QACxC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,MAAM,CAAC;QACzC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,SAAS,CAAC;QAE1C,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,IAAY;QAC5B,OAAO,IAAI;aACR,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,+BAA+B;aAC1D,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,uBAAuB;aAC5C,OAAO,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,OAAO;aACjC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,2BAA2B;IACrD,CAAC;CACF;AA/UD,gCA+UC"}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
36
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
37
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
38
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
39
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
40
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
41
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
|
+
exports.ImageReader = void 0;
|
|
46
|
+
const fs = __importStar(require("fs"));
|
|
47
|
+
const path = __importStar(require("path"));
|
|
48
|
+
const FileReader_1 = require("./FileReader");
|
|
49
|
+
const imageMetadata_1 = require("./image/imageMetadata");
|
|
50
|
+
/**
|
|
51
|
+
* Reader for image files. Returns a placeholder text + the image buffer for the
|
|
52
|
+
* multimodal (VLM) path, and — when enabled — stashes deterministic metadata
|
|
53
|
+
* (`metadata.exif`/`.c2pa`) + an opt-in CV object-detection signal
|
|
54
|
+
* (`metadata.cvDetection`) that `buildImageMetaGraph` turns into graph facts
|
|
55
|
+
* AUGMENTING (not replacing) the VLM read. The detection summary is also appended
|
|
56
|
+
* to the chunk content so the vision model sees it as context. All enrichment is
|
|
57
|
+
* additive and independent of the VLM mode; off by default ⇒ byte-identical run.
|
|
58
|
+
*/
|
|
59
|
+
class ImageReader extends FileReader_1.FileReader {
|
|
60
|
+
constructor(chunker, logger, opts = { exif: false, c2pa: { enabled: false, command: "c2patool" } }, detector) {
|
|
61
|
+
super([
|
|
62
|
+
".jpg",
|
|
63
|
+
".jpeg",
|
|
64
|
+
".jpe",
|
|
65
|
+
".jif",
|
|
66
|
+
".jfif",
|
|
67
|
+
".jfi",
|
|
68
|
+
".png",
|
|
69
|
+
".gif",
|
|
70
|
+
".webp",
|
|
71
|
+
".tiff",
|
|
72
|
+
".tif",
|
|
73
|
+
".bmp",
|
|
74
|
+
".dib",
|
|
75
|
+
".svg",
|
|
76
|
+
".svgz",
|
|
77
|
+
".ico",
|
|
78
|
+
".cur",
|
|
79
|
+
".pbm",
|
|
80
|
+
".pgm",
|
|
81
|
+
".ppm",
|
|
82
|
+
".pnm",
|
|
83
|
+
".heif",
|
|
84
|
+
".heic",
|
|
85
|
+
".avif",
|
|
86
|
+
".apng",
|
|
87
|
+
], chunker, logger);
|
|
88
|
+
this.opts = opts;
|
|
89
|
+
this.detector = detector;
|
|
90
|
+
}
|
|
91
|
+
getName() {
|
|
92
|
+
return "ImageReader";
|
|
93
|
+
}
|
|
94
|
+
read(filePath) {
|
|
95
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
96
|
+
yield this.validateFile(filePath);
|
|
97
|
+
try {
|
|
98
|
+
this.logger.debug(`Reading image file: ${filePath}`);
|
|
99
|
+
const imageBuffer = yield fs.promises.readFile(filePath);
|
|
100
|
+
const fileName = path.basename(filePath);
|
|
101
|
+
const stats = yield fs.promises.stat(filePath);
|
|
102
|
+
const metadata = {
|
|
103
|
+
type: "image",
|
|
104
|
+
fileName,
|
|
105
|
+
size: stats.size,
|
|
106
|
+
extension: path.extname(filePath).toLowerCase(),
|
|
107
|
+
};
|
|
108
|
+
// Deterministic enrichment (opt-in): stash structured metadata for the
|
|
109
|
+
// graph fragment. Best-effort — a failed/empty read leaves metadata absent.
|
|
110
|
+
if (this.opts.exif) {
|
|
111
|
+
const exif = yield (0, imageMetadata_1.readExif)(filePath, this.logger);
|
|
112
|
+
if (exif)
|
|
113
|
+
metadata.exif = exif;
|
|
114
|
+
}
|
|
115
|
+
if (this.opts.c2pa.enabled) {
|
|
116
|
+
metadata.c2pa = yield (0, imageMetadata_1.readC2pa)(filePath, this.opts.c2pa.command, this.logger);
|
|
117
|
+
}
|
|
118
|
+
// CV pre-pass (opt-in): object detections feed BOTH the graph fragment
|
|
119
|
+
// (metadata.cvDetection) AND the VLM as context (appended to the chunk
|
|
120
|
+
// content, which reaches the vision model verbatim). Detector never throws.
|
|
121
|
+
let content = `[Image file: ${fileName}]`;
|
|
122
|
+
if (this.detector) {
|
|
123
|
+
const objects = yield this.detector.detect(filePath);
|
|
124
|
+
if (objects.length) {
|
|
125
|
+
metadata.cvDetection = { objects };
|
|
126
|
+
content += `\nCV pre-pass detected: ${this.summarize(objects)}`;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return {
|
|
130
|
+
chunks: [
|
|
131
|
+
{
|
|
132
|
+
content,
|
|
133
|
+
index: 1,
|
|
134
|
+
totalChunks: 1,
|
|
135
|
+
startOffset: 0,
|
|
136
|
+
endOffset: imageBuffer.length,
|
|
137
|
+
images: [{ path: fileName, buffer: imageBuffer }],
|
|
138
|
+
},
|
|
139
|
+
],
|
|
140
|
+
metadata,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
catch (error) {
|
|
144
|
+
this.logger.error(`Failed to read image file ${filePath}: ${error}`);
|
|
145
|
+
throw new Error(`Failed to read image file: ${error}`);
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
/** "person ×3, motorbike ×2" — counts per label, most frequent first. */
|
|
150
|
+
summarize(objects) {
|
|
151
|
+
var _a;
|
|
152
|
+
const counts = new Map();
|
|
153
|
+
for (const o of objects)
|
|
154
|
+
counts.set(o.label, ((_a = counts.get(o.label)) !== null && _a !== void 0 ? _a : 0) + 1);
|
|
155
|
+
return [...counts.entries()]
|
|
156
|
+
.sort((a, b) => b[1] - a[1])
|
|
157
|
+
.map(([label, n]) => (n > 1 ? `${label} ×${n}` : label))
|
|
158
|
+
.join(", ");
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
exports.ImageReader = ImageReader;
|
|
162
|
+
//# sourceMappingURL=ImageReader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ImageReader.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/ImageReader.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,6CAA0D;AAG1D,yDAA2D;AAW3D;;;;;;;;GAQG;AACH,MAAa,WAAY,SAAQ,uBAAU;IACzC,YACE,OAAoB,EACpB,MAAc,EACG,OAA2B,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EACzF,QAA0B;QAE3C,KAAK,CACH;YACE,MAAM;YACN,OAAO;YACP,MAAM;YACN,MAAM;YACN,OAAO;YACP,MAAM;YACN,MAAM;YACN,MAAM;YACN,OAAO;YACP,OAAO;YACP,MAAM;YACN,MAAM;YACN,MAAM;YACN,MAAM;YACN,OAAO;YACP,MAAM;YACN,MAAM;YACN,MAAM;YACN,MAAM;YACN,MAAM;YACN,MAAM;YACN,OAAO;YACP,OAAO;YACP,OAAO;YACP,OAAO;SACR,EACD,OAAO,EACP,MAAM,CACP,CAAC;QAjCe,SAAI,GAAJ,IAAI,CAAqF;QACzF,aAAQ,GAAR,QAAQ,CAAkB;IAiC7C,CAAC;IAED,OAAO;QACL,OAAO,aAAa,CAAC;IACvB,CAAC;IAEK,IAAI,CAAC,QAAgB;;YACzB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;YAElC,IAAI,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,QAAQ,EAAE,CAAC,CAAC;gBACrD,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACzC,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAE/C,MAAM,QAAQ,GAAwB;oBACpC,IAAI,EAAE,OAAO;oBACb,QAAQ;oBACR,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE;iBAChD,CAAC;gBACF,uEAAuE;gBACvE,4EAA4E;gBAC5E,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACnB,MAAM,IAAI,GAAG,MAAM,IAAA,wBAAQ,EAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;oBACnD,IAAI,IAAI;wBAAE,QAAQ,CAAC,IAAI,GAAG,IAAI,CAAC;gBACjC,CAAC;gBACD,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;oBAC3B,QAAQ,CAAC,IAAI,GAAG,MAAM,IAAA,wBAAQ,EAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBAChF,CAAC;gBAED,uEAAuE;gBACvE,uEAAuE;gBACvE,4EAA4E;gBAC5E,IAAI,OAAO,GAAG,gBAAgB,QAAQ,GAAG,CAAC;gBAC1C,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;oBACrD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;wBACnB,QAAQ,CAAC,WAAW,GAAG,EAAE,OAAO,EAAE,CAAC;wBACnC,OAAO,IAAI,2BAA2B,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;oBAClE,CAAC;gBACH,CAAC;gBAED,OAAO;oBACL,MAAM,EAAE;wBACN;4BACE,OAAO;4BACP,KAAK,EAAE,CAAC;4BACR,WAAW,EAAE,CAAC;4BACd,WAAW,EAAE,CAAC;4BACd,SAAS,EAAE,WAAW,CAAC,MAAM;4BAC7B,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;yBAClD;qBACF;oBACD,QAAQ;iBACT,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;KAAA;IAED,yEAAyE;IACjE,SAAS,CAAC,OAAoB;;QACpC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,OAAO;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,MAAA,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,mCAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC7E,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aACzB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;aACvD,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;CACF;AA7GD,kCA6GC"}
|