@wanshi-kg/wanshi 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +458 -0
- package/dist/__tests__/helpers.js +27 -0
- package/dist/__tests__/helpers.js.map +1 -0
- package/dist/cli/commands/export.command.js +99 -0
- package/dist/cli/commands/export.command.js.map +1 -0
- package/dist/cli/commands/index.js +22 -0
- package/dist/cli/commands/index.js.map +1 -0
- package/dist/cli/commands/inspectMerges.command.js +84 -0
- package/dist/cli/commands/inspectMerges.command.js.map +1 -0
- package/dist/cli/commands/metrics.command.js +196 -0
- package/dist/cli/commands/metrics.command.js.map +1 -0
- package/dist/cli/commands/process.command.js +82 -0
- package/dist/cli/commands/process.command.js.map +1 -0
- package/dist/cli/commands/watch.command.js +91 -0
- package/dist/cli/commands/watch.command.js.map +1 -0
- package/dist/cli/index.js +269 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/optionsToConfig.js +160 -0
- package/dist/cli/optionsToConfig.js.map +1 -0
- package/dist/config/index.js +59 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/legacyHints.js +113 -0
- package/dist/config/legacyHints.js.map +1 -0
- package/dist/config/schema.js +803 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/ui.js +221 -0
- package/dist/config/ui.js.map +1 -0
- package/dist/core/DirectoryProcessor.js +725 -0
- package/dist/core/DirectoryProcessor.js.map +1 -0
- package/dist/core/adapters/IStructuredAdapter.js +3 -0
- package/dist/core/adapters/IStructuredAdapter.js.map +1 -0
- package/dist/core/adapters/SqliteAdapter.js +267 -0
- package/dist/core/adapters/SqliteAdapter.js.map +1 -0
- package/dist/core/adapters/StructuredAdapterRegistry.js +31 -0
- package/dist/core/adapters/StructuredAdapterRegistry.js.map +1 -0
- package/dist/core/adapters/index.js +20 -0
- package/dist/core/adapters/index.js.map +1 -0
- package/dist/core/checkpoint/CheckpointService.js +188 -0
- package/dist/core/checkpoint/CheckpointService.js.map +1 -0
- package/dist/core/checkpoint/index.js +18 -0
- package/dist/core/checkpoint/index.js.map +1 -0
- package/dist/core/corpus/CorpusAnalyzer.js +266 -0
- package/dist/core/corpus/CorpusAnalyzer.js.map +1 -0
- package/dist/core/corpus/CorpusProfileStore.js +92 -0
- package/dist/core/corpus/CorpusProfileStore.js.map +1 -0
- package/dist/core/corpus/index.js +21 -0
- package/dist/core/corpus/index.js.map +1 -0
- package/dist/core/corpus/normalizeGlossary.js +60 -0
- package/dist/core/corpus/normalizeGlossary.js.map +1 -0
- package/dist/core/corpus/relPath.js +52 -0
- package/dist/core/corpus/relPath.js.map +1 -0
- package/dist/core/corpus/termFrequency.js +86 -0
- package/dist/core/corpus/termFrequency.js.map +1 -0
- package/dist/core/cost/CostMeter.js +235 -0
- package/dist/core/cost/CostMeter.js.map +1 -0
- package/dist/core/cost/index.js +19 -0
- package/dist/core/cost/index.js.map +1 -0
- package/dist/core/cost/prices.js +38 -0
- package/dist/core/cost/prices.js.map +1 -0
- package/dist/core/cv/ObjectDetectionService.js +119 -0
- package/dist/core/cv/ObjectDetectionService.js.map +1 -0
- package/dist/core/di/ContainerFactory.js +670 -0
- package/dist/core/di/ContainerFactory.js.map +1 -0
- package/dist/core/di/DIContainer.js +103 -0
- package/dist/core/di/DIContainer.js.map +1 -0
- package/dist/core/di/index.js +19 -0
- package/dist/core/di/index.js.map +1 -0
- package/dist/core/errors/CustomErrors.js +342 -0
- package/dist/core/errors/CustomErrors.js.map +1 -0
- package/dist/core/errors/index.js +18 -0
- package/dist/core/errors/index.js.map +1 -0
- package/dist/core/export/KnowledgeGraphExportService.js +56 -0
- package/dist/core/export/KnowledgeGraphExportService.js.map +1 -0
- package/dist/core/export/index.js +19 -0
- package/dist/core/export/index.js.map +1 -0
- package/dist/core/export/strategies/GraphitiExportStrategy.js +115 -0
- package/dist/core/export/strategies/GraphitiExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/GraphvizDotExportStrategy.js +331 -0
- package/dist/core/export/strategies/GraphvizDotExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/IExportStrategy.js +3 -0
- package/dist/core/export/strategies/IExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/JsonExportStrategy.js +19 -0
- package/dist/core/export/strategies/JsonExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/JsonlExportStrategy.js +69 -0
- package/dist/core/export/strategies/JsonlExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/KblamExportStrategy.js +36 -0
- package/dist/core/export/strategies/KblamExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/LoraExportStrategy.js +46 -0
- package/dist/core/export/strategies/LoraExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/McpExportStrategy.js +67 -0
- package/dist/core/export/strategies/McpExportStrategy.js.map +1 -0
- package/dist/core/export/strategies/index.js +25 -0
- package/dist/core/export/strategies/index.js.map +1 -0
- package/dist/core/export/strategies/kbTriples.js +60 -0
- package/dist/core/export/strategies/kbTriples.js.map +1 -0
- package/dist/core/index.js +22 -0
- package/dist/core/index.js.map +1 -0
- package/dist/core/knowledge/KnowledgeGraphBuilder.js +627 -0
- package/dist/core/knowledge/KnowledgeGraphBuilder.js.map +1 -0
- package/dist/core/knowledge/MergeRecord.js +3 -0
- package/dist/core/knowledge/MergeRecord.js.map +1 -0
- package/dist/core/knowledge/canon/Canonicalizer.js +414 -0
- package/dist/core/knowledge/canon/Canonicalizer.js.map +1 -0
- package/dist/core/knowledge/canon/index.js +18 -0
- package/dist/core/knowledge/canon/index.js.map +1 -0
- package/dist/core/knowledge/contradiction/HeuristicContradictionChecker.js +92 -0
- package/dist/core/knowledge/contradiction/HeuristicContradictionChecker.js.map +1 -0
- package/dist/core/knowledge/contradiction/LlmContradictionChecker.js +52 -0
- package/dist/core/knowledge/contradiction/LlmContradictionChecker.js.map +1 -0
- package/dist/core/knowledge/contradiction/index.js +19 -0
- package/dist/core/knowledge/contradiction/index.js.map +1 -0
- package/dist/core/knowledge/grounding/KeywordGroundingChecker.js +33 -0
- package/dist/core/knowledge/grounding/KeywordGroundingChecker.js.map +1 -0
- package/dist/core/knowledge/grounding/MiniCheckGroundingChecker.js +82 -0
- package/dist/core/knowledge/grounding/MiniCheckGroundingChecker.js.map +1 -0
- package/dist/core/knowledge/grounding/index.js +20 -0
- package/dist/core/knowledge/grounding/index.js.map +1 -0
- package/dist/core/knowledge/grounding/verbalize.js +38 -0
- package/dist/core/knowledge/grounding/verbalize.js.map +1 -0
- package/dist/core/knowledge/images/imageMetaGraph.js +136 -0
- package/dist/core/knowledge/images/imageMetaGraph.js.map +1 -0
- package/dist/core/knowledge/index.js +20 -0
- package/dist/core/knowledge/index.js.map +1 -0
- package/dist/core/knowledge/merging/KnowledgeMerger.js +624 -0
- package/dist/core/knowledge/merging/KnowledgeMerger.js.map +1 -0
- package/dist/core/knowledge/references/ReferenceResolver.js +184 -0
- package/dist/core/knowledge/references/ReferenceResolver.js.map +1 -0
- package/dist/core/knowledge/references/citations/CitationEvidenceProcessor.js +401 -0
- package/dist/core/knowledge/references/citations/CitationEvidenceProcessor.js.map +1 -0
- package/dist/core/knowledge/references/citations/CitationResolver.js +95 -0
- package/dist/core/knowledge/references/citations/CitationResolver.js.map +1 -0
- package/dist/core/knowledge/references/citations/GrobidClient.js +143 -0
- package/dist/core/knowledge/references/citations/GrobidClient.js.map +1 -0
- package/dist/core/knowledge/references/citations/TitleIdResolver.js +101 -0
- package/dist/core/knowledge/references/citations/TitleIdResolver.js.map +1 -0
- package/dist/core/knowledge/references/web/FetchCacheService.js +114 -0
- package/dist/core/knowledge/references/web/FetchCacheService.js.map +1 -0
- package/dist/core/knowledge/references/web/GatedFetcher.js +228 -0
- package/dist/core/knowledge/references/web/GatedFetcher.js.map +1 -0
- package/dist/core/knowledge/references/web/WebReferenceProcessor.js +164 -0
- package/dist/core/knowledge/references/web/WebReferenceProcessor.js.map +1 -0
- package/dist/core/knowledge/search/KnowledgeGraphSearch.js +261 -0
- package/dist/core/knowledge/search/KnowledgeGraphSearch.js.map +1 -0
- package/dist/core/knowledge/vocabulary.js +162 -0
- package/dist/core/knowledge/vocabulary.js.map +1 -0
- package/dist/core/llm/EmbeddingService.js +113 -0
- package/dist/core/llm/EmbeddingService.js.map +1 -0
- package/dist/core/llm/OllamaService.js +146 -0
- package/dist/core/llm/OllamaService.js.map +1 -0
- package/dist/core/llm/OpenAICompatibleService.js +190 -0
- package/dist/core/llm/OpenAICompatibleService.js.map +1 -0
- package/dist/core/llm/OpenAIEmbeddingService.js +129 -0
- package/dist/core/llm/OpenAIEmbeddingService.js.map +1 -0
- package/dist/core/llm/embeddingUtils.js +25 -0
- package/dist/core/llm/embeddingUtils.js.map +1 -0
- package/dist/core/llm/index.js +23 -0
- package/dist/core/llm/index.js.map +1 -0
- package/dist/core/llm/prompts/PromptManager.js +388 -0
- package/dist/core/llm/prompts/PromptManager.js.map +1 -0
- package/dist/core/llm/prompts/PromptTemplateEngine.js +257 -0
- package/dist/core/llm/prompts/PromptTemplateEngine.js.map +1 -0
- package/dist/core/llm/prompts/templates/partials/examples/EXAMPLE_STYLE_GUIDE.md +84 -0
- package/dist/core/llm/prompts/templates/partials/examples/article.md +187 -0
- package/dist/core/llm/prompts/templates/partials/examples/code.md +229 -0
- package/dist/core/llm/prompts/templates/partials/examples/communication.md +205 -0
- package/dist/core/llm/prompts/templates/partials/examples/documentation.md +262 -0
- package/dist/core/llm/prompts/templates/partials/examples/financial.md +157 -0
- package/dist/core/llm/prompts/templates/partials/examples/legal.md +153 -0
- package/dist/core/llm/prompts/templates/partials/examples/logs.md +127 -0
- package/dist/core/llm/prompts/templates/partials/examples/medical.md +218 -0
- package/dist/core/llm/prompts/templates/partials/examples/notes.md +201 -0
- package/dist/core/llm/prompts/templates/partials/examples/research.md +208 -0
- package/dist/core/llm/prompts/templates/partials/examples/tabular.md +178 -0
- package/dist/core/llm/prompts/templates/partials/examples/transcript.md +204 -0
- package/dist/core/llm/prompts/templates/partials/retrieved-context.hbs +18 -0
- package/dist/core/llm/prompts/templates/v1/system.hbs +371 -0
- package/dist/core/llm/prompts/templates/v1/user.hbs +20 -0
- package/dist/core/llm/prompts/templates/v2/system.hbs +573 -0
- package/dist/core/llm/prompts/templates/v2/user.hbs +20 -0
- package/dist/core/llm/prompts/templates/v3/system.hbs +861 -0
- package/dist/core/llm/prompts/templates/v3/user.hbs +16 -0
- package/dist/core/llm/prompts/templates/v4/system.hbs +800 -0
- package/dist/core/llm/prompts/templates/v4/user.hbs +40 -0
- package/dist/core/llm/prompts/templates/v4.5/system.hbs +71 -0
- package/dist/core/llm/prompts/templates/v4.5/user.hbs +46 -0
- package/dist/core/llm/prompts/templates/v5/glossary/system.hbs +40 -0
- package/dist/core/llm/prompts/templates/v5/glossary/user.hbs +11 -0
- package/dist/core/llm/prompts/templates/v5/system.hbs +163 -0
- package/dist/core/llm/prompts/templates/v5/user.hbs +55 -0
- package/dist/core/pipeline/GroundingTransform.js +52 -0
- package/dist/core/pipeline/GroundingTransform.js.map +1 -0
- package/dist/core/pipeline/PipelineRunner.js +51 -0
- package/dist/core/pipeline/PipelineRunner.js.map +1 -0
- package/dist/core/pipeline/RelationFilterTransform.js +72 -0
- package/dist/core/pipeline/RelationFilterTransform.js.map +1 -0
- package/dist/core/pipeline/index.js +20 -0
- package/dist/core/pipeline/index.js.map +1 -0
- package/dist/core/processor/FileProcessor.js +184 -0
- package/dist/core/processor/FileProcessor.js.map +1 -0
- package/dist/core/processor/ProcessedRegistry.js +38 -0
- package/dist/core/processor/ProcessedRegistry.js.map +1 -0
- package/dist/core/processor/ast/AstSeedService.js +0 -0
- package/dist/core/processor/ast/AstSeedService.js.map +1 -0
- package/dist/core/processor/ast/AstSymbolStore.js +110 -0
- package/dist/core/processor/ast/AstSymbolStore.js.map +1 -0
- package/dist/core/processor/ast/index.js +19 -0
- package/dist/core/processor/ast/index.js.map +1 -0
- package/dist/core/processor/chunking/TextChunker.js +98 -0
- package/dist/core/processor/chunking/TextChunker.js.map +1 -0
- package/dist/core/processor/chunking/index.js +18 -0
- package/dist/core/processor/chunking/index.js.map +1 -0
- package/dist/core/processor/classifier/CONTENT_CLASSES.js +294 -0
- package/dist/core/processor/classifier/CONTENT_CLASSES.js.map +1 -0
- package/dist/core/processor/classifier/CascadeContentClassifier.js +107 -0
- package/dist/core/processor/classifier/CascadeContentClassifier.js.map +1 -0
- package/dist/core/processor/classifier/HeuristicContentClassifier.js +113 -0
- package/dist/core/processor/classifier/HeuristicContentClassifier.js.map +1 -0
- package/dist/core/processor/classifier/IContentTypeClassifier.js +3 -0
- package/dist/core/processor/classifier/IContentTypeClassifier.js.map +1 -0
- package/dist/core/processor/classifier/LlmContentClassifier.js +107 -0
- package/dist/core/processor/classifier/LlmContentClassifier.js.map +1 -0
- package/dist/core/processor/classifier/NER_DOMAIN_EXAMPLES.js +498 -0
- package/dist/core/processor/classifier/NER_DOMAIN_EXAMPLES.js.map +1 -0
- package/dist/core/processor/classifier/index.js +21 -0
- package/dist/core/processor/classifier/index.js.map +1 -0
- package/dist/core/processor/classifier/mergeClassifications.js +32 -0
- package/dist/core/processor/classifier/mergeClassifications.js.map +1 -0
- package/dist/core/processor/index.js +20 -0
- package/dist/core/processor/index.js.map +1 -0
- package/dist/core/processor/readers/AudioReader.js +462 -0
- package/dist/core/processor/readers/AudioReader.js.map +1 -0
- package/dist/core/processor/readers/BinaryReader.js +90 -0
- package/dist/core/processor/readers/BinaryReader.js.map +1 -0
- package/dist/core/processor/readers/ChandraPdfReader.js +187 -0
- package/dist/core/processor/readers/ChandraPdfReader.js.map +1 -0
- package/dist/core/processor/readers/ChatExportReader.js +365 -0
- package/dist/core/processor/readers/ChatExportReader.js.map +1 -0
- package/dist/core/processor/readers/DoclingReader.js +445 -0
- package/dist/core/processor/readers/DoclingReader.js.map +1 -0
- package/dist/core/processor/readers/EmailReader.js +259 -0
- package/dist/core/processor/readers/EmailReader.js.map +1 -0
- package/dist/core/processor/readers/EpubReader.js +175 -0
- package/dist/core/processor/readers/EpubReader.js.map +1 -0
- package/dist/core/processor/readers/FileReader.js +90 -0
- package/dist/core/processor/readers/FileReader.js.map +1 -0
- package/dist/core/processor/readers/FileReaderFactory.js +49 -0
- package/dist/core/processor/readers/FileReaderFactory.js.map +1 -0
- package/dist/core/processor/readers/HtmlReader.js +371 -0
- package/dist/core/processor/readers/HtmlReader.js.map +1 -0
- package/dist/core/processor/readers/ImageReader.js +162 -0
- package/dist/core/processor/readers/ImageReader.js.map +1 -0
- package/dist/core/processor/readers/JsonFileReader.js +232 -0
- package/dist/core/processor/readers/JsonFileReader.js.map +1 -0
- package/dist/core/processor/readers/JupyterReader.js +178 -0
- package/dist/core/processor/readers/JupyterReader.js.map +1 -0
- package/dist/core/processor/readers/LatexReader.js +176 -0
- package/dist/core/processor/readers/LatexReader.js.map +1 -0
- package/dist/core/processor/readers/MarkdownReader.js +289 -0
- package/dist/core/processor/readers/MarkdownReader.js.map +1 -0
- package/dist/core/processor/readers/MarkerPdfReader.js +193 -0
- package/dist/core/processor/readers/MarkerPdfReader.js.map +1 -0
- package/dist/core/processor/readers/MistralOcrReader.js +198 -0
- package/dist/core/processor/readers/MistralOcrReader.js.map +1 -0
- package/dist/core/processor/readers/OfficeReader.js +174 -0
- package/dist/core/processor/readers/OfficeReader.js.map +1 -0
- package/dist/core/processor/readers/PdfReader.js +116 -0
- package/dist/core/processor/readers/PdfReader.js.map +1 -0
- package/dist/core/processor/readers/RtfReader.js +107 -0
- package/dist/core/processor/readers/RtfReader.js.map +1 -0
- package/dist/core/processor/readers/SubtitleReader.js +145 -0
- package/dist/core/processor/readers/SubtitleReader.js.map +1 -0
- package/dist/core/processor/readers/TesseractPdfReader.js +183 -0
- package/dist/core/processor/readers/TesseractPdfReader.js.map +1 -0
- package/dist/core/processor/readers/TextReader.js +129 -0
- package/dist/core/processor/readers/TextReader.js.map +1 -0
- package/dist/core/processor/readers/TranscriptReader.js +234 -0
- package/dist/core/processor/readers/TranscriptReader.js.map +1 -0
- package/dist/core/processor/readers/image/imageMetadata.js +155 -0
- package/dist/core/processor/readers/image/imageMetadata.js.map +1 -0
- package/dist/core/processor/readers/index.js +41 -0
- package/dist/core/processor/readers/index.js.map +1 -0
- package/dist/core/processor/readers/referenceExtraction.js +198 -0
- package/dist/core/processor/readers/referenceExtraction.js.map +1 -0
- package/dist/core/processor/readers/stripReferences.js +59 -0
- package/dist/core/processor/readers/stripReferences.js.map +1 -0
- package/dist/core/processor/readers/transcript/turnPacking.js +81 -0
- package/dist/core/processor/readers/transcript/turnPacking.js.map +1 -0
- package/dist/core/progress/NdjsonProgressEmitter.js +30 -0
- package/dist/core/progress/NdjsonProgressEmitter.js.map +1 -0
- package/dist/core/progress/NoopProgressEmitter.js +15 -0
- package/dist/core/progress/NoopProgressEmitter.js.map +1 -0
- package/dist/core/progress/index.js +19 -0
- package/dist/core/progress/index.js.map +1 -0
- package/dist/core/trace/TraceWriter.js +100 -0
- package/dist/core/trace/TraceWriter.js.map +1 -0
- package/dist/core/trace/events.js +13 -0
- package/dist/core/trace/events.js.map +1 -0
- package/dist/core/trace/index.js +20 -0
- package/dist/core/trace/index.js.map +1 -0
- package/dist/core/trace/lineage.js +97 -0
- package/dist/core/trace/lineage.js.map +1 -0
- package/dist/evaluation/BenchmarkRunner.js +171 -0
- package/dist/evaluation/BenchmarkRunner.js.map +1 -0
- package/dist/evaluation/classifier/ClassifierAccuracy.js +185 -0
- package/dist/evaluation/classifier/ClassifierAccuracy.js.map +1 -0
- package/dist/evaluation/classifier/labeledSamples.js +379 -0
- package/dist/evaluation/classifier/labeledSamples.js.map +1 -0
- package/dist/evaluation/compare/goldCompare.js +126 -0
- package/dist/evaluation/compare/goldCompare.js.map +1 -0
- package/dist/evaluation/crossre/compareScoring.js +30 -0
- package/dist/evaluation/crossre/compareScoring.js.map +1 -0
- package/dist/evaluation/datasets/CrossREDataset.js +170 -0
- package/dist/evaluation/datasets/CrossREDataset.js.map +1 -0
- package/dist/evaluation/datasets/IDataset.js +3 -0
- package/dist/evaluation/datasets/IDataset.js.map +1 -0
- package/dist/evaluation/datasets/RebelDataset.js +117 -0
- package/dist/evaluation/datasets/RebelDataset.js.map +1 -0
- package/dist/evaluation/datasets/RedocredDataset.js +218 -0
- package/dist/evaluation/datasets/RedocredDataset.js.map +1 -0
- package/dist/evaluation/datasets/SemEval2010Dataset.js +150 -0
- package/dist/evaluation/datasets/SemEval2010Dataset.js.map +1 -0
- package/dist/evaluation/index.js +33 -0
- package/dist/evaluation/index.js.map +1 -0
- package/dist/evaluation/matching/ExactMatcher.js +75 -0
- package/dist/evaluation/matching/ExactMatcher.js.map +1 -0
- package/dist/evaluation/matching/SemanticMatcher.js +143 -0
- package/dist/evaluation/matching/SemanticMatcher.js.map +1 -0
- package/dist/evaluation/metrics/TripleMetrics.js +64 -0
- package/dist/evaluation/metrics/TripleMetrics.js.map +1 -0
- package/dist/evaluation/mine/MineCheckpoint.js +114 -0
- package/dist/evaluation/mine/MineCheckpoint.js.map +1 -0
- package/dist/evaluation/mine/MineDataset.js +208 -0
- package/dist/evaluation/mine/MineDataset.js.map +1 -0
- package/dist/evaluation/mine/MineReporter.js +98 -0
- package/dist/evaluation/mine/MineReporter.js.map +1 -0
- package/dist/evaluation/mine/MineRunner.js +148 -0
- package/dist/evaluation/mine/MineRunner.js.map +1 -0
- package/dist/evaluation/mine/MineScorer.js +127 -0
- package/dist/evaluation/mine/MineScorer.js.map +1 -0
- package/dist/evaluation/mine/types.js +12 -0
- package/dist/evaluation/mine/types.js.map +1 -0
- package/dist/evaluation/reporters/ConsoleReporter.js +55 -0
- package/dist/evaluation/reporters/ConsoleReporter.js.map +1 -0
- package/dist/evaluation/reporters/JsonReporter.js +50 -0
- package/dist/evaluation/reporters/JsonReporter.js.map +1 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/quality/CompositeScore.js +61 -0
- package/dist/quality/CompositeScore.js.map +1 -0
- package/dist/quality/ConsistencyMetrics.js +70 -0
- package/dist/quality/ConsistencyMetrics.js.map +1 -0
- package/dist/quality/FactualMetrics.js +76 -0
- package/dist/quality/FactualMetrics.js.map +1 -0
- package/dist/quality/GraphHealthMetrics.js +68 -0
- package/dist/quality/GraphHealthMetrics.js.map +1 -0
- package/dist/quality/SemanticMetrics.js +102 -0
- package/dist/quality/SemanticMetrics.js.map +1 -0
- package/dist/quality/StructuralMetrics.js +60 -0
- package/dist/quality/StructuralMetrics.js.map +1 -0
- package/dist/quality/index.js +23 -0
- package/dist/quality/index.js.map +1 -0
- package/dist/shared/index.js +20 -0
- package/dist/shared/index.js.map +1 -0
- package/dist/shared/logger/Logger.js +3 -0
- package/dist/shared/logger/Logger.js.map +1 -0
- package/dist/shared/logger/LoggerFactory.js +75 -0
- package/dist/shared/logger/LoggerFactory.js.map +1 -0
- package/dist/shared/logger/index.js +19 -0
- package/dist/shared/logger/index.js.map +1 -0
- package/dist/shared/shutdown.js +30 -0
- package/dist/shared/shutdown.js.map +1 -0
- package/dist/shared/utils/agglomerativeCluster.js +269 -0
- package/dist/shared/utils/agglomerativeCluster.js.map +1 -0
- package/dist/shared/utils/astSymbols.js +69 -0
- package/dist/shared/utils/astSymbols.js.map +1 -0
- package/dist/shared/utils/cosineSimilarity.js +18 -0
- package/dist/shared/utils/cosineSimilarity.js.map +1 -0
- package/dist/shared/utils/directoryTree.js +184 -0
- package/dist/shared/utils/directoryTree.js.map +1 -0
- package/dist/shared/utils/documentOutline.js +74 -0
- package/dist/shared/utils/documentOutline.js.map +1 -0
- package/dist/shared/utils/index.js +24 -0
- package/dist/shared/utils/index.js.map +1 -0
- package/dist/shared/utils/jaroWinklerSimilarity.js +60 -0
- package/dist/shared/utils/jaroWinklerSimilarity.js.map +1 -0
- package/dist/shared/utils/parseJsonLenient.js +27 -0
- package/dist/shared/utils/parseJsonLenient.js.map +1 -0
- package/dist/shared/utils/readConfig.js +42 -0
- package/dist/shared/utils/readConfig.js.map +1 -0
- package/dist/shared/utils/readRtf.js +216 -0
- package/dist/shared/utils/readRtf.js.map +1 -0
- package/dist/shared/utils/softmax.js +26 -0
- package/dist/shared/utils/softmax.js.map +1 -0
- package/dist/types/ContentClass.js +3 -0
- package/dist/types/ContentClass.js.map +1 -0
- package/dist/types/CorpusProfile.js +3 -0
- package/dist/types/CorpusProfile.js.map +1 -0
- package/dist/types/IContradictionChecker.js +3 -0
- package/dist/types/IContradictionChecker.js.map +1 -0
- package/dist/types/ICorpusAnalyzer.js +3 -0
- package/dist/types/ICorpusAnalyzer.js.map +1 -0
- package/dist/types/IDirectoryProcessor.js +3 -0
- package/dist/types/IDirectoryProcessor.js.map +1 -0
- package/dist/types/IEmbeddingProvider.js +3 -0
- package/dist/types/IEmbeddingProvider.js.map +1 -0
- package/dist/types/IEmbeddingService.js +6 -0
- package/dist/types/IEmbeddingService.js.map +1 -0
- package/dist/types/IFileProcessor.js +3 -0
- package/dist/types/IFileProcessor.js.map +1 -0
- package/dist/types/IGroundingChecker.js +3 -0
- package/dist/types/IGroundingChecker.js.map +1 -0
- package/dist/types/IKnowledgeGraphBuilder.js +3 -0
- package/dist/types/IKnowledgeGraphBuilder.js.map +1 -0
- package/dist/types/IKnowledgeGraphExporter.js +3 -0
- package/dist/types/IKnowledgeGraphExporter.js.map +1 -0
- package/dist/types/IKnowledgeGraphMerger.js +3 -0
- package/dist/types/IKnowledgeGraphMerger.js.map +1 -0
- package/dist/types/IKnowledgeGraphSearch.js +3 -0
- package/dist/types/IKnowledgeGraphSearch.js.map +1 -0
- package/dist/types/ILLMProvider.js +3 -0
- package/dist/types/ILLMProvider.js.map +1 -0
- package/dist/types/ILLMService.js +3 -0
- package/dist/types/ILLMService.js.map +1 -0
- package/dist/types/IObjectDetector.js +3 -0
- package/dist/types/IObjectDetector.js.map +1 -0
- package/dist/types/IProcessingService.js +3 -0
- package/dist/types/IProcessingService.js.map +1 -0
- package/dist/types/IProgressEmitter.js +3 -0
- package/dist/types/IProgressEmitter.js.map +1 -0
- package/dist/types/IPromptManager.js +3 -0
- package/dist/types/IPromptManager.js.map +1 -0
- package/dist/types/KnowledgeGraph.js +3 -0
- package/dist/types/KnowledgeGraph.js.map +1 -0
- package/dist/types/MCPKnowledgeGraph.js +3 -0
- package/dist/types/MCPKnowledgeGraph.js.map +1 -0
- package/dist/types/Observation.js +21 -0
- package/dist/types/Observation.js.map +1 -0
- package/dist/types/ProcessingOptions.js +3 -0
- package/dist/types/ProcessingOptions.js.map +1 -0
- package/dist/types/index.js +40 -0
- package/dist/types/index.js.map +1 -0
- package/package.json +122 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"referenceExtraction.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/referenceExtraction.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;GAaG;;AAoEH,4CAEC;AAKD,oDAoBC;AAGD,4CAUC;AASD,0CAUC;AA4ED,4CAkCC;AA9ND,IAAI,QAAqC,CAAC;AAC1C,SAAS,QAAQ;IACf,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,OAAO,CAAC,mBAAmB,CAAuB,CAAC;QAChE,OAAO,CAAC,4BAA4B,CAAC,CAAC,CAAC,6CAA6C;QACpF,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;IACvB,CAAC;IAAC,WAAM,CAAC;QACP,QAAQ,GAAG,IAAI,CAAC;IAClB,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AA6BD,iFAAiF;AACjF,MAAM,QAAQ,GAAG,sCAAsC,CAAC;AACxD,MAAM,MAAM,GAAG,oCAAoC,CAAC;AACpD,MAAM,OAAO,GAAG,yBAAyB,CAAC;AAC1C,MAAM,cAAc,GAClB,qFAAqF,CAAC;AAExF,yEAAyE;AACzE,SAAS,iBAAiB,CAAC,CAAS;IAClC,OAAO,CAAC,CAAC,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC;AACtC,CAAC;AAED,uFAAuF;AACvF,SAAgB,gBAAgB,CAAC,MAAc;IAC7C,OAAO,+BAA+B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,iFAAiF;AAEjF,wFAAwF;AACxF,SAAgB,oBAAoB,CAAC,IAAY;;IAC/C,MAAM,KAAK,GAAc,EAAE,CAAC;IAE5B,8EAA8E;IAC9E,MAAM,MAAM,GAAG,0CAA0C,CAAC;IAC1D,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,sCAAsC;IACtC,MAAM,IAAI,GAAG,oCAAoC,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM;YAAE,SAAS;QACtB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA,MAAA,CAAC,CAAC,CAAC,CAAC,0CAAE,IAAI,EAAE,KAAI,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,oEAAoE;AACpE,SAAgB,gBAAgB,CAAC,IAAY;IAC3C,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,+DAA+D,CAAC;IAC/E,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3B,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAChD,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACjD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,IAAI,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,eAAe,CAAC,IAAY;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAc,EAAE,CAAC;IAC1B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gCAAgC,CAAC,EAAE,CAAC;QAChE,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,SAAS;QAC/B,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjB,GAAG,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,iFAAiF;AAEjF,SAAS,WAAW,CAAC,CAAc;;IACjC,OAAO,CACL,MAAA,MAAA,MAAA,MAAA,MAAA,CAAC,CAAC,OAAO,0CAAE,WAAW,EAAE,mCACxB,MAAA,CAAC,CAAC,GAAG,0CAAE,WAAW,EAAE,mCACpB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,mCACvC,MAAA,CAAC,CAAC,KAAK,0CAAE,WAAW,EAAE,mCACtB,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CACpB,CAAC;AACJ,CAAC;AAED,mEAAmE;AACnE,SAAS,UAAU,CAAC,CAAS;IAC3B,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACvD,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACrD,qDACK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GACpC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAC/C,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAClC;AACJ,CAAC;AAED,oFAAoF;AACpF,SAAS,eAAe,CAAC,KAAa;;IACpC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC,CAAC,kCAAkC;IAC3E,MAAM,IAAI,GAAG,QAAQ,EAAE,CAAC;IACxB,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,CAAC;IACrB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAA,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,mCAAI,EAAE,CAAC;QAC3C,OAAO,OAAO;aACX,GAAG,CAAC,CAAC,CAAC,EAAe,EAAE;;YACtB,MAAM,GAAG,GAAG,CAAC,MAAA,CAAC,CAAC,KAAK,mCAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAC;YAC9C,qCACE,GAAG,EAAE,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IACxC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAC3C,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EACxC;QACJ,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,CAAC;IAAC,WAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,sFAAsF;AACtF,SAAS,iBAAiB,CAAC,KAAa;IACtC,MAAM,KAAK,GAAG,KAAK;SAChB,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,yBAAyB;SACvF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAExB,MAAM,MAAM,GAAG,oCAAoC,CAAC;IACpD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAErD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI;gBAAE,SAAS;YACpB,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;iBACzD,IAAI,OAAO,CAAC,MAAM;gBAAE,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;;gBAC9D,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,8CAA8C;IAC9C,OAAO,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC/B,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAC9B,eAAmC,EACnC,QAAgB;IAEhB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC3C,MAAM,GAAG,GAAG,CAAC,CAAc,EAAE,EAAE;QAC7B,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC;IAEF,IAAI,eAAe,IAAI,eAAe,CAAC,IAAI,EAAE,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,eAAe,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACtB,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,WAAC,OAAA,GAAG,iCAAM,CAAC,GAAK,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC,MAAA,CAAC,CAAC,GAAG,mCAAI,EAAE,CAAC,CAAC,EAAG,CAAA,EAAA,CAAC,CAAC;QACvF,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,KAAK,IAAI,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;gBACvD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;gBAC9B,GAAG,iBAAG,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAK,GAAG,EAAG,CAAC;YAC5C,CAAC;QACH,CAAC;IACH,CAAC;IAED,gFAAgF;IAChF,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;QACrE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;QACnE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACnD,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;QACpE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;AAClC,CAAC"}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Trailing references/bibliography quarantine (academic-paper hygiene).
|
|
4
|
+
*
|
|
5
|
+
* Bibliography text extracted as content turns citation titles into observations
|
|
6
|
+
* and author surnames into garbage person-entities tangled into nonsense relations
|
|
7
|
+
* (`B. Thirion has_attribute red pepper`). These helpers split the references
|
|
8
|
+
* section off BEFORE chunking/extraction; the tail is dropped, not extracted.
|
|
9
|
+
*
|
|
10
|
+
* Heuristic: the LAST line that is a references-style heading, located in the
|
|
11
|
+
* final 40% of the document — early "References" mentions (e.g. a section that
|
|
12
|
+
* discusses related work) are never treated as the bibliography.
|
|
13
|
+
*/
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.splitTrailingReferences = splitTrailingReferences;
|
|
16
|
+
exports.splitPagesAtReferences = splitPagesAtReferences;
|
|
17
|
+
const REF_HEADING = /^(?:#{1,6}\s*)?(?:[\divxlc]+\.?\s+)?(references|bibliography|works cited)\s*:?\s*$/i;
|
|
18
|
+
/** Split plain/markdown text at the trailing references heading. */
|
|
19
|
+
function splitTrailingReferences(text) {
|
|
20
|
+
const lines = text.split("\n");
|
|
21
|
+
const earliest = Math.floor(lines.length * 0.6);
|
|
22
|
+
for (let i = lines.length - 1; i >= earliest; i--) {
|
|
23
|
+
if (REF_HEADING.test(lines[i].trim())) {
|
|
24
|
+
return {
|
|
25
|
+
body: lines.slice(0, i).join("\n"),
|
|
26
|
+
references: lines.slice(i).join("\n"),
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return { body: text };
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Page-array variant (PDF readers): truncate the page containing the trailing
|
|
34
|
+
* references heading and drop all subsequent pages.
|
|
35
|
+
*/
|
|
36
|
+
function splitPagesAtReferences(pages) {
|
|
37
|
+
const totalChars = pages.reduce((acc, p) => acc + p.length, 0);
|
|
38
|
+
const earliestChar = totalChars * 0.6;
|
|
39
|
+
let offset = totalChars;
|
|
40
|
+
for (let p = pages.length - 1; p >= 0; p--) {
|
|
41
|
+
offset -= pages[p].length;
|
|
42
|
+
const lines = pages[p].split("\n");
|
|
43
|
+
for (let i = lines.length - 1; i >= 0; i--) {
|
|
44
|
+
if (!REF_HEADING.test(lines[i].trim()))
|
|
45
|
+
continue;
|
|
46
|
+
const headingChar = offset + lines.slice(0, i).join("\n").length;
|
|
47
|
+
if (headingChar < earliestChar)
|
|
48
|
+
continue;
|
|
49
|
+
const body = lines.slice(0, i).join("\n");
|
|
50
|
+
const tail = [lines.slice(i).join("\n"), ...pages.slice(p + 1)].join("\n");
|
|
51
|
+
const kept = [...pages.slice(0, p)];
|
|
52
|
+
if (body.trim().length > 0)
|
|
53
|
+
kept.push(body);
|
|
54
|
+
return { pages: kept, references: tail };
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return { pages };
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=stripReferences.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stripReferences.js","sourceRoot":"","sources":["../../../../src/core/processor/readers/stripReferences.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;GAWG;;AAWH,0DAYC;AAWD,wDAoBC;AApDD,MAAM,WAAW,GACf,qFAAqF,CAAC;AAOxF,oEAAoE;AACpE,SAAgB,uBAAuB,CAAC,IAAY;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IAChD,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;QAClD,IAAI,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YACtC,OAAO;gBACL,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBAClC,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;aACtC,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AACxB,CAAC;AAOD;;;GAGG;AACH,SAAgB,sBAAsB,CAAC,KAAe;IACpD,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,UAAU,GAAG,GAAG,CAAC;IAEtC,IAAI,MAAM,GAAG,UAAU,CAAC;IACxB,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;gBAAE,SAAS;YACjD,MAAM,WAAW,GAAG,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YACjE,IAAI,WAAW,GAAG,YAAY;gBAAE,SAAS;YACzC,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1C,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACpC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.renderTurn = renderTurn;
|
|
13
|
+
exports.packTurns = packTurns;
|
|
14
|
+
/** Render a turn with its inline speaker label (`speaker: text`). */
|
|
15
|
+
function renderTurn(turn) {
|
|
16
|
+
return `${turn.speaker}: ${turn.text}`;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Pack consecutive turns into chunks up to `maxChunkSize` (regardless of
|
|
20
|
+
* speaker), rendering each turn inline as `speaker: text`. Provenance carries
|
|
21
|
+
* the speaker only when a chunk turns out to be single-speaker; a turn longer
|
|
22
|
+
* than the budget on its own is split with the label kept on every piece.
|
|
23
|
+
*
|
|
24
|
+
* Shared by `TranscriptReader` (parsed transcript files) and `AudioReader`'s
|
|
25
|
+
* dual-STT engine (turns emitted by the vendored Python pipeline) so both
|
|
26
|
+
* produce identical speaker-stamped chunk provenance.
|
|
27
|
+
*/
|
|
28
|
+
function packTurns(turns, sourcePath, maxChunkSize, chunker) {
|
|
29
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
30
|
+
const SEP = "\n\n";
|
|
31
|
+
const chunks = [];
|
|
32
|
+
let buf = [];
|
|
33
|
+
const renderLen = (t) => renderTurn(t).length;
|
|
34
|
+
const bufLen = () => buf.reduce((n, t) => n + renderLen(t) + SEP.length, 0);
|
|
35
|
+
const flush = () => {
|
|
36
|
+
var _a;
|
|
37
|
+
if (buf.length === 0)
|
|
38
|
+
return;
|
|
39
|
+
const speakers = new Set(buf.map((t) => t.speaker));
|
|
40
|
+
const occurredAt = (_a = buf.find((t) => t.occurredAt)) === null || _a === void 0 ? void 0 : _a.occurredAt;
|
|
41
|
+
const text = buf.map((t) => renderTurn(t)).join(SEP).trim();
|
|
42
|
+
const provenance = Object.assign(Object.assign({ source: sourcePath }, (speakers.size === 1 && { speaker: buf[0].speaker })), (occurredAt && { occurredAt }));
|
|
43
|
+
chunks.push({ content: text, index: 0, totalChunks: 0, startOffset: 0, endOffset: text.length, provenance });
|
|
44
|
+
buf = [];
|
|
45
|
+
};
|
|
46
|
+
// Split a single oversized turn, keeping its speaker label on each piece.
|
|
47
|
+
const flushOversized = (turn) => __awaiter(this, void 0, void 0, function* () {
|
|
48
|
+
const provenance = Object.assign({ source: sourcePath, speaker: turn.speaker }, (turn.occurredAt && { occurredAt: turn.occurredAt }));
|
|
49
|
+
for (const p of yield chunker.chunk(turn.text)) {
|
|
50
|
+
const content = `${turn.speaker}: ${p.content}`;
|
|
51
|
+
chunks.push({ content, index: 0, totalChunks: 0, startOffset: p.startOffset, endOffset: p.endOffset, provenance });
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
for (const turn of turns) {
|
|
55
|
+
if (!turn.text)
|
|
56
|
+
continue;
|
|
57
|
+
// Conversation boundary (KG-10): a chat export's conversations must never
|
|
58
|
+
// share a chunk, so flush before a turn from a different conversation —
|
|
59
|
+
// regardless of remaining budget — keeping validAt/speaker per-conversation.
|
|
60
|
+
if (buf.length > 0 && buf[buf.length - 1].conversation !== turn.conversation) {
|
|
61
|
+
flush();
|
|
62
|
+
}
|
|
63
|
+
if (renderLen(turn) > maxChunkSize) {
|
|
64
|
+
flush();
|
|
65
|
+
yield flushOversized(turn);
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
if (buf.length > 0 && bufLen() + renderLen(turn) + SEP.length > maxChunkSize) {
|
|
69
|
+
flush();
|
|
70
|
+
}
|
|
71
|
+
buf.push(turn);
|
|
72
|
+
}
|
|
73
|
+
flush();
|
|
74
|
+
chunks.forEach((c, i) => {
|
|
75
|
+
c.index = i + 1;
|
|
76
|
+
c.totalChunks = chunks.length;
|
|
77
|
+
});
|
|
78
|
+
return chunks;
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=turnPacking.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"turnPacking.js","sourceRoot":"","sources":["../../../../../src/core/processor/readers/transcript/turnPacking.ts"],"names":[],"mappings":";;;;;;;;;;;AAgBA,gCAEC;AAYD,8BAgEC;AA/ED,qEAAqE;AACrE,SAAgB,UAAU,CAAC,IAAU;IACnC,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC;AAED;;;;;;;;;GASG;AACH,SAAsB,SAAS,CAC7B,KAAa,EACb,UAAkB,EAClB,YAAoB,EACpB,OAAoB;;QAEpB,MAAM,GAAG,GAAG,MAAM,CAAC;QACnB,MAAM,MAAM,GAAkB,EAAE,CAAC;QACjC,IAAI,GAAG,GAAW,EAAE,CAAC;QACrB,MAAM,SAAS,GAAG,CAAC,CAAO,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QACpD,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAE5E,MAAM,KAAK,GAAG,GAAG,EAAE;;YACjB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAC7B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;YACpD,MAAM,UAAU,GAAG,MAAA,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,0CAAE,UAAU,CAAC;YAC7D,MAAM,IAAI,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5D,MAAM,UAAU,iCACd,MAAM,EAAE,UAAU,IACf,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,GACpD,CAAC,UAAU,IAAI,EAAE,UAAU,EAAE,CAAC,CAClC,CAAC;YACF,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YAC7G,GAAG,GAAG,EAAE,CAAC;QACX,CAAC,CAAC;QAEF,0EAA0E;QAC1E,MAAM,cAAc,GAAG,CAAO,IAAU,EAAE,EAAE;YAC1C,MAAM,UAAU,mBACd,MAAM,EAAE,UAAU,EAClB,OAAO,EAAE,IAAI,CAAC,OAAO,IAClB,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CACxD,CAAC;YACF,KAAK,MAAM,CAAC,IAAI,MAAM,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC/C,MAAM,OAAO,GAAG,GAAG,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;gBAChD,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC;YACrH,CAAC;QACH,CAAC,CAAA,CAAC;QAEF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,IAAI,CAAC,IAAI;gBAAE,SAAS;YACzB,0EAA0E;YAC1E,wEAAwE;YACxE,6EAA6E;YAC7E,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;gBAC7E,KAAK,EAAE,CAAC;YACV,CAAC;YACD,IAAI,SAAS,CAAC,IAAI,CAAC,GAAG,YAAY,EAAE,CAAC;gBACnC,KAAK,EAAE,CAAC;gBACR,MAAM,cAAc,CAAC,IAAI,CAAC,CAAC;gBAC3B,SAAS;YACX,CAAC;YACD,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBAC7E,KAAK,EAAE,CAAC;YACV,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC;QACD,KAAK,EAAE,CAAC;QAER,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACtB,CAAC,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;YAChB,CAAC,CAAC,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC;QAChC,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;CAAA"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NdjsonProgressEmitter = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Emits each progress event as a single newline-delimited JSON object on a
|
|
6
|
+
* writable stream (stdout by default). Lines are wrapped in an envelope with a
|
|
7
|
+
* `channel` discriminator so the same stream can also carry log lines
|
|
8
|
+
* (`channel: "log"`, written by the logger in NDJSON mode) — a consumer reading
|
|
9
|
+
* the stream demuxes the two by `channel`.
|
|
10
|
+
*
|
|
11
|
+
* {"channel":"progress","ts":1717600000000,"event":{"type":"file_start",...}}
|
|
12
|
+
*
|
|
13
|
+
* Writes are best-effort and never throw, so a broken pipe (consumer went away)
|
|
14
|
+
* can't take down a run.
|
|
15
|
+
*/
|
|
16
|
+
class NdjsonProgressEmitter {
|
|
17
|
+
constructor(stream = process.stdout) {
|
|
18
|
+
this.stream = stream;
|
|
19
|
+
}
|
|
20
|
+
emit(event) {
|
|
21
|
+
try {
|
|
22
|
+
this.stream.write(JSON.stringify({ channel: "progress", ts: Date.now(), event }) + "\n");
|
|
23
|
+
}
|
|
24
|
+
catch (_a) {
|
|
25
|
+
// side-channel only — swallow write errors (e.g. EPIPE)
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
exports.NdjsonProgressEmitter = NdjsonProgressEmitter;
|
|
30
|
+
//# sourceMappingURL=NdjsonProgressEmitter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"NdjsonProgressEmitter.js","sourceRoot":"","sources":["../../../src/core/progress/NdjsonProgressEmitter.ts"],"names":[],"mappings":";;;AAEA;;;;;;;;;;;GAWG;AACH,MAAa,qBAAqB;IAChC,YAA6B,SAAgC,OAAO,CAAC,MAAM;QAA9C,WAAM,GAAN,MAAM,CAAwC;IAAG,CAAC;IAE/E,IAAI,CAAC,KAAoB;QACvB,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,IAAI,CACtE,CAAC;QACJ,CAAC;QAAC,WAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;IACH,CAAC;CACF;AAZD,sDAYC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NoopProgressEmitter = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Default progress emitter: discards every event. Used whenever structured
|
|
6
|
+
* progress isn't requested, so the normal CLI path has zero overhead and
|
|
7
|
+
* unchanged behavior.
|
|
8
|
+
*/
|
|
9
|
+
class NoopProgressEmitter {
|
|
10
|
+
emit() {
|
|
11
|
+
// intentionally empty
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
exports.NoopProgressEmitter = NoopProgressEmitter;
|
|
15
|
+
//# sourceMappingURL=NoopProgressEmitter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"NoopProgressEmitter.js","sourceRoot":"","sources":["../../../src/core/progress/NoopProgressEmitter.ts"],"names":[],"mappings":";;;AAEA;;;;GAIG;AACH,MAAa,mBAAmB;IAC9B,IAAI;QACF,sBAAsB;IACxB,CAAC;CACF;AAJD,kDAIC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./NoopProgressEmitter"), exports);
|
|
18
|
+
__exportStar(require("./NdjsonProgressEmitter"), exports);
|
|
19
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/progress/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,wDAAsC;AACtC,0DAAwC"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.trace = exports.TraceWriter = void 0;
|
|
37
|
+
const fs = __importStar(require("fs"));
|
|
38
|
+
const path = __importStar(require("path"));
|
|
39
|
+
const events_1 = require("./events");
|
|
40
|
+
const lineage_1 = require("./lineage");
|
|
41
|
+
/**
|
|
42
|
+
* The debug trace sidecar writer — a module singleton (à la `shared/shutdown.ts`
|
|
43
|
+
* and `vocabulary.ts`'s domain-gate) so every pipeline stage can `trace.emit(...)`
|
|
44
|
+
* without re-plumbing the DI graph. `ContainerFactory` calls `configure` once.
|
|
45
|
+
*
|
|
46
|
+
* Append-only JSONL (`<output>.trace.jsonl`), mirroring `CheckpointService`:
|
|
47
|
+
* parent-dir ensured on first write, best-effort, **never throws** — a trace I/O
|
|
48
|
+
* failure must not take down (or alter) a run. Off by default ⇒ `emit` is an
|
|
49
|
+
* early return and call sites guard rich payloads with `if (trace.enabled)`, so a
|
|
50
|
+
* default run carries zero overhead and is byte-identical.
|
|
51
|
+
*/
|
|
52
|
+
class TraceWriter {
|
|
53
|
+
constructor() {
|
|
54
|
+
this._enabled = false;
|
|
55
|
+
this.runId = "";
|
|
56
|
+
this.seq = 0;
|
|
57
|
+
this.dirEnsured = false;
|
|
58
|
+
/** Run-scoped lineage index (mention IDs live here, never on the graph). */
|
|
59
|
+
this.lineage = new lineage_1.LineageRegistry();
|
|
60
|
+
}
|
|
61
|
+
get enabled() {
|
|
62
|
+
return this._enabled;
|
|
63
|
+
}
|
|
64
|
+
configure(config) {
|
|
65
|
+
this._enabled = config.enabled && !!config.path;
|
|
66
|
+
this.outputPath = config.path;
|
|
67
|
+
this.runId = config.runId;
|
|
68
|
+
this.seq = 0;
|
|
69
|
+
this.dirEnsured = false;
|
|
70
|
+
this.lineage.reset();
|
|
71
|
+
}
|
|
72
|
+
emit(event) {
|
|
73
|
+
if (!this._enabled || !this.outputPath)
|
|
74
|
+
return;
|
|
75
|
+
const record = Object.assign({ v: events_1.TRACE_VERSION, runId: this.runId, ts: new Date().toISOString(), seq: this.seq++ }, event);
|
|
76
|
+
try {
|
|
77
|
+
if (!this.dirEnsured) {
|
|
78
|
+
fs.mkdirSync(path.dirname(this.outputPath), { recursive: true });
|
|
79
|
+
this.dirEnsured = true;
|
|
80
|
+
}
|
|
81
|
+
fs.appendFileSync(this.outputPath, JSON.stringify(record) + "\n");
|
|
82
|
+
}
|
|
83
|
+
catch (_a) {
|
|
84
|
+
// Best-effort side channel — swallow (e.g. ENOSPC/EPIPE) so the run is unaffected.
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/** Test/reset hook — restore the disabled default + clear lineage. */
|
|
88
|
+
reset() {
|
|
89
|
+
this._enabled = false;
|
|
90
|
+
this.outputPath = undefined;
|
|
91
|
+
this.runId = "";
|
|
92
|
+
this.seq = 0;
|
|
93
|
+
this.dirEnsured = false;
|
|
94
|
+
this.lineage.reset();
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
exports.TraceWriter = TraceWriter;
|
|
98
|
+
/** The process-wide trace singleton. */
|
|
99
|
+
exports.trace = new TraceWriter();
|
|
100
|
+
//# sourceMappingURL=TraceWriter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"TraceWriter.js","sourceRoot":"","sources":["../../../src/core/trace/TraceWriter.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,qCAAkE;AAClE,uCAA4C;AAQ5C;;;;;;;;;;GAUG;AACH,MAAa,WAAW;IAAxB;QACU,aAAQ,GAAG,KAAK,CAAC;QAEjB,UAAK,GAAG,EAAE,CAAC;QACX,QAAG,GAAG,CAAC,CAAC;QACR,eAAU,GAAG,KAAK,CAAC;QAC3B,4EAA4E;QACnE,YAAO,GAAG,IAAI,yBAAe,EAAE,CAAC;IA4C3C,CAAC;IA1CC,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC;IACvB,CAAC;IAED,SAAS,CAAC,MAAmB;QAC3B,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;QAChD,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QAC9B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QAC1B,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;IAED,IAAI,CAAC,KAAiB;QACpB,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,IAAI,CAAC,UAAU;YAAE,OAAO;QAC/C,MAAM,MAAM,mBACV,CAAC,EAAE,sBAAa,EAChB,KAAK,EAAE,IAAI,CAAC,KAAK,EACjB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAC5B,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IACZ,KAAK,CACT,CAAC;QACF,IAAI,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrB,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;gBACjE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACzB,CAAC;YACD,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC;QACpE,CAAC;QAAC,WAAM,CAAC;YACP,mFAAmF;QACrF,CAAC;IACH,CAAC;IAED,sEAAsE;IACtE,KAAK;QACH,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC;QACb,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QACxB,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACF;AAnDD,kCAmDC;AAED,wCAAwC;AAC3B,QAAA,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TRACE_VERSION = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Debug trace event taxonomy. One `type` per pipeline decision point. Every event
|
|
6
|
+
* is wrapped by `TraceWriter` in an envelope (`v`/`runId`/`ts`/`seq`) before it
|
|
7
|
+
* hits the JSONL sidecar — these interfaces describe the per-event *payload*.
|
|
8
|
+
*
|
|
9
|
+
* Bump TRACE_VERSION whenever an event's shape changes (same discipline as the
|
|
10
|
+
* checkpoint key / prompt version) so a future inspector can branch on it.
|
|
11
|
+
*/
|
|
12
|
+
exports.TRACE_VERSION = 1;
|
|
13
|
+
//# sourceMappingURL=events.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"events.js","sourceRoot":"","sources":["../../../src/core/trace/events.ts"],"names":[],"mappings":";;;AAAA;;;;;;;GAOG;AACU,QAAA,aAAa,GAAG,CAAC,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
|
+
};
|
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
__exportStar(require("./events"), exports);
|
|
18
|
+
__exportStar(require("./lineage"), exports);
|
|
19
|
+
__exportStar(require("./TraceWriter"), exports);
|
|
20
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/trace/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,2CAAyB;AACzB,4CAA0B;AAC1B,gDAA8B"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.LineageRegistry = void 0;
|
|
37
|
+
const crypto = __importStar(require("crypto"));
|
|
38
|
+
const sha8 = (s) => crypto.createHash("sha1").update(s).digest("hex").slice(0, 8);
|
|
39
|
+
/**
|
|
40
|
+
* Run-scoped lineage index that lives **entirely outside** the knowledge graph —
|
|
41
|
+
* nothing here is ever written onto an `Entity`/`Observation`/`Relation` or any
|
|
42
|
+
* export, so the serialized graph is byte-identical whether tracing is on or off
|
|
43
|
+
* (the observe-only gate holds by construction).
|
|
44
|
+
*
|
|
45
|
+
* It mints deterministic, reconstructable IDs and tracks, per entity NAME, the
|
|
46
|
+
* list of pre-merge mention instances. As the merger folds one name into another
|
|
47
|
+
* (`fold`), the loser's mentions are reattributed to the winner — so a final
|
|
48
|
+
* canonical node's full set of contributing mentions is `mentionsFor(canonical)`.
|
|
49
|
+
*/
|
|
50
|
+
class LineageRegistry {
|
|
51
|
+
constructor() {
|
|
52
|
+
this.byName = new Map();
|
|
53
|
+
}
|
|
54
|
+
static entityMentionId(extractionId, name) {
|
|
55
|
+
return `${extractionId}|e|${name}`;
|
|
56
|
+
}
|
|
57
|
+
static observationId(extractionId, name, text) {
|
|
58
|
+
return `${extractionId}|o|${sha8(`${name}␟${text}`)}`;
|
|
59
|
+
}
|
|
60
|
+
static relationMentionId(extractionId, from, to) {
|
|
61
|
+
return `${extractionId}|r|${from}>${to}`;
|
|
62
|
+
}
|
|
63
|
+
/** Record one entity mention under its name; returns the minted ref. */
|
|
64
|
+
registerEntity(ref) {
|
|
65
|
+
const list = this.byName.get(ref.name);
|
|
66
|
+
if (list)
|
|
67
|
+
list.push(ref);
|
|
68
|
+
else
|
|
69
|
+
this.byName.set(ref.name, [ref]);
|
|
70
|
+
return ref;
|
|
71
|
+
}
|
|
72
|
+
/** Reassign the loser name's mentions onto the winner (a merge fusion). */
|
|
73
|
+
fold(loserName, winnerName) {
|
|
74
|
+
var _a, _b;
|
|
75
|
+
if (loserName === winnerName)
|
|
76
|
+
return [];
|
|
77
|
+
const losers = (_a = this.byName.get(loserName)) !== null && _a !== void 0 ? _a : [];
|
|
78
|
+
if (losers.length === 0)
|
|
79
|
+
return [];
|
|
80
|
+
const winners = (_b = this.byName.get(winnerName)) !== null && _b !== void 0 ? _b : [];
|
|
81
|
+
this.byName.set(winnerName, winners.concat(losers));
|
|
82
|
+
this.byName.delete(loserName);
|
|
83
|
+
return losers;
|
|
84
|
+
}
|
|
85
|
+
mentionsFor(name) {
|
|
86
|
+
var _a;
|
|
87
|
+
return (_a = this.byName.get(name)) !== null && _a !== void 0 ? _a : [];
|
|
88
|
+
}
|
|
89
|
+
mentionIdsFor(name) {
|
|
90
|
+
return this.mentionsFor(name).map((m) => m.mentionId);
|
|
91
|
+
}
|
|
92
|
+
reset() {
|
|
93
|
+
this.byName.clear();
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
exports.LineageRegistry = LineageRegistry;
|
|
97
|
+
//# sourceMappingURL=lineage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lineage.js","sourceRoot":"","sources":["../../../src/core/trace/lineage.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AAYjC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAE1F;;;;;;;;;;GAUG;AACH,MAAa,eAAe;IAA5B;QACU,WAAM,GAAG,IAAI,GAAG,EAAwB,CAAC;IA0CnD,CAAC;IAxCC,MAAM,CAAC,eAAe,CAAC,YAAoB,EAAE,IAAY;QACvD,OAAO,GAAG,YAAY,MAAM,IAAI,EAAE,CAAC;IACrC,CAAC;IACD,MAAM,CAAC,aAAa,CAAC,YAAoB,EAAE,IAAY,EAAE,IAAY;QACnE,OAAO,GAAG,YAAY,MAAM,IAAI,CAAC,GAAG,IAAI,IAAI,IAAI,EAAE,CAAC,EAAE,CAAC;IACxD,CAAC;IACD,MAAM,CAAC,iBAAiB,CAAC,YAAoB,EAAE,IAAY,EAAE,EAAU;QACrE,OAAO,GAAG,YAAY,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;IAC3C,CAAC;IAED,wEAAwE;IACxE,cAAc,CAAC,GAAe;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACpB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC;IACb,CAAC;IAED,2EAA2E;IAC3E,IAAI,CAAC,SAAiB,EAAE,UAAkB;;QACxC,IAAI,SAAS,KAAK,UAAU;YAAE,OAAO,EAAE,CAAC;QACxC,MAAM,MAAM,GAAG,MAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,mCAAI,EAAE,CAAC;QAChD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,MAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,mCAAI,EAAE,CAAC;QAClD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACpD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QAC9B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,WAAW,CAAC,IAAY;;QACtB,OAAO,MAAA,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAI,EAAE,CAAC;IACrC,CAAC;IAED,aAAa,CAAC,IAAY;QACxB,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IAED,KAAK;QACH,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;CACF;AA3CD,0CA2CC"}
|