@seanhogg/builderforce-memory 2026.7.14 → 2026.9.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/bridges/AnthropicBridge.d.ts +9 -1
- package/dist/bridges/AnthropicBridge.d.ts.map +1 -1
- package/dist/bridges/AnthropicBridge.js +58 -2
- package/dist/bridges/AnthropicBridge.js.map +1 -1
- package/dist/bridges/CachingBridge.d.ts +9 -1
- package/dist/bridges/CachingBridge.d.ts.map +1 -1
- package/dist/bridges/CachingBridge.js +24 -1
- package/dist/bridges/CachingBridge.js.map +1 -1
- package/dist/bridges/OpenAIBridge.d.ts +4 -1
- package/dist/bridges/OpenAIBridge.d.ts.map +1 -1
- package/dist/bridges/OpenAIBridge.js +40 -2
- package/dist/bridges/OpenAIBridge.js.map +1 -1
- package/dist/bridges/SemanticCachingBridge.d.ts +8 -1
- package/dist/bridges/SemanticCachingBridge.d.ts.map +1 -1
- package/dist/bridges/SemanticCachingBridge.js +26 -1
- package/dist/bridges/SemanticCachingBridge.js.map +1 -1
- package/dist/bridges/TransformerBridge.d.ts +13 -0
- package/dist/bridges/TransformerBridge.d.ts.map +1 -1
- package/dist/bridges/VertexAIBridge.d.ts +102 -0
- package/dist/bridges/VertexAIBridge.d.ts.map +1 -0
- package/dist/bridges/VertexAIBridge.js +224 -0
- package/dist/bridges/VertexAIBridge.js.map +1 -0
- package/dist/bridges/index.d.ts +3 -1
- package/dist/bridges/index.d.ts.map +1 -1
- package/dist/bridges/index.js +1 -0
- package/dist/bridges/index.js.map +1 -1
- package/dist/eval/EvalHarness.d.ts +37 -0
- package/dist/eval/EvalHarness.d.ts.map +1 -0
- package/dist/eval/EvalHarness.js +178 -0
- package/dist/eval/EvalHarness.js.map +1 -0
- package/dist/eval/graders.d.ts +76 -0
- package/dist/eval/graders.d.ts.map +1 -0
- package/dist/eval/graders.js +168 -0
- package/dist/eval/graders.js.map +1 -0
- package/dist/eval/index.d.ts +26 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +26 -0
- package/dist/eval/index.js.map +1 -0
- package/dist/eval/types.d.ts +97 -0
- package/dist/eval/types.d.ts.map +1 -0
- package/dist/eval/types.js +5 -0
- package/dist/eval/types.js.map +1 -0
- package/dist/evermind/contract.d.ts +40 -0
- package/dist/evermind/contract.d.ts.map +1 -0
- package/dist/evermind/contract.js +13 -0
- package/dist/evermind/contract.js.map +1 -0
- package/dist/evermind/index.d.ts +15 -0
- package/dist/evermind/index.d.ts.map +1 -0
- package/dist/evermind/index.js +4 -0
- package/dist/evermind/index.js.map +1 -0
- package/dist/evermind/memoryBlock.d.ts +7 -0
- package/dist/evermind/memoryBlock.d.ts.map +1 -0
- package/dist/evermind/memoryBlock.js +37 -0
- package/dist/evermind/memoryBlock.js.map +1 -0
- package/dist/evermind/recall.d.ts +48 -0
- package/dist/evermind/recall.d.ts.map +1 -0
- package/dist/evermind/recall.js +105 -0
- package/dist/evermind/recall.js.map +1 -0
- package/dist/index.d.ts +18 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -1
- package/dist/index.js.map +1 -1
- package/dist/ingest/IngestionPipeline.d.ts +95 -0
- package/dist/ingest/IngestionPipeline.d.ts.map +1 -0
- package/dist/ingest/IngestionPipeline.js +295 -0
- package/dist/ingest/IngestionPipeline.js.map +1 -0
- package/dist/ingest/index.d.ts +10 -0
- package/dist/ingest/index.d.ts.map +1 -0
- package/dist/ingest/index.js +8 -0
- package/dist/ingest/index.js.map +1 -0
- package/dist/ingest/parsers.d.ts +53 -0
- package/dist/ingest/parsers.d.ts.map +1 -0
- package/dist/ingest/parsers.js +306 -0
- package/dist/ingest/parsers.js.map +1 -0
- package/dist/ingest/types.d.ts +114 -0
- package/dist/ingest/types.d.ts.map +1 -0
- package/dist/ingest/types.js +19 -0
- package/dist/ingest/types.js.map +1 -0
- package/dist/memory/MemoryStore.d.ts +33 -0
- package/dist/memory/MemoryStore.d.ts.map +1 -1
- package/dist/memory/MemoryStore.js +29 -3
- package/dist/memory/MemoryStore.js.map +1 -1
- package/dist/memory/index.d.ts +1 -1
- package/dist/memory/index.d.ts.map +1 -1
- package/dist/orchestration/AgentGraph.d.ts +104 -0
- package/dist/orchestration/AgentGraph.d.ts.map +1 -0
- package/dist/orchestration/AgentGraph.js +364 -0
- package/dist/orchestration/AgentGraph.js.map +1 -0
- package/dist/orchestration/checkpoint.d.ts +58 -0
- package/dist/orchestration/checkpoint.d.ts.map +1 -0
- package/dist/orchestration/checkpoint.js +92 -0
- package/dist/orchestration/checkpoint.js.map +1 -0
- package/dist/orchestration/index.d.ts +14 -0
- package/dist/orchestration/index.d.ts.map +1 -0
- package/dist/orchestration/index.js +10 -0
- package/dist/orchestration/index.js.map +1 -0
- package/dist/orchestration/patterns.d.ts +159 -0
- package/dist/orchestration/patterns.d.ts.map +1 -0
- package/dist/orchestration/patterns.js +371 -0
- package/dist/orchestration/patterns.js.map +1 -0
- package/dist/orchestration/types.d.ts +146 -0
- package/dist/orchestration/types.d.ts.map +1 -0
- package/dist/orchestration/types.js +20 -0
- package/dist/orchestration/types.js.map +1 -0
- package/dist/rag/EnterpriseRetriever.d.ts +114 -0
- package/dist/rag/EnterpriseRetriever.d.ts.map +1 -0
- package/dist/rag/EnterpriseRetriever.js +240 -0
- package/dist/rag/EnterpriseRetriever.js.map +1 -0
- package/dist/rag/index.d.ts +6 -0
- package/dist/rag/index.d.ts.map +1 -0
- package/dist/rag/index.js +5 -0
- package/dist/rag/index.js.map +1 -0
- package/dist/retrieval/bm25.d.ts +48 -1
- package/dist/retrieval/bm25.d.ts.map +1 -1
- package/dist/retrieval/bm25.js +59 -13
- package/dist/retrieval/bm25.js.map +1 -1
- package/dist/retrieval/index.d.ts +2 -2
- package/dist/retrieval/index.d.ts.map +1 -1
- package/dist/retrieval/index.js +1 -1
- package/dist/retrieval/index.js.map +1 -1
- package/dist/runtime/SSMRuntime.d.ts +1 -1
- package/dist/runtime/SSMRuntime.d.ts.map +1 -1
- package/dist/telemetry/InstrumentedBridge.d.ts +62 -0
- package/dist/telemetry/InstrumentedBridge.d.ts.map +1 -0
- package/dist/telemetry/InstrumentedBridge.js +120 -0
- package/dist/telemetry/InstrumentedBridge.js.map +1 -0
- package/dist/telemetry/MetricsRegistry.d.ts +88 -0
- package/dist/telemetry/MetricsRegistry.d.ts.map +1 -0
- package/dist/telemetry/MetricsRegistry.js +167 -0
- package/dist/telemetry/MetricsRegistry.js.map +1 -0
- package/dist/telemetry/Tracer.d.ts +137 -0
- package/dist/telemetry/Tracer.d.ts.map +1 -0
- package/dist/telemetry/Tracer.js +244 -0
- package/dist/telemetry/Tracer.js.map +1 -0
- package/dist/telemetry/exporters.d.ts +68 -0
- package/dist/telemetry/exporters.d.ts.map +1 -0
- package/dist/telemetry/exporters.js +145 -0
- package/dist/telemetry/exporters.js.map +1 -0
- package/dist/telemetry/index.d.ts +19 -0
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/index.js +12 -0
- package/dist/telemetry/index.js.map +1 -0
- package/dist/telemetry/pricing.d.ts +74 -0
- package/dist/telemetry/pricing.d.ts.map +1 -0
- package/dist/telemetry/pricing.js +98 -0
- package/dist/telemetry/pricing.js.map +1 -0
- package/dist/telemetry/traceparent.d.ts +50 -0
- package/dist/telemetry/traceparent.d.ts.map +1 -0
- package/dist/telemetry/traceparent.js +77 -0
- package/dist/telemetry/traceparent.js.map +1 -0
- package/dist/telemetry/types.d.ts +90 -0
- package/dist/telemetry/types.d.ts.map +1 -0
- package/dist/telemetry/types.js +17 -0
- package/dist/telemetry/types.js.map +1 -0
- package/dist/vectorstore/MemoryVectorStore.d.ts +72 -0
- package/dist/vectorstore/MemoryVectorStore.d.ts.map +1 -0
- package/dist/vectorstore/MemoryVectorStore.js +205 -0
- package/dist/vectorstore/MemoryVectorStore.js.map +1 -0
- package/dist/vectorstore/RestVectorStore.d.ts +79 -0
- package/dist/vectorstore/RestVectorStore.d.ts.map +1 -0
- package/dist/vectorstore/RestVectorStore.js +170 -0
- package/dist/vectorstore/RestVectorStore.js.map +1 -0
- package/dist/vectorstore/dialects.d.ts +101 -0
- package/dist/vectorstore/dialects.d.ts.map +1 -0
- package/dist/vectorstore/dialects.js +618 -0
- package/dist/vectorstore/dialects.js.map +1 -0
- package/dist/vectorstore/filter.d.ts +42 -0
- package/dist/vectorstore/filter.d.ts.map +1 -0
- package/dist/vectorstore/filter.js +132 -0
- package/dist/vectorstore/filter.js.map +1 -0
- package/dist/vectorstore/index.d.ts +15 -0
- package/dist/vectorstore/index.d.ts.map +1 -0
- package/dist/vectorstore/index.js +11 -0
- package/dist/vectorstore/index.js.map +1 -0
- package/dist/vectorstore/types.d.ts +151 -0
- package/dist/vectorstore/types.d.ts.map +1 -0
- package/dist/vectorstore/types.js +30 -0
- package/dist/vectorstore/types.js.map +1 -0
- package/package.json +39 -5
- package/src/bridges/AnthropicBridge.ts +68 -3
- package/src/bridges/CachingBridge.ts +28 -2
- package/src/bridges/OpenAIBridge.ts +48 -3
- package/src/bridges/SemanticCachingBridge.ts +29 -2
- package/src/bridges/TransformerBridge.ts +16 -0
- package/src/bridges/VertexAIBridge.ts +297 -0
- package/src/bridges/index.ts +3 -1
- package/src/eval/EvalHarness.ts +214 -0
- package/src/eval/graders.ts +218 -0
- package/src/eval/index.ts +26 -0
- package/src/eval/types.ts +107 -0
- package/src/evermind/contract.ts +39 -0
- package/src/evermind/index.ts +18 -0
- package/src/evermind/memoryBlock.ts +38 -0
- package/src/evermind/recall.ts +122 -0
- package/src/index.ts +232 -0
- package/src/ingest/IngestionPipeline.ts +378 -0
- package/src/ingest/index.ts +46 -0
- package/src/ingest/parsers.ts +318 -0
- package/src/ingest/types.ts +116 -0
- package/src/memory/MemoryStore.ts +55 -3
- package/src/memory/index.ts +1 -1
- package/src/orchestration/AgentGraph.ts +462 -0
- package/src/orchestration/checkpoint.ts +129 -0
- package/src/orchestration/index.ts +63 -0
- package/src/orchestration/patterns.ts +532 -0
- package/src/orchestration/types.ts +128 -0
- package/src/rag/EnterpriseRetriever.ts +348 -0
- package/src/rag/index.ts +18 -0
- package/src/retrieval/bm25.ts +71 -13
- package/src/retrieval/index.ts +2 -2
- package/src/telemetry/InstrumentedBridge.ts +152 -0
- package/src/telemetry/MetricsRegistry.ts +263 -0
- package/src/telemetry/Tracer.ts +310 -0
- package/src/telemetry/exporters.ts +187 -0
- package/src/telemetry/index.ts +56 -0
- package/src/telemetry/pricing.ts +129 -0
- package/src/telemetry/traceparent.ts +89 -0
- package/src/telemetry/types.ts +105 -0
- package/src/vectorstore/MemoryVectorStore.ts +250 -0
- package/src/vectorstore/RestVectorStore.ts +236 -0
- package/src/vectorstore/dialects.ts +702 -0
- package/src/vectorstore/filter.ts +148 -0
- package/src/vectorstore/index.ts +53 -0
- package/src/vectorstore/types.ts +149 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"IngestionPipeline.js","sourceRoot":"","sources":["../../src/ingest/IngestionPipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,SAAS,EAAqB,MAAM,uBAAuB,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EACH,SAAS,EACT,UAAU,EACV,iBAAiB,EACjB,YAAY,GAIf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EACH,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,eAAe,EACf,WAAW,EACX,SAAS,GAQZ,MAAM,YAAY,CAAC;AAwBpB,MAAM,OAAO,iBAAiB;IACT,MAAM,CAAc;IACpB,MAAM,CAAgB;IACtB,UAAU,CAAe;IACzB,QAAQ,CAAiB;IACzB,SAAS,CAA6B;IACtC,SAAS,CAAe;IACxB,WAAW,CAAS;IACpB,YAAY,CAAS;IACrB,YAAY,CAAS;IACrB,eAAe,CAAS;IACxB,OAAO,CAAqB;IAE7C,YAAY,IAA8B;QACtC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,YAAY,cAAc;YAClD,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,eAAe,IAAI,GAAG,CAAC,CAAC;QAC7D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,cAAc,IAAI,WAAW,CAAC;QAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAkC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAoB;YAC5B,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC9D,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;SACpE,CAAC;QAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,YAAY,EAAE;YAC/C,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,SAAS,EAAE;SACtF,CAAC,CAAC;QACH,IAAI,IAAI;YAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAExC,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAClE,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACnD,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;gBACtB,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC/B,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,gBAAgB,CAAC;gBACnD,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;YAC/C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,oEAAoE;gBACpE,+DAA+D;gBAC/D,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACjB,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACnB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1D,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACzC,IAAI,EAAE,aAAa,CAAC;YAChB,eAAe,EAAE,MAAM,CAAC,MAAM;YAC9B,iBAAiB,EAAE,MAAM,CAAC,QAAQ;YAClC,0BAA0B,EAAE,MAAM,CAAC,gBAAgB;YACnD,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;SAC5C,CAAC,CAAC;QACH,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjF,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,MAAM,CAAC,QAAgB;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;YAC7C,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ;SACpD,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvC,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,UAAU,CACpB,MAAsB,EACtB,MAA8C;QAK9C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,iBAAiB,EAAE;YACpD,IAAI,EAAE,QAAQ;YACd,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,UAAU,EAAE,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE;SAClF,CAAC,CAAC;QAEH,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YAEzE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YAEnD,sEAAsE;YACtE,4DAA4D;YAC5D,MAAM,OAAO,GAAkD,EAAE,CAAC;YAClE,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,QAAQ,GAAmB;wBAC7B,GAAG,IAAI;wBACP,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;wBAC3B,CAAC,iBAAiB,CAAC,EAAE,KAAK;wBAC1B,CAAC,kBAAkB,CAAC,EAAE,IAAI;qBAC7B,CAAC;oBACF,OAAO,CAAC,IAAI,CAAC;wBACT,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,EAAE,IAAI,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE;wBACnE,IAAI;qBACP,CAAC,CAAC;oBACH,KAAK,IAAI,CAAC,CAAC;gBACf,CAAC;YACL,CAAC;YAED,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAEhF,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC7E,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACxB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;oBAC1B,IAAI,MAAM;wBAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvC,CAAC,CAAC,CAAC;gBACH,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;YAC7B,CAAC;YAED,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACzD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;gBACjF,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;YAChC,CAAC;YAED,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACnC,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC;YACnC,CAAC;YAED,MAAM,IAAI,CAAC,SAAS,EAAE,GAAG,CACrB,MAAM,CAAC,EAAE,EACT,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAoB,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CACjF,CAAC;YAEF,IAAI,EAAE,aAAa,CAAC;gBAChB,iBAAiB,EAAE,QAAQ,CAAC,MAAM;gBAClC,eAAe,EAAE,OAAO,CAAC,MAAM;gBAC/B,iBAAiB,EAAE,QAAQ;gBAC3B,0BAA0B,EAAE,OAAO;gBACnC,eAAe,EAAE,MAAM,CAAC,EAAE;aAC7B,CAAC,CAAC;YACH,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAEhB,OAAO;gBACH,QAAQ,EAAE,QAAQ,CAAC,MAAM;gBACzB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,QAAQ;gBACR,QAAQ;gBACR,gBAAgB,EAAE,OAAO;gBACzB,YAAY;aACf,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,SAAS,CACnB,QAAgB,EAChB,OAAsD;QAEtD,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5D,MAAM,OAAO,GAAmB,EAAE,CAAC;YACnC,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,CAAC;gBACrC,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI;oBAAE,OAAO,IAAI,CAAC,CAAC;;oBACpD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5F,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC3B,KAAe,EACf,MAA8C;QAE9C,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE7C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,EAAE;YAC/C,IAAI,EAAE,WAAW;YACjB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,UAAU,EAAE,EAAE,sBAAsB,EAAE,KAAK,CAAC,MAAM,EAAE;SACvD,CAAC,CAAC;QACH,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,CAAC,WAAW,CAAC;gBACb,KAAK,EAAE,IAAI,CAAC,eAAe;gBAC3B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACjE,YAAY,EAAE,CAAC;gBACf,SAAS,EAAE,IAAI;aAClB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,OAAO,OAAO,CAAC;QACnB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAEO,aAAa,CAAC,MAAsB,EAAE,QAAgB;QAC1D,MAAM,QAAQ,GAAmB;YAC7B,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;YAC1B,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,QAAQ;YAC/B,sEAAsE;YACtE,sEAAsE;YACtE,kCAAkC;YAClC,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YAC3D,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC,EAAE;YAC5B,CAAC,YAAY,CAAC,EAAE,QAAQ;SAC3B,CAAC;QACF,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS;YAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QACvF,IAAI,MAAM,CAAC,KAAK;YAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;QACvD,IAAI,MAAM,CAAC,GAAG;YAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;QACjD,OAAO,QAAQ,CAAC;IACpB,CAAC;CACJ;AAED,iFAAiF;AACjF,MAAM,OAAO,sBAAsB;IACd,QAAQ,GAAG,IAAI,GAAG,EAA8B,CAAC;IAElE,KAAK,CAAC,GAAG,CAAC,QAAgB;QACtB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,QAAgB,EAAE,YAAgC;QACxD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB;QACzB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,IAAI,KAAa,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;CACpD;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACjC,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACpC,KAAmB,EACnB,KAAa,EACb,EAA6C;IAE7C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC7E,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;YACvB,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAM,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest — one pipeline for structured and unstructured sources, with incremental
|
|
3
|
+
* sync so a nightly re-run re-embeds only what changed.
|
|
4
|
+
*/
|
|
5
|
+
export { IngestionPipeline, InMemoryIngestManifest, hashText, mapWithConcurrency, } from './IngestionPipeline.js';
|
|
6
|
+
export type { IngestionPipelineOptions } from './IngestionPipeline.js';
|
|
7
|
+
export { ParserRegistry, BUILTIN_PARSERS, rowsParser, csvParser, jsonParser, markdownParser, htmlParser, plainTextParser, parseCsv, serializeRow, } from './parsers.js';
|
|
8
|
+
export { SOURCE_ID_FIELD, CHUNK_INDEX_FIELD, CONTENT_HASH_FIELD, TITLE_FIELD, URI_FIELD, PARSER_FIELD, } from './types.js';
|
|
9
|
+
export type { BatchEmbedder, ChunkFingerprint, DocumentParser, IngestManifest, IngestionReport, ParsedSection, SourceContent, SourceDocument, StructuredRow, SyncStrategy, } from './types.js';
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ingest/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACH,iBAAiB,EACjB,sBAAsB,EACtB,QAAQ,EACR,kBAAkB,GACrB,MAAM,wBAAwB,CAAC;AAChC,YAAY,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,EACH,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,EACT,UAAU,EACV,cAAc,EACd,UAAU,EACV,eAAe,EACf,QAAQ,EACR,YAAY,GACf,MAAM,cAAc,CAAC;AAEtB,OAAO,EACH,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,YAAY,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACR,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,eAAe,EACf,aAAa,EACb,aAAa,EACb,cAAc,EACd,aAAa,EACb,YAAY,GACf,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest — one pipeline for structured and unstructured sources, with incremental
|
|
3
|
+
* sync so a nightly re-run re-embeds only what changed.
|
|
4
|
+
*/
|
|
5
|
+
export { IngestionPipeline, InMemoryIngestManifest, hashText, mapWithConcurrency, } from './IngestionPipeline.js';
|
|
6
|
+
export { ParserRegistry, BUILTIN_PARSERS, rowsParser, csvParser, jsonParser, markdownParser, htmlParser, plainTextParser, parseCsv, serializeRow, } from './parsers.js';
|
|
7
|
+
export { SOURCE_ID_FIELD, CHUNK_INDEX_FIELD, CONTENT_HASH_FIELD, TITLE_FIELD, URI_FIELD, PARSER_FIELD, } from './types.js';
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ingest/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACH,iBAAiB,EACjB,sBAAsB,EACtB,QAAQ,EACR,kBAAkB,GACrB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACH,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,EACT,UAAU,EACV,cAAc,EACd,UAAU,EACV,eAAe,EACf,QAAQ,EACR,YAAY,GACf,MAAM,cAAc,CAAC;AAEtB,OAAO,EACH,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,YAAY,GACf,MAAM,YAAY,CAAC"}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest/parsers.ts — format handlers, as a registry.
|
|
3
|
+
*
|
|
4
|
+
* Each parser turns a source into sections and recovers whatever STRUCTURE the
|
|
5
|
+
* format carries, because structure that survives parsing becomes filterable
|
|
6
|
+
* metadata, and metadata is what makes retrieval precise:
|
|
7
|
+
*
|
|
8
|
+
* • Markdown keeps its heading path, so a chunk knows it lives under
|
|
9
|
+
* "Security › Data retention" and a citation can say so.
|
|
10
|
+
* • Rows keep every typed column, so `status = 'open'` is a database predicate
|
|
11
|
+
* rather than something the embedding has to imply.
|
|
12
|
+
* • JSON keeps its pointer, so a value can be traced back to its exact location.
|
|
13
|
+
*
|
|
14
|
+
* A parser that returned only text would throw all of that away and force the
|
|
15
|
+
* retriever to recover it statistically — the single most common reason a RAG
|
|
16
|
+
* pilot answers plausibly and cites nothing verifiable.
|
|
17
|
+
*/
|
|
18
|
+
import type { DocumentParser, SourceDocument, StructuredRow } from './types.js';
|
|
19
|
+
/**
|
|
20
|
+
* Serialises a row as `field: value` lines.
|
|
21
|
+
*
|
|
22
|
+
* Field names are kept in the text on purpose: an embedding of `"status: open"`
|
|
23
|
+
* sits near a question about open items, whereas a bare `"open"` sits near
|
|
24
|
+
* nothing useful. The typed values are ALSO emitted as metadata, so the same fact
|
|
25
|
+
* is both semantically searchable and exactly filterable.
|
|
26
|
+
*/
|
|
27
|
+
export declare function serializeRow(row: StructuredRow): string;
|
|
28
|
+
export declare const rowsParser: DocumentParser;
|
|
29
|
+
/**
|
|
30
|
+
* RFC 4180 subset: quoted fields, doubled quotes, embedded newlines and commas.
|
|
31
|
+
* Enough for the exports enterprises actually hand over, without a dependency.
|
|
32
|
+
*/
|
|
33
|
+
export declare function parseCsv(text: string, delimiter?: string): StructuredRow[];
|
|
34
|
+
export declare const csvParser: DocumentParser;
|
|
35
|
+
/**
|
|
36
|
+
* An array of objects is treated as rows (the common export shape); anything else
|
|
37
|
+
* is flattened to leaf paths so a value keeps its JSON pointer.
|
|
38
|
+
*/
|
|
39
|
+
export declare const jsonParser: DocumentParser;
|
|
40
|
+
export declare const markdownParser: DocumentParser;
|
|
41
|
+
export declare const htmlParser: DocumentParser;
|
|
42
|
+
export declare const plainTextParser: DocumentParser;
|
|
43
|
+
/** Order matters: the first parser that accepts a source handles it. */
|
|
44
|
+
export declare const BUILTIN_PARSERS: readonly DocumentParser[];
|
|
45
|
+
export declare class ParserRegistry {
|
|
46
|
+
private readonly _parsers;
|
|
47
|
+
constructor(parsers?: readonly DocumentParser[]);
|
|
48
|
+
/** Registers a parser at the FRONT, so it can override a builtin. */
|
|
49
|
+
register(parser: DocumentParser): this;
|
|
50
|
+
resolve(source: SourceDocument): DocumentParser | undefined;
|
|
51
|
+
list(): string[];
|
|
52
|
+
}
|
|
53
|
+
//# sourceMappingURL=parsers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parsers.d.ts","sourceRoot":"","sources":["../../src/ingest/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAiB,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAU/F;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,aAAa,GAAG,MAAM,CAKvD;AAED,eAAO,MAAM,UAAU,EAAE,cAexB,CAAC;AAIF;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,aAAa,EAAE,CA4CvE;AAmBD,eAAO,MAAM,SAAS,EAAE,cAWvB,CAAC;AAIF;;;GAGG;AACH,eAAO,MAAM,UAAU,EAAE,cA2BxB,CAAC;AAmBF,eAAO,MAAM,cAAc,EAAE,cA4C5B,CAAC;AAIF,eAAO,MAAM,UAAU,EAAE,cA6BxB,CAAC;AAIF,eAAO,MAAM,eAAe,EAAE,cAO7B,CAAC;AAIF,wEAAwE;AACxE,eAAO,MAAM,eAAe,EAAE,SAAS,cAAc,EAOpD,CAAC;AAEF,qBAAa,cAAc;IACvB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmB;gBAEhC,OAAO,GAAE,SAAS,cAAc,EAAoB;IAIhE,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI;IAKtC,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,GAAG,SAAS;IAI3D,IAAI,IAAI,MAAM,EAAE;CAGnB"}
|
|
@@ -0,0 +1,306 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest/parsers.ts — format handlers, as a registry.
|
|
3
|
+
*
|
|
4
|
+
* Each parser turns a source into sections and recovers whatever STRUCTURE the
|
|
5
|
+
* format carries, because structure that survives parsing becomes filterable
|
|
6
|
+
* metadata, and metadata is what makes retrieval precise:
|
|
7
|
+
*
|
|
8
|
+
* • Markdown keeps its heading path, so a chunk knows it lives under
|
|
9
|
+
* "Security › Data retention" and a citation can say so.
|
|
10
|
+
* • Rows keep every typed column, so `status = 'open'` is a database predicate
|
|
11
|
+
* rather than something the embedding has to imply.
|
|
12
|
+
* • JSON keeps its pointer, so a value can be traced back to its exact location.
|
|
13
|
+
*
|
|
14
|
+
* A parser that returned only text would throw all of that away and force the
|
|
15
|
+
* retriever to recover it statistically — the single most common reason a RAG
|
|
16
|
+
* pilot answers plausibly and cites nothing verifiable.
|
|
17
|
+
*/
|
|
18
|
+
const textOf = (source) => source.content.kind === 'text' ? source.content.text : '';
|
|
19
|
+
const mediaTypeOf = (source) => source.content.kind === 'text' ? (source.content.mediaType ?? 'text/plain').toLowerCase() : '';
|
|
20
|
+
// ── structured rows ───────────────────────────────────────────────────────────
|
|
21
|
+
/**
|
|
22
|
+
* Serialises a row as `field: value` lines.
|
|
23
|
+
*
|
|
24
|
+
* Field names are kept in the text on purpose: an embedding of `"status: open"`
|
|
25
|
+
* sits near a question about open items, whereas a bare `"open"` sits near
|
|
26
|
+
* nothing useful. The typed values are ALSO emitted as metadata, so the same fact
|
|
27
|
+
* is both semantically searchable and exactly filterable.
|
|
28
|
+
*/
|
|
29
|
+
export function serializeRow(row) {
|
|
30
|
+
return Object.entries(row)
|
|
31
|
+
.filter(([, v]) => v !== null && v !== undefined && v !== '')
|
|
32
|
+
.map(([k, v]) => `${k}: ${Array.isArray(v) ? v.join(', ') : String(v)}`)
|
|
33
|
+
.join('\n');
|
|
34
|
+
}
|
|
35
|
+
export const rowsParser = {
|
|
36
|
+
id: 'rows',
|
|
37
|
+
accepts: (source) => source.content.kind === 'rows',
|
|
38
|
+
parse: (source) => {
|
|
39
|
+
if (source.content.kind !== 'rows')
|
|
40
|
+
return [];
|
|
41
|
+
const { rows, rowIdField } = source.content;
|
|
42
|
+
return rows.map((row, index) => {
|
|
43
|
+
const metadata = { ...row };
|
|
44
|
+
metadata['rowIndex'] = index;
|
|
45
|
+
if (rowIdField && row[rowIdField] !== undefined) {
|
|
46
|
+
metadata['rowId'] = String(row[rowIdField]);
|
|
47
|
+
}
|
|
48
|
+
return { text: serializeRow(row), metadata };
|
|
49
|
+
});
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
// ── CSV ───────────────────────────────────────────────────────────────────────
|
|
53
|
+
/**
|
|
54
|
+
* RFC 4180 subset: quoted fields, doubled quotes, embedded newlines and commas.
|
|
55
|
+
* Enough for the exports enterprises actually hand over, without a dependency.
|
|
56
|
+
*/
|
|
57
|
+
export function parseCsv(text, delimiter = ',') {
|
|
58
|
+
const rows = [];
|
|
59
|
+
let field = '';
|
|
60
|
+
let record = [];
|
|
61
|
+
let quoted = false;
|
|
62
|
+
for (let i = 0; i < text.length; i++) {
|
|
63
|
+
const ch = text[i];
|
|
64
|
+
if (quoted) {
|
|
65
|
+
if (ch === '"') {
|
|
66
|
+
if (text[i + 1] === '"') {
|
|
67
|
+
field += '"';
|
|
68
|
+
i++;
|
|
69
|
+
}
|
|
70
|
+
else
|
|
71
|
+
quoted = false;
|
|
72
|
+
}
|
|
73
|
+
else
|
|
74
|
+
field += ch;
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
if (ch === '"') {
|
|
78
|
+
quoted = true;
|
|
79
|
+
continue;
|
|
80
|
+
}
|
|
81
|
+
if (ch === delimiter) {
|
|
82
|
+
record.push(field);
|
|
83
|
+
field = '';
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
if (ch === '\n' || ch === '\r') {
|
|
87
|
+
if (ch === '\r' && text[i + 1] === '\n')
|
|
88
|
+
i++;
|
|
89
|
+
record.push(field);
|
|
90
|
+
rows.push(record);
|
|
91
|
+
record = [];
|
|
92
|
+
field = '';
|
|
93
|
+
continue;
|
|
94
|
+
}
|
|
95
|
+
field += ch;
|
|
96
|
+
}
|
|
97
|
+
if (field !== '' || record.length > 0) {
|
|
98
|
+
record.push(field);
|
|
99
|
+
rows.push(record);
|
|
100
|
+
}
|
|
101
|
+
const header = rows.shift();
|
|
102
|
+
if (!header)
|
|
103
|
+
return [];
|
|
104
|
+
return rows
|
|
105
|
+
.filter((r) => r.some((cell) => cell.trim() !== ''))
|
|
106
|
+
.map((cells) => {
|
|
107
|
+
const row = {};
|
|
108
|
+
header.forEach((name, i) => {
|
|
109
|
+
row[name.trim()] = coerce(cells[i] ?? '');
|
|
110
|
+
});
|
|
111
|
+
return row;
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Numbers and booleans are recovered from CSV text so range filters work.
|
|
116
|
+
* A value that merely *looks* numeric but has a leading zero (an account number,
|
|
117
|
+
* a zip code) stays a string — coercing it would silently destroy the identifier.
|
|
118
|
+
*/
|
|
119
|
+
function coerce(raw) {
|
|
120
|
+
const value = raw.trim();
|
|
121
|
+
if (value === '')
|
|
122
|
+
return '';
|
|
123
|
+
if (value === 'true')
|
|
124
|
+
return true;
|
|
125
|
+
if (value === 'false')
|
|
126
|
+
return false;
|
|
127
|
+
if (/^-?(0|[1-9]\d*)(\.\d+)?$/.test(value)) {
|
|
128
|
+
const n = Number(value);
|
|
129
|
+
if (Number.isFinite(n))
|
|
130
|
+
return n;
|
|
131
|
+
}
|
|
132
|
+
return value;
|
|
133
|
+
}
|
|
134
|
+
export const csvParser = {
|
|
135
|
+
id: 'csv',
|
|
136
|
+
accepts: (source) => {
|
|
137
|
+
const type = mediaTypeOf(source);
|
|
138
|
+
return type.includes('csv') || type.includes('tab-separated');
|
|
139
|
+
},
|
|
140
|
+
parse: (source) => {
|
|
141
|
+
const delimiter = mediaTypeOf(source).includes('tab-separated') ? '\t' : ',';
|
|
142
|
+
const rows = parseCsv(textOf(source), delimiter);
|
|
143
|
+
return rowsParser.parse({ ...source, content: { kind: 'rows', rows } });
|
|
144
|
+
},
|
|
145
|
+
};
|
|
146
|
+
// ── JSON ──────────────────────────────────────────────────────────────────────
|
|
147
|
+
/**
|
|
148
|
+
* An array of objects is treated as rows (the common export shape); anything else
|
|
149
|
+
* is flattened to leaf paths so a value keeps its JSON pointer.
|
|
150
|
+
*/
|
|
151
|
+
export const jsonParser = {
|
|
152
|
+
id: 'json',
|
|
153
|
+
accepts: (source) => mediaTypeOf(source).includes('json'),
|
|
154
|
+
parse: (source) => {
|
|
155
|
+
let parsed;
|
|
156
|
+
try {
|
|
157
|
+
parsed = JSON.parse(textOf(source));
|
|
158
|
+
}
|
|
159
|
+
catch {
|
|
160
|
+
// Malformed JSON is still text worth indexing; degrading beats dropping
|
|
161
|
+
// the document, and the parser name on the chunk records what happened.
|
|
162
|
+
return plainTextParser.parse(source);
|
|
163
|
+
}
|
|
164
|
+
if (Array.isArray(parsed) && parsed.every((v) => v && typeof v === 'object' && !Array.isArray(v))) {
|
|
165
|
+
return rowsParser.parse({
|
|
166
|
+
...source,
|
|
167
|
+
content: { kind: 'rows', rows: parsed },
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
const leaves = flatten(parsed, '');
|
|
171
|
+
if (leaves.length === 0)
|
|
172
|
+
return [];
|
|
173
|
+
return [{
|
|
174
|
+
text: leaves.map(([path, value]) => `${path}: ${value}`).join('\n'),
|
|
175
|
+
metadata: { leafCount: leaves.length },
|
|
176
|
+
}];
|
|
177
|
+
},
|
|
178
|
+
};
|
|
179
|
+
function flatten(value, path, out = []) {
|
|
180
|
+
if (value === null || typeof value !== 'object') {
|
|
181
|
+
out.push([path || '/', String(value)]);
|
|
182
|
+
return out;
|
|
183
|
+
}
|
|
184
|
+
if (Array.isArray(value)) {
|
|
185
|
+
value.forEach((v, i) => flatten(v, `${path}/${i}`, out));
|
|
186
|
+
return out;
|
|
187
|
+
}
|
|
188
|
+
for (const [key, v] of Object.entries(value)) {
|
|
189
|
+
flatten(v, `${path}/${key}`, out);
|
|
190
|
+
}
|
|
191
|
+
return out;
|
|
192
|
+
}
|
|
193
|
+
// ── Markdown ──────────────────────────────────────────────────────────────────
|
|
194
|
+
export const markdownParser = {
|
|
195
|
+
id: 'markdown',
|
|
196
|
+
accepts: (source) => {
|
|
197
|
+
const type = mediaTypeOf(source);
|
|
198
|
+
return type.includes('markdown') || type.includes('/md');
|
|
199
|
+
},
|
|
200
|
+
parse: (source) => {
|
|
201
|
+
const lines = textOf(source).split('\n');
|
|
202
|
+
const sections = [];
|
|
203
|
+
// Heading level -> text, so a chunk carries its full breadcrumb rather than
|
|
204
|
+
// only its nearest heading.
|
|
205
|
+
const path = [];
|
|
206
|
+
let buffer = [];
|
|
207
|
+
const flush = () => {
|
|
208
|
+
const text = buffer.join('\n').trim();
|
|
209
|
+
buffer = [];
|
|
210
|
+
if (!text)
|
|
211
|
+
return;
|
|
212
|
+
const metadata = {};
|
|
213
|
+
if (path.length) {
|
|
214
|
+
metadata['headingPath'] = path.join(' › ');
|
|
215
|
+
metadata['heading'] = path[path.length - 1];
|
|
216
|
+
metadata['headingDepth'] = path.length;
|
|
217
|
+
}
|
|
218
|
+
sections.push({ text, metadata });
|
|
219
|
+
};
|
|
220
|
+
let inFence = false;
|
|
221
|
+
for (const line of lines) {
|
|
222
|
+
if (/^\s*```/.test(line))
|
|
223
|
+
inFence = !inFence;
|
|
224
|
+
const heading = inFence ? null : /^(#{1,6})\s+(.*)$/.exec(line);
|
|
225
|
+
if (heading) {
|
|
226
|
+
flush();
|
|
227
|
+
const depth = heading[1].length;
|
|
228
|
+
path.length = Math.min(path.length, depth - 1);
|
|
229
|
+
path[depth - 1] = heading[2].trim();
|
|
230
|
+
path.length = depth;
|
|
231
|
+
continue;
|
|
232
|
+
}
|
|
233
|
+
buffer.push(line);
|
|
234
|
+
}
|
|
235
|
+
flush();
|
|
236
|
+
return sections;
|
|
237
|
+
},
|
|
238
|
+
};
|
|
239
|
+
// ── HTML ──────────────────────────────────────────────────────────────────────
|
|
240
|
+
export const htmlParser = {
|
|
241
|
+
id: 'html',
|
|
242
|
+
accepts: (source) => mediaTypeOf(source).includes('html') || mediaTypeOf(source).includes('xml'),
|
|
243
|
+
parse: (source) => {
|
|
244
|
+
const raw = textOf(source);
|
|
245
|
+
const titleMatch = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(raw);
|
|
246
|
+
const text = raw
|
|
247
|
+
// script/style content is code, not prose — indexing it poisons recall.
|
|
248
|
+
.replace(/<(script|style)[^>]*>[\s\S]*?<\/\1>/gi, ' ')
|
|
249
|
+
.replace(/<!--[\s\S]*?-->/g, ' ')
|
|
250
|
+
.replace(/<\/(p|div|section|article|li|tr|h[1-6])>/gi, '\n')
|
|
251
|
+
.replace(/<br\s*\/?>/gi, '\n')
|
|
252
|
+
.replace(/<[^>]+>/g, ' ')
|
|
253
|
+
.replace(/ /g, ' ')
|
|
254
|
+
.replace(/&/g, '&')
|
|
255
|
+
.replace(/</g, '<')
|
|
256
|
+
.replace(/>/g, '>')
|
|
257
|
+
.replace(/"/g, '"')
|
|
258
|
+
.replace(/'/g, "'")
|
|
259
|
+
.replace(/[ \t]+/g, ' ')
|
|
260
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
261
|
+
.trim();
|
|
262
|
+
if (!text)
|
|
263
|
+
return [];
|
|
264
|
+
const metadata = {};
|
|
265
|
+
if (titleMatch?.[1])
|
|
266
|
+
metadata['htmlTitle'] = titleMatch[1].trim();
|
|
267
|
+
return [{ text, metadata }];
|
|
268
|
+
},
|
|
269
|
+
};
|
|
270
|
+
// ── plain text ────────────────────────────────────────────────────────────────
|
|
271
|
+
export const plainTextParser = {
|
|
272
|
+
id: 'text',
|
|
273
|
+
accepts: (source) => source.content.kind === 'text',
|
|
274
|
+
parse: (source) => {
|
|
275
|
+
const text = textOf(source).trim();
|
|
276
|
+
return text ? [{ text }] : [];
|
|
277
|
+
},
|
|
278
|
+
};
|
|
279
|
+
// ── registry ──────────────────────────────────────────────────────────────────
|
|
280
|
+
/** Order matters: the first parser that accepts a source handles it. */
|
|
281
|
+
export const BUILTIN_PARSERS = [
|
|
282
|
+
rowsParser,
|
|
283
|
+
csvParser,
|
|
284
|
+
jsonParser,
|
|
285
|
+
markdownParser,
|
|
286
|
+
htmlParser,
|
|
287
|
+
plainTextParser,
|
|
288
|
+
];
|
|
289
|
+
export class ParserRegistry {
|
|
290
|
+
_parsers;
|
|
291
|
+
constructor(parsers = BUILTIN_PARSERS) {
|
|
292
|
+
this._parsers = [...parsers];
|
|
293
|
+
}
|
|
294
|
+
/** Registers a parser at the FRONT, so it can override a builtin. */
|
|
295
|
+
register(parser) {
|
|
296
|
+
this._parsers.unshift(parser);
|
|
297
|
+
return this;
|
|
298
|
+
}
|
|
299
|
+
resolve(source) {
|
|
300
|
+
return this._parsers.find((p) => p.accepts(source));
|
|
301
|
+
}
|
|
302
|
+
list() {
|
|
303
|
+
return this._parsers.map((p) => p.id);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
//# sourceMappingURL=parsers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../src/ingest/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,MAAM,MAAM,GAAG,CAAC,MAAsB,EAAU,EAAE,CAC9C,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;AAE9D,MAAM,WAAW,GAAG,CAAC,MAAsB,EAAU,EAAE,CACnD,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAEnG,iFAAiF;AAEjF;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,GAAkB;IAC3C,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC;SACrB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE,CAAC;SAC5D,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;SACvE,IAAI,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAmB;IACtC,EAAE,EAAE,MAAM;IACV,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;IACnD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,EAAE,CAAC;QAC9C,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;QAC5C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YAC3B,MAAM,QAAQ,GAAmB,EAAE,GAAG,GAAG,EAAE,CAAC;YAC5C,QAAQ,CAAC,UAAU,CAAC,GAAG,KAAK,CAAC;YAC7B,IAAI,UAAU,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC9C,QAAQ,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;YAChD,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;IACP,CAAC;CACJ,CAAC;AAEF,iFAAiF;AAEjF;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,SAAS,GAAG,GAAG;IAClD,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,MAAM,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAW,CAAC;QAC7B,IAAI,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBAAC,KAAK,IAAI,GAAG,CAAC;oBAAC,CAAC,EAAE,CAAC;gBAAC,CAAC;;oBAC1C,MAAM,GAAG,KAAK,CAAC;YACxB,CAAC;;gBAAM,KAAK,IAAI,EAAE,CAAC;YACnB,SAAS;QACb,CAAC;QACD,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC;YAAC,SAAS;QAAC,CAAC;QAC5C,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAC,KAAK,GAAG,EAAE,CAAC;YAAC,SAAS;QAAC,CAAC;QACnE,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7B,IAAI,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI;gBAAE,CAAC,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClB,MAAM,GAAG,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,CAAC;YACX,SAAS;QACb,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;IAChB,CAAC;IACD,IAAI,KAAK,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC5B,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAEvB,OAAO,IAAI;SACN,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SACnD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACX,MAAM,GAAG,GAAkB,EAAE,CAAC;QAC9B,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;YACvB,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QACH,OAAO,GAAG,CAAC;IACf,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;;;GAIG;AACH,SAAS,MAAM,CAAC,GAAW;IACvB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACzB,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,EAAE,CAAC;IAC5B,IAAI,KAAK,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACpC,IAAI,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAmB;IACrC,EAAE,EAAE,KAAK;IACT,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE;QAChB,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IAClE,CAAC;IACD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;QAC7E,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IAC5E,CAAC;CACJ,CAAC;AAEF,iFAAiF;AAEjF;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAmB;IACtC,EAAE,EAAE,MAAM;IACV,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;IACzD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACL,wEAAwE;YACxE,wEAAwE;YACxE,OAAO,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAChG,OAAO,UAAU,CAAC,KAAK,CAAC;gBACpB,GAAG,MAAM;gBACT,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAyB,EAAE;aAC7D,CAAC,CAAC;QACP,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACnC,OAAO,CAAC;gBACJ,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBACnE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE;aACzC,CAAC,CAAC;IACP,CAAC;CACJ,CAAC;AAEF,SAAS,OAAO,CAAC,KAAc,EAAE,IAAY,EAAE,MAA+B,EAAE;IAC5E,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9C,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,GAAG,CAAC;IACf,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;QACzD,OAAO,GAAG,CAAC;IACf,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;QACtE,OAAO,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,iFAAiF;AAEjF,MAAM,CAAC,MAAM,cAAc,GAAmB;IAC1C,EAAE,EAAE,UAAU;IACd,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE;QAChB,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IACD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAoB,EAAE,CAAC;QACrC,4EAA4E;QAC5E,4BAA4B;QAC5B,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,IAAI,MAAM,GAAa,EAAE,CAAC;QAE1B,MAAM,KAAK,GAAG,GAAG,EAAE;YACf,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,GAAG,EAAE,CAAC;YACZ,IAAI,CAAC,IAAI;gBAAE,OAAO;YAClB,MAAM,QAAQ,GAAmB,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,QAAQ,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3C,QAAQ,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAW,CAAC;gBACtD,QAAQ,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;YAC3C,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC;QAEF,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,GAAG,CAAC,OAAO,CAAC;YAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChE,IAAI,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,KAAK,GAAI,OAAO,CAAC,CAAC,CAAY,CAAC,MAAM,CAAC;gBAC5C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBAC/C,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAI,OAAO,CAAC,CAAC,CAAY,CAAC,IAAI,EAAE,CAAC;gBAChD,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;gBACpB,SAAS;YACb,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QACD,KAAK,EAAE,CAAC;QACR,OAAO,QAAQ,CAAC;IACpB,CAAC;CACJ,CAAC;AAEF,iFAAiF;AAEjF,MAAM,CAAC,MAAM,UAAU,GAAmB;IACtC,EAAE,EAAE,MAAM;IACV,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;IAChG,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,UAAU,GAAG,kCAAkC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhE,MAAM,IAAI,GAAG,GAAG;YACZ,wEAAwE;aACvE,OAAO,CAAC,uCAAuC,EAAE,GAAG,CAAC;aACrD,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC;aAChC,OAAO,CAAC,4CAA4C,EAAE,IAAI,CAAC;aAC3D,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC;aAC7B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aAC1B,IAAI,EAAE,CAAC;QAEZ,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClE,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAChC,CAAC;CACJ,CAAC;AAEF,iFAAiF;AAEjF,MAAM,CAAC,MAAM,eAAe,GAAmB;IAC3C,EAAE,EAAE,MAAM;IACV,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;IACnD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClC,CAAC;CACJ,CAAC;AAEF,iFAAiF;AAEjF,wEAAwE;AACxE,MAAM,CAAC,MAAM,eAAe,GAA8B;IACtD,UAAU;IACV,SAAS;IACT,UAAU;IACV,cAAc;IACd,UAAU;IACV,eAAe;CAClB,CAAC;AAEF,MAAM,OAAO,cAAc;IACN,QAAQ,CAAmB;IAE5C,YAAY,UAAqC,eAAe;QAC5D,IAAI,CAAC,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;IACjC,CAAC;IAED,qEAAqE;IACrE,QAAQ,CAAC,MAAsB;QAC3B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,CAAC,MAAsB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,IAAI;QACA,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;CACJ"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest/types.ts — the ingestion vocabulary.
|
|
3
|
+
*
|
|
4
|
+
* The pipeline handles STRUCTURED and UNSTRUCTURED sources through one path, which
|
|
5
|
+
* is the point. A support ticket export (rows, typed columns, foreign keys) and a
|
|
6
|
+
* policy PDF's text both end up as chunks with typed metadata; the difference is
|
|
7
|
+
* which parser ran, not which pipeline. That is what makes a query like
|
|
8
|
+
* "summarise open P1 tickets about billing" answerable — `priority` and `status`
|
|
9
|
+
* survive as *filterable metadata* instead of being flattened into prose the
|
|
10
|
+
* embedding has to recover.
|
|
11
|
+
*/
|
|
12
|
+
import type { MetadataRecord, MetadataValue } from '../vectorstore/types.js';
|
|
13
|
+
/** One row of a structured source (a DB result set, a CSV, a spreadsheet). */
|
|
14
|
+
export type StructuredRow = Record<string, MetadataValue>;
|
|
15
|
+
export type SourceContent = {
|
|
16
|
+
kind: 'text';
|
|
17
|
+
text: string;
|
|
18
|
+
mediaType?: string;
|
|
19
|
+
} | {
|
|
20
|
+
kind: 'rows';
|
|
21
|
+
rows: StructuredRow[];
|
|
22
|
+
rowIdField?: string;
|
|
23
|
+
};
|
|
24
|
+
export interface SourceDocument {
|
|
25
|
+
/** Stable id in the ORIGIN system. Re-ingesting the same id replaces its chunks. */
|
|
26
|
+
id: string;
|
|
27
|
+
content: SourceContent;
|
|
28
|
+
/** Owning tenant. Written to every chunk and enforced on every read. */
|
|
29
|
+
tenantId: string;
|
|
30
|
+
/**
|
|
31
|
+
* Principal ids permitted to retrieve this document. Defaults to `['*']`
|
|
32
|
+
* (tenant-public). Written to every chunk so ACL evaluation happens inside the
|
|
33
|
+
* database query, not after the fact.
|
|
34
|
+
*/
|
|
35
|
+
acl?: string[];
|
|
36
|
+
/** Classification level; readers declare a ceiling via `AccessScope`. */
|
|
37
|
+
sensitivity?: number;
|
|
38
|
+
/** Arbitrary filterable metadata copied onto every chunk. */
|
|
39
|
+
metadata?: MetadataRecord;
|
|
40
|
+
/** Human-facing title, used in citations. */
|
|
41
|
+
title?: string;
|
|
42
|
+
/** Origin URI, used in citations. */
|
|
43
|
+
uri?: string;
|
|
44
|
+
}
|
|
45
|
+
/** A parser's output: a span of text plus whatever structure it recovered. */
|
|
46
|
+
export interface ParsedSection {
|
|
47
|
+
text: string;
|
|
48
|
+
/** Section-scoped metadata — heading path, row id, JSON pointer, … */
|
|
49
|
+
metadata?: MetadataRecord;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Parsers are registry entries, so a new format (PDF text layer, DOCX, a bespoke
|
|
53
|
+
* log) is registered rather than branched into the pipeline.
|
|
54
|
+
*/
|
|
55
|
+
export interface DocumentParser {
|
|
56
|
+
readonly id: string;
|
|
57
|
+
/** True when this parser handles the source. First match in registry order wins. */
|
|
58
|
+
accepts(source: SourceDocument): boolean;
|
|
59
|
+
parse(source: SourceDocument): ParsedSection[];
|
|
60
|
+
}
|
|
61
|
+
/** Identity of one stored chunk, for incremental sync. */
|
|
62
|
+
export interface ChunkFingerprint {
|
|
63
|
+
id: string;
|
|
64
|
+
/** Content hash — an unchanged hash means the chunk need not be re-embedded. */
|
|
65
|
+
hash: string;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Records which chunks each source produced last time.
|
|
69
|
+
*
|
|
70
|
+
* Supplying one enables `diff` sync: only changed chunks are re-embedded (the
|
|
71
|
+
* expensive step) and only vanished chunks are deleted. Without it the pipeline
|
|
72
|
+
* falls back to `replace`, which is correct but re-embeds everything.
|
|
73
|
+
*/
|
|
74
|
+
export interface IngestManifest {
|
|
75
|
+
get(sourceId: string): Promise<ChunkFingerprint[] | undefined>;
|
|
76
|
+
set(sourceId: string, fingerprints: ChunkFingerprint[]): Promise<void>;
|
|
77
|
+
delete(sourceId: string): Promise<void>;
|
|
78
|
+
}
|
|
79
|
+
/** Batch embedder. Batched because per-text calls are the classic N+1 of RAG. */
|
|
80
|
+
export type BatchEmbedder = (texts: string[]) => Promise<Float32Array[]>;
|
|
81
|
+
export type SyncStrategy =
|
|
82
|
+
/** Re-embed only changed chunks, delete only vanished ones. Needs a manifest. */
|
|
83
|
+
'diff'
|
|
84
|
+
/** Delete the source's existing chunks, then write the new set. */
|
|
85
|
+
| 'replace'
|
|
86
|
+
/** Write without deleting anything — for append-only corpora. */
|
|
87
|
+
| 'append';
|
|
88
|
+
export interface IngestionReport {
|
|
89
|
+
documents: number;
|
|
90
|
+
sections: number;
|
|
91
|
+
chunks: number;
|
|
92
|
+
/** Chunks sent to the embedder — the cost driver. */
|
|
93
|
+
embedded: number;
|
|
94
|
+
upserted: number;
|
|
95
|
+
/** Chunks whose hash was unchanged and were skipped entirely. */
|
|
96
|
+
skippedUnchanged: number;
|
|
97
|
+
/** Chunks deleted because they no longer exist in the source. */
|
|
98
|
+
deletedStale: number;
|
|
99
|
+
failures: Array<{
|
|
100
|
+
sourceId: string;
|
|
101
|
+
error: string;
|
|
102
|
+
}>;
|
|
103
|
+
durationMs: number;
|
|
104
|
+
/** Trace id covering the whole run, when a tracer is attached. */
|
|
105
|
+
traceId?: string;
|
|
106
|
+
}
|
|
107
|
+
/** Metadata keys the pipeline reserves on every chunk it writes. */
|
|
108
|
+
export declare const SOURCE_ID_FIELD = "sourceId";
|
|
109
|
+
export declare const CHUNK_INDEX_FIELD = "chunkIndex";
|
|
110
|
+
export declare const CONTENT_HASH_FIELD = "contentHash";
|
|
111
|
+
export declare const TITLE_FIELD = "title";
|
|
112
|
+
export declare const URI_FIELD = "uri";
|
|
113
|
+
export declare const PARSER_FIELD = "parser";
|
|
114
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/ingest/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAE7E,8EAA8E;AAC9E,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;AAE1D,MAAM,MAAM,aAAa,GACnB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAClD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,aAAa,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnE,MAAM,WAAW,cAAc;IAC3B,oFAAoF;IACpF,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,aAAa,CAAC;IACvB,wEAAwE;IACxE,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,yEAAyE;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,6CAA6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,GAAG,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8EAA8E;AAC9E,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,sEAAsE;IACtE,QAAQ,CAAC,EAAE,cAAc,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,oFAAoF;IACpF,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC;IACzC,KAAK,CAAC,MAAM,EAAE,cAAc,GAAG,aAAa,EAAE,CAAC;CAClD;AAED,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC3B,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,GAAG,SAAS,CAAC,CAAC;IAC/D,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvE,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3C;AAED,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;AAEzE,MAAM,MAAM,YAAY;AACpB,iFAAiF;AAC/E,MAAM;AACR,mEAAmE;GACjE,SAAS;AACX,iEAAiE;GAC/D,QAAQ,CAAC;AAEf,MAAM,WAAW,eAAe;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,gBAAgB,EAAE,MAAM,CAAC;IACzB,iEAAiE;IACjE,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oEAAoE;AACpE,eAAO,MAAM,eAAe,aAAa,CAAC;AAC1C,eAAO,MAAM,iBAAiB,eAAe,CAAC;AAC9C,eAAO,MAAM,kBAAkB,gBAAgB,CAAC;AAChD,eAAO,MAAM,WAAW,UAAU,CAAC;AACnC,eAAO,MAAM,SAAS,QAAQ,CAAC;AAC/B,eAAO,MAAM,YAAY,WAAW,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest/types.ts — the ingestion vocabulary.
|
|
3
|
+
*
|
|
4
|
+
* The pipeline handles STRUCTURED and UNSTRUCTURED sources through one path, which
|
|
5
|
+
* is the point. A support ticket export (rows, typed columns, foreign keys) and a
|
|
6
|
+
* policy PDF's text both end up as chunks with typed metadata; the difference is
|
|
7
|
+
* which parser ran, not which pipeline. That is what makes a query like
|
|
8
|
+
* "summarise open P1 tickets about billing" answerable — `priority` and `status`
|
|
9
|
+
* survive as *filterable metadata* instead of being flattened into prose the
|
|
10
|
+
* embedding has to recover.
|
|
11
|
+
*/
|
|
12
|
+
/** Metadata keys the pipeline reserves on every chunk it writes. */
|
|
13
|
+
export const SOURCE_ID_FIELD = 'sourceId';
|
|
14
|
+
export const CHUNK_INDEX_FIELD = 'chunkIndex';
|
|
15
|
+
export const CONTENT_HASH_FIELD = 'contentHash';
|
|
16
|
+
export const TITLE_FIELD = 'title';
|
|
17
|
+
export const URI_FIELD = 'uri';
|
|
18
|
+
export const PARSER_FIELD = 'parser';
|
|
19
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/ingest/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAmGH,oEAAoE;AACpE,MAAM,CAAC,MAAM,eAAe,GAAG,UAAU,CAAC;AAC1C,MAAM,CAAC,MAAM,iBAAiB,GAAG,YAAY,CAAC;AAC9C,MAAM,CAAC,MAAM,kBAAkB,GAAG,aAAa,CAAC;AAChD,MAAM,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC;AACnC,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,CAAC;AAC/B,MAAM,CAAC,MAAM,YAAY,GAAG,QAAQ,CAAC"}
|
|
@@ -30,6 +30,23 @@ export interface MemoryEntry {
|
|
|
30
30
|
/** Importance weight in the range 0–1. Default: 0.5. */
|
|
31
31
|
importance?: number;
|
|
32
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Which ranker produced a recall ordering. Matches the vocabulary the gateway's
|
|
35
|
+
* Evermind recall/validate surfaces already report, so a host can render one chip
|
|
36
|
+
* ("Semantic recall" / "Lexical recall (fallback)") regardless of which layer
|
|
37
|
+
* answered.
|
|
38
|
+
*/
|
|
39
|
+
export type RecallMethod = 'embedding' | 'lexical';
|
|
40
|
+
/** A recalled entry with its relevance score (higher = closer). */
|
|
41
|
+
export interface ScoredMemory {
|
|
42
|
+
entry: MemoryEntry;
|
|
43
|
+
score: number;
|
|
44
|
+
}
|
|
45
|
+
/** A ranked recall: the hits plus the ranker that ordered them. */
|
|
46
|
+
export interface RankedRecall {
|
|
47
|
+
hits: ScoredMemory[];
|
|
48
|
+
method: RecallMethod;
|
|
49
|
+
}
|
|
33
50
|
export interface RememberOptions {
|
|
34
51
|
/** Override the store-level defaultTtlMs for this entry. */
|
|
35
52
|
ttlMs?: number;
|
|
@@ -171,6 +188,22 @@ export declare class MemoryStore {
|
|
|
171
188
|
* so it is always strictly at least as good as the lexical fallback.
|
|
172
189
|
*/
|
|
173
190
|
recallHybrid(query: string, topK: number, runtime?: SSMRuntimeRef, opts?: HybridRetrieveOptions): Promise<MemoryEntry[]>;
|
|
191
|
+
/**
|
|
192
|
+
* The hybrid retrieval path with its evidence attached: each hit's fused score,
|
|
193
|
+
* plus WHICH ranker produced the ordering.
|
|
194
|
+
*
|
|
195
|
+
* {@link recallHybrid} is the convenience wrapper that throws both away, so
|
|
196
|
+
* there is exactly one implementation of the blend — and it is the shared
|
|
197
|
+
* `hybridRetrieve` → `reciprocalRankFusion` primitive, never a second hand-
|
|
198
|
+
* rolled mix of cosine and word overlap.
|
|
199
|
+
*
|
|
200
|
+
* `method` is honest rather than aspirational: `'embedding'` means the query
|
|
201
|
+
* really did embed and the dense ranking really did participate in the fusion;
|
|
202
|
+
* `'lexical'` means no model was available (or embedding failed) and the
|
|
203
|
+
* ordering is BM25 alone. Callers surface it so a user can tell semantic
|
|
204
|
+
* recall from the fallback instead of guessing.
|
|
205
|
+
*/
|
|
206
|
+
recallRanked(query: string, topK: number, runtime?: SSMRuntimeRef, opts?: HybridRetrieveOptions): Promise<RankedRecall>;
|
|
174
207
|
/**
|
|
175
208
|
* Returns a cached embedding for `text`, computing it via `runtime.embed()`
|
|
176
209
|
* on a cache miss. Returns `null` (never throws) when embedding is
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MemoryStore.d.ts","sourceRoot":"","sources":["../../src/memory/MemoryStore.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,EAAmE,KAAK,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAEpI,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE9D,MAAM,WAAW,WAAW;IACxB,GAAG,EAAU,MAAM,CAAC;IACpB,OAAO,EAAM,MAAM,CAAC;IACpB,SAAS,EAAI,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,GAAG,CAAC,EAAS,MAAM,CAAC;IACpB,6CAA6C;IAC7C,KAAK,CAAC,EAAO,MAAM,CAAC;IACpB,0DAA0D;IAC1D,IAAI,CAAC,EAAQ,QAAQ,CAAC;IACtB,uCAAuC;IACvC,IAAI,CAAC,EAAQ,MAAM,EAAE,CAAC;IACtB,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,eAAe;IAC5B,4DAA4D;IAC5D,KAAK,CAAC,EAAO,MAAM,CAAC;IACpB,IAAI,CAAC,EAAQ,QAAQ,CAAC;IACtB,IAAI,CAAC,EAAQ,MAAM,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IAC/B,iDAAiD;IACjD,MAAM,CAAC,EAAS,MAAM,CAAC;IACvB,+FAA+F;IAC/F,UAAU,CAAC,EAAK,MAAM,CAAC;IACvB;;;;;OAKG;IACH,UAAU,CAAC,EAAK,UAAU,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAG,MAAM,CAAC;IACvB;;;;OAIG;IACH,YAAY,CAAC,EAAG,MAAM,CAAC;IACvB;;;;OAIG;IACH,UAAU,CAAC,EAAK,MAAM,CAAC;IACvB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAcD,UAAU,eAAe;IACrB,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,EAAE,WAAW,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAClD;AAID,KAAK,aAAa,GAAG,GAAG,CAAC;AAKzB,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAU;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IACzC,OAAO,CAAC,GAAG,CAA4B;IACvC,OAAO,CAAC,WAAW,CAA+C;IAClE,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmC;IAC/D,gFAAgF;IAChF,OAAO,CAAC,aAAa,CAAiB;IAOtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAmC;IACjE,OAAO,CAAC,WAAW,CAA0B;IAC7C,kEAAkE;IAClE,OAAO,CAAC,SAAS,CAAM;IACvB,sEAAsE;IACtE,OAAO,CAAC,aAAa,CAAiB;gBAE1B,IAAI,GAAE,kBAAuB;IAYzC,OAAO,CAAC,KAAK;IAwCb,OAAO,CAAC,UAAU;IAOlB,mCAAmC;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBnF;;;;OAIG;YACW,WAAW;IA0BzB;;;;OAIG;IACH,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAYzC,2EAA2E;IAC3E,cAAc,IAAI,IAAI;IAOtB;;;OAGG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAW3D,0DAA0D;IACpD,SAAS,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAgBzC;;;OAGG;IACG,YAAY,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAKrD;;OAEG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAKtD;;;;;;;;;;;OAWG;IACG,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAIjG;;;;;OAKG;IACG,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAqCtI;;;;;;;;;OASG;IACG,YAAY,CACd,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,aAAa,EACvB,IAAI,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"MemoryStore.d.ts","sourceRoot":"","sources":["../../src/memory/MemoryStore.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,EAAmE,KAAK,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAEpI,MAAM,MAAM,QAAQ,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE9D,MAAM,WAAW,WAAW;IACxB,GAAG,EAAU,MAAM,CAAC;IACpB,OAAO,EAAM,MAAM,CAAC;IACpB,SAAS,EAAI,MAAM,CAAC;IACpB;;;;;;OAMG;IACH,GAAG,CAAC,EAAS,MAAM,CAAC;IACpB,6CAA6C;IAC7C,KAAK,CAAC,EAAO,MAAM,CAAC;IACpB,0DAA0D;IAC1D,IAAI,CAAC,EAAQ,QAAQ,CAAC;IACtB,uCAAuC;IACvC,IAAI,CAAC,EAAQ,MAAM,EAAE,CAAC;IACtB,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;GAKG;AACH,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,SAAS,CAAC;AAEnD,mEAAmE;AACnE,MAAM,WAAW,YAAY;IACzB,KAAK,EAAE,WAAW,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,mEAAmE;AACnE,MAAM,WAAW,YAAY;IACzB,IAAI,EAAE,YAAY,EAAE,CAAC;IACrB,MAAM,EAAE,YAAY,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC5B,4DAA4D;IAC5D,KAAK,CAAC,EAAO,MAAM,CAAC;IACpB,IAAI,CAAC,EAAQ,QAAQ,CAAC;IACtB,IAAI,CAAC,EAAQ,MAAM,EAAE,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,kBAAkB;IAC/B,iDAAiD;IACjD,MAAM,CAAC,EAAS,MAAM,CAAC;IACvB,+FAA+F;IAC/F,UAAU,CAAC,EAAK,MAAM,CAAC;IACvB;;;;;OAKG;IACH,UAAU,CAAC,EAAK,UAAU,CAAC;IAC3B;;;OAGG;IACH,YAAY,CAAC,EAAG,MAAM,CAAC;IACvB;;;;OAIG;IACH,YAAY,CAAC,EAAG,MAAM,CAAC;IACvB;;;;OAIG;IACH,UAAU,CAAC,EAAK,MAAM,CAAC;IACvB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CAC1B;AAcD,UAAU,eAAe;IACrB,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,OAAO,EAAE,WAAW,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClE,IAAI,CAAC,IAAI,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CAClD;AAID,KAAK,aAAa,GAAG,GAAG,CAAC;AAKzB,qBAAa,WAAW;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAc;IACtC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAU;IACtC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAiC;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAU;IACxC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IACzC,OAAO,CAAC,GAAG,CAA4B;IACvC,OAAO,CAAC,WAAW,CAA+C;IAClE,sEAAsE;IACtE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAmC;IAC/D,gFAAgF;IAChF,OAAO,CAAC,aAAa,CAAiB;IAOtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAmC;IACjE,OAAO,CAAC,WAAW,CAA0B;IAC7C,kEAAkE;IAClE,OAAO,CAAC,SAAS,CAAM;IACvB,sEAAsE;IACtE,OAAO,CAAC,aAAa,CAAiB;gBAE1B,IAAI,GAAE,kBAAuB;IAYzC,OAAO,CAAC,KAAK;IAwCb,OAAO,CAAC,UAAU;IAOlB,mCAAmC;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBnF;;;;OAIG;YACW,WAAW;IA0BzB;;;;OAIG;IACH,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAYzC,2EAA2E;IAC3E,cAAc,IAAI,IAAI;IAOtB;;;OAGG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;IAW3D,0DAA0D;IACpD,SAAS,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAgBzC;;;OAGG;IACG,YAAY,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAKrD;;OAEG;IACG,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAKtD;;;;;;;;;;;OAWG;IACG,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAIjG;;;;;OAKG;IACG,mBAAmB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,WAAW,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAqCtI;;;;;;;;;OASG;IACG,YAAY,CACd,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,aAAa,EACvB,IAAI,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,WAAW,EAAE,CAAC;IAIzB;;;;;;;;;;;;;;OAcG;IACG,YAAY,CACd,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,aAAa,EACvB,IAAI,CAAC,EAAE,qBAAqB,GAC7B,OAAO,CAAC,YAAY,CAAC;IA4BxB;;;;OAIG;YACW,eAAe;IAiC7B;;;;;;;;OAQG;YACW,UAAU;IAqDxB;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC;IAkBrC,0DAA0D;IACpD,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOxC,wDAAwD;IAClD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAc5B;;;OAGG;IACG,SAAS,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;IAIzC;;;;;;OAMG;IACG,SAAS,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,QAAQ,EAAE,OAAO,GAAG,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAavF,mFAAmF;YACrE,OAAO;IAQrB;;;;OAIG;IACG,WAAW,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAI1D;;;OAGG;IACG,WAAW,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,OAAO,CAAC;CAGhE"}
|