@seanhogg/builderforce-memory 2026.10.28 → 2026.10.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ingest/IngestionPipeline.d.ts +1 -8
- package/dist/ingest/IngestionPipeline.d.ts.map +1 -1
- package/dist/ingest/IngestionPipeline.js +3 -15
- package/dist/ingest/IngestionPipeline.js.map +1 -1
- package/dist/ingest/docsExtract.d.ts +30 -0
- package/dist/ingest/docsExtract.d.ts.map +1 -0
- package/dist/ingest/docsExtract.js +221 -0
- package/dist/ingest/docsExtract.js.map +1 -0
- package/dist/ingest/hash.d.ts +15 -0
- package/dist/ingest/hash.d.ts.map +1 -0
- package/dist/ingest/hash.js +22 -0
- package/dist/ingest/hash.js.map +1 -0
- package/dist/ingest/parsers.d.ts.map +1 -1
- package/dist/ingest/parsers.js +4 -0
- package/dist/ingest/parsers.js.map +1 -1
- package/dist/ingest/types.d.ts +7 -0
- package/dist/ingest/types.d.ts.map +1 -1
- package/dist/ingest/types.js.map +1 -1
- package/dist/knowledge/build.d.ts +24 -0
- package/dist/knowledge/build.d.ts.map +1 -0
- package/dist/knowledge/build.js +263 -0
- package/dist/knowledge/build.js.map +1 -0
- package/dist/knowledge/cards.d.ts +17 -0
- package/dist/knowledge/cards.d.ts.map +1 -0
- package/dist/knowledge/cards.js +118 -0
- package/dist/knowledge/cards.js.map +1 -0
- package/dist/knowledge/findings.d.ts +24 -0
- package/dist/knowledge/findings.d.ts.map +1 -0
- package/dist/knowledge/findings.js +152 -0
- package/dist/knowledge/findings.js.map +1 -0
- package/dist/knowledge/importance.d.ts +21 -0
- package/dist/knowledge/importance.d.ts.map +1 -0
- package/dist/knowledge/importance.js +70 -0
- package/dist/knowledge/importance.js.map +1 -0
- package/dist/knowledge/index.d.ts +31 -0
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/index.js +26 -0
- package/dist/knowledge/index.js.map +1 -0
- package/dist/knowledge/judge.d.ts +86 -0
- package/dist/knowledge/judge.d.ts.map +1 -0
- package/dist/knowledge/judge.js +213 -0
- package/dist/knowledge/judge.js.map +1 -0
- package/dist/knowledge/model.d.ts +333 -0
- package/dist/knowledge/model.d.ts.map +1 -0
- package/dist/knowledge/model.js +13 -0
- package/dist/knowledge/model.js.map +1 -0
- package/dist/knowledge/packs/docs.d.ts +17 -0
- package/dist/knowledge/packs/docs.d.ts.map +1 -0
- package/dist/knowledge/packs/docs.js +89 -0
- package/dist/knowledge/packs/docs.js.map +1 -0
- package/dist/knowledge/partitions.d.ts +31 -0
- package/dist/knowledge/partitions.d.ts.map +1 -0
- package/dist/knowledge/partitions.js +135 -0
- package/dist/knowledge/partitions.js.map +1 -0
- package/dist/knowledge/plan.d.ts +33 -0
- package/dist/knowledge/plan.d.ts.map +1 -0
- package/dist/knowledge/plan.js +89 -0
- package/dist/knowledge/plan.js.map +1 -0
- package/dist/knowledge/refs.d.ts +40 -0
- package/dist/knowledge/refs.d.ts.map +1 -0
- package/dist/knowledge/refs.js +102 -0
- package/dist/knowledge/refs.js.map +1 -0
- package/dist/knowledge/resolve.d.ts +35 -0
- package/dist/knowledge/resolve.d.ts.map +1 -0
- package/dist/knowledge/resolve.js +67 -0
- package/dist/knowledge/resolve.js.map +1 -0
- package/dist/knowledge/review.d.ts +23 -0
- package/dist/knowledge/review.d.ts.map +1 -0
- package/dist/knowledge/review.js +233 -0
- package/dist/knowledge/review.js.map +1 -0
- package/dist/knowledge/sources.d.ts +47 -0
- package/dist/knowledge/sources.d.ts.map +1 -0
- package/dist/knowledge/sources.js +136 -0
- package/dist/knowledge/sources.js.map +1 -0
- package/dist/knowledge/stores/localSnapshotStore.d.ts +50 -0
- package/dist/knowledge/stores/localSnapshotStore.d.ts.map +1 -0
- package/dist/knowledge/stores/localSnapshotStore.js +125 -0
- package/dist/knowledge/stores/localSnapshotStore.js.map +1 -0
- package/dist/knowledge/trace.d.ts +32 -0
- package/dist/knowledge/trace.d.ts.map +1 -0
- package/dist/knowledge/trace.js +102 -0
- package/dist/knowledge/trace.js.map +1 -0
- package/dist/knowledge/truncate.d.ts +24 -0
- package/dist/knowledge/truncate.d.ts.map +1 -0
- package/dist/knowledge/truncate.js +128 -0
- package/dist/knowledge/truncate.js.map +1 -0
- package/dist/memory/SemanticRecall.d.ts.map +1 -1
- package/dist/memory/SemanticRecall.js +3 -1
- package/dist/memory/SemanticRecall.js.map +1 -1
- package/dist/retrieval/HybridRetriever.d.ts.map +1 -1
- package/dist/retrieval/HybridRetriever.js +3 -1
- package/dist/retrieval/HybridRetriever.js.map +1 -1
- package/dist/retrieval/bm25.d.ts +3 -2
- package/dist/retrieval/bm25.d.ts.map +1 -1
- package/dist/retrieval/bm25.js +5 -3
- package/dist/retrieval/bm25.js.map +1 -1
- package/dist/retrieval/fusion.d.ts +2 -1
- package/dist/retrieval/fusion.d.ts.map +1 -1
- package/dist/retrieval/fusion.js +4 -2
- package/dist/retrieval/fusion.js.map +1 -1
- package/dist/retrieval/hnsw.d.ts +49 -11
- package/dist/retrieval/hnsw.d.ts.map +1 -1
- package/dist/retrieval/hnsw.js +57 -29
- package/dist/retrieval/hnsw.js.map +1 -1
- package/dist/retrieval/index.d.ts +7 -3
- package/dist/retrieval/index.d.ts.map +1 -1
- package/dist/retrieval/index.js +5 -2
- package/dist/retrieval/index.js.map +1 -1
- package/dist/retrieval/knowledge-index/bytes.d.ts +43 -0
- package/dist/retrieval/knowledge-index/bytes.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/bytes.js +151 -0
- package/dist/retrieval/knowledge-index/bytes.js.map +1 -0
- package/dist/retrieval/knowledge-index/cascade.d.ts +37 -0
- package/dist/retrieval/knowledge-index/cascade.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/cascade.js +111 -0
- package/dist/retrieval/knowledge-index/cascade.js.map +1 -0
- package/dist/retrieval/knowledge-index/compact.d.ts +18 -0
- package/dist/retrieval/knowledge-index/compact.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/compact.js +64 -0
- package/dist/retrieval/knowledge-index/compact.js.map +1 -0
- package/dist/retrieval/knowledge-index/csr.d.ts +41 -0
- package/dist/retrieval/knowledge-index/csr.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/csr.js +76 -0
- package/dist/retrieval/knowledge-index/csr.js.map +1 -0
- package/dist/retrieval/knowledge-index/digest.d.ts +21 -0
- package/dist/retrieval/knowledge-index/digest.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/digest.js +50 -0
- package/dist/retrieval/knowledge-index/digest.js.map +1 -0
- package/dist/retrieval/knowledge-index/embeddings.d.ts +32 -0
- package/dist/retrieval/knowledge-index/embeddings.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/embeddings.js +78 -0
- package/dist/retrieval/knowledge-index/embeddings.js.map +1 -0
- package/dist/retrieval/knowledge-index/expand.d.ts +19 -0
- package/dist/retrieval/knowledge-index/expand.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/expand.js +49 -0
- package/dist/retrieval/knowledge-index/expand.js.map +1 -0
- package/dist/retrieval/knowledge-index/index.d.ts +40 -0
- package/dist/retrieval/knowledge-index/index.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/index.js +134 -0
- package/dist/retrieval/knowledge-index/index.js.map +1 -0
- package/dist/retrieval/knowledge-index/lexical.d.ts +26 -0
- package/dist/retrieval/knowledge-index/lexical.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/lexical.js +76 -0
- package/dist/retrieval/knowledge-index/lexical.js.map +1 -0
- package/dist/retrieval/knowledge-index/pairs.d.ts +22 -0
- package/dist/retrieval/knowledge-index/pairs.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/pairs.js +51 -0
- package/dist/retrieval/knowledge-index/pairs.js.map +1 -0
- package/dist/retrieval/knowledge-index/prune.d.ts +21 -0
- package/dist/retrieval/knowledge-index/prune.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/prune.js +43 -0
- package/dist/retrieval/knowledge-index/prune.js.map +1 -0
- package/dist/retrieval/knowledge-index/quantize.d.ts +37 -0
- package/dist/retrieval/knowledge-index/quantize.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/quantize.js +77 -0
- package/dist/retrieval/knowledge-index/quantize.js.map +1 -0
- package/dist/retrieval/knowledge-index/serialize.d.ts +18 -0
- package/dist/retrieval/knowledge-index/serialize.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/serialize.js +123 -0
- package/dist/retrieval/knowledge-index/serialize.js.map +1 -0
- package/dist/retrieval/knowledge-index/sha256.d.ts +30 -0
- package/dist/retrieval/knowledge-index/sha256.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/sha256.js +130 -0
- package/dist/retrieval/knowledge-index/sha256.js.map +1 -0
- package/dist/retrieval/knowledge-index/state.d.ts +55 -0
- package/dist/retrieval/knowledge-index/state.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/state.js +67 -0
- package/dist/retrieval/knowledge-index/state.js.map +1 -0
- package/dist/retrieval/knowledge-index/tokenize.d.ts +20 -0
- package/dist/retrieval/knowledge-index/tokenize.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/tokenize.js +42 -0
- package/dist/retrieval/knowledge-index/tokenize.js.map +1 -0
- package/dist/retrieval/knowledge-index/traverse.d.ts +26 -0
- package/dist/retrieval/knowledge-index/traverse.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/traverse.js +90 -0
- package/dist/retrieval/knowledge-index/traverse.js.map +1 -0
- package/dist/retrieval/knowledge-index/types.d.ts +85 -0
- package/dist/retrieval/knowledge-index/types.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/types.js +9 -0
- package/dist/retrieval/knowledge-index/types.js.map +1 -0
- package/dist/retrieval/mmr.d.ts +11 -4
- package/dist/retrieval/mmr.d.ts.map +1 -1
- package/dist/retrieval/mmr.js +28 -8
- package/dist/retrieval/mmr.js.map +1 -1
- package/dist/retrieval/order.d.ts +41 -0
- package/dist/retrieval/order.d.ts.map +1 -0
- package/dist/retrieval/order.js +54 -0
- package/dist/retrieval/order.js.map +1 -0
- package/dist/retrieval/weightedFusion.d.ts +2 -1
- package/dist/retrieval/weightedFusion.d.ts.map +1 -1
- package/dist/retrieval/weightedFusion.js +7 -5
- package/dist/retrieval/weightedFusion.js.map +1 -1
- package/dist/vectorstore/MemoryVectorStore.d.ts.map +1 -1
- package/dist/vectorstore/MemoryVectorStore.js +2 -1
- package/dist/vectorstore/MemoryVectorStore.js.map +1 -1
- package/package.json +8 -3
- package/src/ingest/IngestionPipeline.ts +3 -15
- package/src/ingest/docsExtract.ts +219 -0
- package/src/ingest/hash.ts +22 -0
- package/src/ingest/parsers.ts +4 -0
- package/src/ingest/types.ts +7 -0
- package/src/knowledge/build.test.ts +150 -0
- package/src/knowledge/build.ts +266 -0
- package/src/knowledge/cards.ts +130 -0
- package/src/knowledge/docs.test.ts +118 -0
- package/src/knowledge/findings.ts +159 -0
- package/src/knowledge/importance.ts +72 -0
- package/src/knowledge/index.ts +53 -0
- package/src/knowledge/judge.ts +231 -0
- package/src/knowledge/model.ts +312 -0
- package/src/knowledge/packs/docs.ts +91 -0
- package/src/knowledge/partitions.test.ts +98 -0
- package/src/knowledge/partitions.ts +119 -0
- package/src/knowledge/plan.ts +115 -0
- package/src/knowledge/refs.ts +108 -0
- package/src/knowledge/resolve.ts +78 -0
- package/src/knowledge/review.test.ts +180 -0
- package/src/knowledge/review.ts +235 -0
- package/src/knowledge/sources.ts +146 -0
- package/src/knowledge/stores/localSnapshotStore.test.ts +82 -0
- package/src/knowledge/stores/localSnapshotStore.ts +159 -0
- package/src/knowledge/testkit.test-util.ts +92 -0
- package/src/knowledge/trace.test.ts +91 -0
- package/src/knowledge/trace.ts +111 -0
- package/src/knowledge/truncate.ts +118 -0
- package/src/memory/SemanticRecall.ts +3 -1
- package/src/retrieval/HybridRetriever.ts +3 -1
- package/src/retrieval/bm25.ts +5 -3
- package/src/retrieval/fusion.ts +5 -2
- package/src/retrieval/hnsw.ts +82 -35
- package/src/retrieval/index.ts +17 -3
- package/src/retrieval/knowledge-index/bytes.ts +166 -0
- package/src/retrieval/knowledge-index/cascade.ts +125 -0
- package/src/retrieval/knowledge-index/compact.ts +68 -0
- package/src/retrieval/knowledge-index/csr.ts +102 -0
- package/src/retrieval/knowledge-index/digest.ts +54 -0
- package/src/retrieval/knowledge-index/embeddings.ts +85 -0
- package/src/retrieval/knowledge-index/expand.ts +55 -0
- package/src/retrieval/knowledge-index/index.ts +162 -0
- package/src/retrieval/knowledge-index/lexical.ts +78 -0
- package/src/retrieval/knowledge-index/pairs.ts +57 -0
- package/src/retrieval/knowledge-index/prune.ts +48 -0
- package/src/retrieval/knowledge-index/quantize.ts +78 -0
- package/src/retrieval/knowledge-index/serialize.ts +124 -0
- package/src/retrieval/knowledge-index/sha256.ts +122 -0
- package/src/retrieval/knowledge-index/state.ts +108 -0
- package/src/retrieval/knowledge-index/tokenize.ts +41 -0
- package/src/retrieval/knowledge-index/traverse.ts +95 -0
- package/src/retrieval/knowledge-index/types.ts +78 -0
- package/src/retrieval/mmr.ts +35 -10
- package/src/retrieval/order.ts +68 -0
- package/src/retrieval/weightedFusion.ts +8 -5
- package/src/vectorstore/MemoryVectorStore.ts +2 -1
|
@@ -82,14 +82,7 @@ export declare class InMemoryIngestManifest implements IngestManifest {
|
|
|
82
82
|
delete(sourceId: string): Promise<void>;
|
|
83
83
|
get size(): number;
|
|
84
84
|
}
|
|
85
|
-
|
|
86
|
-
* FNV-1a, 32-bit, rendered as hex.
|
|
87
|
-
*
|
|
88
|
-
* A content hash here answers "did this chunk change", not "was this chunk
|
|
89
|
-
* tampered with", so a fast non-cryptographic hash is the right tool — and it is
|
|
90
|
-
* synchronous, unlike WebCrypto, which keeps chunking a pure function.
|
|
91
|
-
*/
|
|
92
|
-
export declare function hashText(text: string): string;
|
|
85
|
+
export { hashText } from './hash.js';
|
|
93
86
|
/** Bounded-concurrency map — the reason a large corpus does not become a 429. */
|
|
94
87
|
export declare function mapWithConcurrency<T>(items: readonly T[], limit: number, fn: (item: T, index: number) => Promise<void>): Promise<void>;
|
|
95
88
|
//# sourceMappingURL=IngestionPipeline.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IngestionPipeline.d.ts","sourceRoot":"","sources":["../../src/ingest/IngestionPipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAa,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,EAOH,KAAK,WAAW,EACnB,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"IngestionPipeline.d.ts","sourceRoot":"","sources":["../../src/ingest/IngestionPipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAa,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAErD,OAAO,EAOH,KAAK,WAAW,EACnB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAOH,KAAK,aAAa,EAClB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,YAAY,EACpB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,wBAAwB;IACrC,KAAK,EAAE,WAAW,CAAC;IACnB,KAAK,EAAE,aAAa,CAAC;IACrB,iEAAiE;IACjE,KAAK,CAAC,EAAE,YAAY,CAAC;IACrB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,cAAc,GAAG,SAAS,cAAc,EAAE,CAAC;IACrD,wDAAwD;IACxD,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,2EAA2E;IAC3E,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,2CAA2C;IAC3C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6CAA6C;IAC7C,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,iBAAiB;IAC1B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAe;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAEjC,IAAI,EAAE,wBAAwB;IAgBpC,MAAM,CAAC,OAAO,EAAE,SAAS,cAAc,EAAE,GAAG,OAAO,CAAC,eAAe,CAAC;IA6C1E,iFAAiF;IAC3E,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAQjC,UAAU;IA8FxB;;;;;;OAMG;YACW,SAAS;YAyBT,iBAAiB;IA2B/B,OAAO,CAAC,aAAa;CAgBxB;AAED,iFAAiF;AACjF,qBAAa,sBAAuB,YAAW,cAAc;IACzD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAyC;IAE5D,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,GAAG,SAAS,CAAC;IAI9D,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAItE,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAI7C,IAAI,IAAI,IAAI,MAAM,CAA+B;CACpD;AAGD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,iFAAiF;AACjF,wBAAsB,kBAAkB,CAAC,CAAC,EACtC,KAAK,EAAE,SAAS,CAAC,EAAE,EACnB,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,GAC9C,OAAO,CAAC,IAAI,CAAC,CASf"}
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
import { chunkText } from '../retrieval/chunk.js';
|
|
24
24
|
import { estimateTokens } from '../telemetry/pricing.js';
|
|
25
25
|
import { ACL_FIELD, ACL_PUBLIC, SENSITIVITY_FIELD, TENANT_FIELD, } from '../vectorstore/types.js';
|
|
26
|
+
import { hashText } from './hash.js';
|
|
26
27
|
import { ParserRegistry } from './parsers.js';
|
|
27
28
|
import { CHUNK_INDEX_FIELD, CONTENT_HASH_FIELD, PARSER_FIELD, SOURCE_ID_FIELD, TITLE_FIELD, URI_FIELD, } from './types.js';
|
|
28
29
|
export class IngestionPipeline {
|
|
@@ -266,21 +267,8 @@ export class InMemoryIngestManifest {
|
|
|
266
267
|
}
|
|
267
268
|
get size() { return this._entries.size; }
|
|
268
269
|
}
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
*
|
|
272
|
-
* A content hash here answers "did this chunk change", not "was this chunk
|
|
273
|
-
* tampered with", so a fast non-cryptographic hash is the right tool — and it is
|
|
274
|
-
* synchronous, unlike WebCrypto, which keeps chunking a pure function.
|
|
275
|
-
*/
|
|
276
|
-
export function hashText(text) {
|
|
277
|
-
let hash = 0x811c9dc5;
|
|
278
|
-
for (let i = 0; i < text.length; i++) {
|
|
279
|
-
hash ^= text.charCodeAt(i);
|
|
280
|
-
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
281
|
-
}
|
|
282
|
-
return hash.toString(16).padStart(8, '0');
|
|
283
|
-
}
|
|
270
|
+
// hashText moved to ./hash.ts (shared with the knowledge graph); re-exported for callers.
|
|
271
|
+
export { hashText } from './hash.js';
|
|
284
272
|
/** Bounded-concurrency map — the reason a large corpus does not become a 429. */
|
|
285
273
|
export async function mapWithConcurrency(items, limit, fn) {
|
|
286
274
|
let cursor = 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"IngestionPipeline.js","sourceRoot":"","sources":["../../src/ingest/IngestionPipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,SAAS,EAAqB,MAAM,uBAAuB,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EACH,SAAS,EACT,UAAU,EACV,iBAAiB,EACjB,YAAY,GAIf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EACH,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,eAAe,EACf,WAAW,EACX,SAAS,GAQZ,MAAM,YAAY,CAAC;AAwBpB,MAAM,OAAO,iBAAiB;IACT,MAAM,CAAc;IACpB,MAAM,CAAgB;IACtB,UAAU,CAAe;IACzB,QAAQ,CAAiB;IACzB,SAAS,CAA6B;IACtC,SAAS,CAAe;IACxB,WAAW,CAAS;IACpB,YAAY,CAAS;IACrB,YAAY,CAAS;IACrB,eAAe,CAAS;IACxB,OAAO,CAAqB;IAE7C,YAAY,IAA8B;QACtC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,YAAY,cAAc;YAClD,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,eAAe,IAAI,GAAG,CAAC,CAAC;QAC7D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,cAAc,IAAI,WAAW,CAAC;QAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAkC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAoB;YAC5B,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC9D,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;SACpE,CAAC;QAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,YAAY,EAAE;YAC/C,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,SAAS,EAAE;SACtF,CAAC,CAAC;QACH,IAAI,IAAI;YAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAExC,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAClE,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACnD,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;gBACtB,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC/B,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,gBAAgB,CAAC;gBACnD,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;YAC/C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,oEAAoE;gBACpE,+DAA+D;gBAC/D,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACjB,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACnB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1D,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACzC,IAAI,EAAE,aAAa,CAAC;YAChB,eAAe,EAAE,MAAM,CAAC,MAAM;YAC9B,iBAAiB,EAAE,MAAM,CAAC,QAAQ;YAClC,0BAA0B,EAAE,MAAM,CAAC,gBAAgB;YACnD,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;SAC5C,CAAC,CAAC;QACH,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjF,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,MAAM,CAAC,QAAgB;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;YAC7C,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ;SACpD,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvC,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,UAAU,CACpB,MAAsB,EACtB,MAA8C;QAK9C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,iBAAiB,EAAE;YACpD,IAAI,EAAE,QAAQ;YACd,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,UAAU,EAAE,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE;SAClF,CAAC,CAAC;QAEH,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YAEzE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YAEnD,sEAAsE;YACtE,4DAA4D;YAC5D,MAAM,OAAO,GAAkD,EAAE,CAAC;YAClE,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,QAAQ,GAAmB;wBAC7B,GAAG,IAAI;wBACP,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;wBAC3B,CAAC,iBAAiB,CAAC,EAAE,KAAK;wBAC1B,CAAC,kBAAkB,CAAC,EAAE,IAAI;qBAC7B,CAAC;oBACF,OAAO,CAAC,IAAI,CAAC;wBACT,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,EAAE,IAAI,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE;wBACnE,IAAI;qBACP,CAAC,CAAC;oBACH,KAAK,IAAI,CAAC,CAAC;gBACf,CAAC;YACL,CAAC;YAED,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAEhF,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC7E,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACxB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;oBAC1B,IAAI,MAAM;wBAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvC,CAAC,CAAC,CAAC;gBACH,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;YAC7B,CAAC;YAED,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACzD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;gBACjF,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;YAChC,CAAC;YAED,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACnC,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC;YACnC,CAAC;YAED,MAAM,IAAI,CAAC,SAAS,EAAE,GAAG,CACrB,MAAM,CAAC,EAAE,EACT,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAoB,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CACjF,CAAC;YAEF,IAAI,EAAE,aAAa,CAAC;gBAChB,iBAAiB,EAAE,QAAQ,CAAC,MAAM;gBAClC,eAAe,EAAE,OAAO,CAAC,MAAM;gBAC/B,iBAAiB,EAAE,QAAQ;gBAC3B,0BAA0B,EAAE,OAAO;gBACnC,eAAe,EAAE,MAAM,CAAC,EAAE;aAC7B,CAAC,CAAC;YACH,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAEhB,OAAO;gBACH,QAAQ,EAAE,QAAQ,CAAC,MAAM;gBACzB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,QAAQ;gBACR,QAAQ;gBACR,gBAAgB,EAAE,OAAO;gBACzB,YAAY;aACf,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,SAAS,CACnB,QAAgB,EAChB,OAAsD;QAEtD,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5D,MAAM,OAAO,GAAmB,EAAE,CAAC;YACnC,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,CAAC;gBACrC,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI;oBAAE,OAAO,IAAI,CAAC,CAAC;;oBACpD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5F,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC3B,KAAe,EACf,MAA8C;QAE9C,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE7C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,EAAE;YAC/C,IAAI,EAAE,WAAW;YACjB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,UAAU,EAAE,EAAE,sBAAsB,EAAE,KAAK,CAAC,MAAM,EAAE;SACvD,CAAC,CAAC;QACH,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,CAAC,WAAW,CAAC;gBACb,KAAK,EAAE,IAAI,CAAC,eAAe;gBAC3B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACjE,YAAY,EAAE,CAAC;gBACf,SAAS,EAAE,IAAI;aAClB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,OAAO,OAAO,CAAC;QACnB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAEO,aAAa,CAAC,MAAsB,EAAE,QAAgB;QAC1D,MAAM,QAAQ,GAAmB;YAC7B,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;YAC1B,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,QAAQ;YAC/B,sEAAsE;YACtE,sEAAsE;YACtE,kCAAkC;YAClC,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YAC3D,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC,EAAE;YAC5B,CAAC,YAAY,CAAC,EAAE,QAAQ;SAC3B,CAAC;QACF,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS;YAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QACvF,IAAI,MAAM,CAAC,KAAK;YAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;QACvD,IAAI,MAAM,CAAC,GAAG;YAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;QACjD,OAAO,QAAQ,CAAC;IACpB,CAAC;CACJ;AAED,iFAAiF;AACjF,MAAM,OAAO,sBAAsB;IACd,QAAQ,GAAG,IAAI,GAAG,EAA8B,CAAC;IAElE,KAAK,CAAC,GAAG,CAAC,QAAgB;QACtB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,QAAgB,EAAE,YAAgC;QACxD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB;QACzB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,IAAI,KAAa,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;CACpD;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACjC,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACpC,KAAmB,EACnB,KAAa,EACb,EAA6C;IAE7C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC7E,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;YACvB,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAM,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC/B,CAAC"}
|
|
1
|
+
{"version":3,"file":"IngestionPipeline.js","sourceRoot":"","sources":["../../src/ingest/IngestionPipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,SAAS,EAAqB,MAAM,uBAAuB,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EACH,SAAS,EACT,UAAU,EACV,iBAAiB,EACjB,YAAY,GAIf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EACH,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,eAAe,EACf,WAAW,EACX,SAAS,GAQZ,MAAM,YAAY,CAAC;AAwBpB,MAAM,OAAO,iBAAiB;IACT,MAAM,CAAc;IACpB,MAAM,CAAgB;IACtB,UAAU,CAAe;IACzB,QAAQ,CAAiB;IACzB,SAAS,CAA6B;IACtC,SAAS,CAAe;IACxB,WAAW,CAAS;IACpB,YAAY,CAAS;IACrB,YAAY,CAAS;IACrB,eAAe,CAAS;IACxB,OAAO,CAAqB;IAE7C,YAAY,IAA8B;QACtC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,YAAY,cAAc;YAClD,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,eAAe,IAAI,GAAG,CAAC,CAAC;QAC7D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,cAAc,IAAI,WAAW,CAAC;QAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAkC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAoB;YAC5B,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC9D,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;SACpE,CAAC;QAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,YAAY,EAAE;YAC/C,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,SAAS,EAAE;SACtF,CAAC,CAAC;QACH,IAAI,IAAI;YAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAExC,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAClE,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACnD,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;gBACtB,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC/B,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,gBAAgB,CAAC;gBACnD,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;YAC/C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,oEAAoE;gBACpE,+DAA+D;gBAC/D,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACjB,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACnB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1D,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACzC,IAAI,EAAE,aAAa,CAAC;YAChB,eAAe,EAAE,MAAM,CAAC,MAAM;YAC9B,iBAAiB,EAAE,MAAM,CAAC,QAAQ;YAClC,0BAA0B,EAAE,MAAM,CAAC,gBAAgB;YACnD,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;SAC5C,CAAC,CAAC;QACH,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjF,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,MAAM,CAAC,QAAgB;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;YAC7C,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ;SACpD,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvC,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,UAAU,CACpB,MAAsB,EACtB,MAA8C;QAK9C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,iBAAiB,EAAE;YACpD,IAAI,EAAE,QAAQ;YACd,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,UAAU,EAAE,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE;SAClF,CAAC,CAAC;QAEH,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YAEzE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YAEnD,sEAAsE;YACtE,4DAA4D;YAC5D,MAAM,OAAO,GAAkD,EAAE,CAAC;YAClE,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,QAAQ,GAAmB;wBAC7B,GAAG,IAAI;wBACP,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;wBAC3B,CAAC,iBAAiB,CAAC,EAAE,KAAK;wBAC1B,CAAC,kBAAkB,CAAC,EAAE,IAAI;qBAC7B,CAAC;oBACF,OAAO,CAAC,IAAI,CAAC;wBACT,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,EAAE,IAAI,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE;wBACnE,IAAI;qBACP,CAAC,CAAC;oBACH,KAAK,IAAI,CAAC,CAAC;gBACf,CAAC;YACL,CAAC;YAED,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAEhF,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC7E,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACxB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;oBAC1B,IAAI,MAAM;wBAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvC,CAAC,CAAC,CAAC;gBACH,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;YAC7B,CAAC;YAED,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACzD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;gBACjF,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;YAChC,CAAC;YAED,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACnC,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC;YACnC,CAAC;YAED,MAAM,IAAI,CAAC,SAAS,EAAE,GAAG,CACrB,MAAM,CAAC,EAAE,EACT,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAoB,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CACjF,CAAC;YAEF,IAAI,EAAE,aAAa,CAAC;gBAChB,iBAAiB,EAAE,QAAQ,CAAC,MAAM;gBAClC,eAAe,EAAE,OAAO,CAAC,MAAM;gBAC/B,iBAAiB,EAAE,QAAQ;gBAC3B,0BAA0B,EAAE,OAAO;gBACnC,eAAe,EAAE,MAAM,CAAC,EAAE;aAC7B,CAAC,CAAC;YACH,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAEhB,OAAO;gBACH,QAAQ,EAAE,QAAQ,CAAC,MAAM;gBACzB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,QAAQ;gBACR,QAAQ;gBACR,gBAAgB,EAAE,OAAO;gBACzB,YAAY;aACf,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,SAAS,CACnB,QAAgB,EAChB,OAAsD;QAEtD,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5D,MAAM,OAAO,GAAmB,EAAE,CAAC;YACnC,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,CAAC;gBACrC,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI;oBAAE,OAAO,IAAI,CAAC,CAAC;;oBACpD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5F,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC3B,KAAe,EACf,MAA8C;QAE9C,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE7C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,EAAE;YAC/C,IAAI,EAAE,WAAW;YACjB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,UAAU,EAAE,EAAE,sBAAsB,EAAE,KAAK,CAAC,MAAM,EAAE;SACvD,CAAC,CAAC;QACH,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,CAAC,WAAW,CAAC;gBACb,KAAK,EAAE,IAAI,CAAC,eAAe;gBAC3B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACjE,YAAY,EAAE,CAAC;gBACf,SAAS,EAAE,IAAI;aAClB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,OAAO,OAAO,CAAC;QACnB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAEO,aAAa,CAAC,MAAsB,EAAE,QAAgB;QAC1D,MAAM,QAAQ,GAAmB;YAC7B,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;YAC1B,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,QAAQ;YAC/B,sEAAsE;YACtE,sEAAsE;YACtE,kCAAkC;YAClC,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YAC3D,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC,EAAE;YAC5B,CAAC,YAAY,CAAC,EAAE,QAAQ;SAC3B,CAAC;QACF,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS;YAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QACvF,IAAI,MAAM,CAAC,KAAK;YAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;QACvD,IAAI,MAAM,CAAC,GAAG;YAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;QACjD,OAAO,QAAQ,CAAC;IACpB,CAAC;CACJ;AAED,iFAAiF;AACjF,MAAM,OAAO,sBAAsB;IACd,QAAQ,GAAG,IAAI,GAAG,EAA8B,CAAC;IAElE,KAAK,CAAC,GAAG,CAAC,QAAgB;QACtB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,QAAgB,EAAE,YAAgC;QACxD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB;QACzB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,IAAI,KAAa,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;CACpD;AAED,0FAA0F;AAC1F,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAErC,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACpC,KAAmB,EACnB,KAAa,EACb,EAA6C;IAE7C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC7E,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;YACvB,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAM,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC/B,CAAC"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest/docsExtract.ts — the knowledge-graph `extract` of the builtin text parsers
|
|
3
|
+
* (markdown, HTML, plain text). PRD 38 §4.5, the docs pack.
|
|
4
|
+
*
|
|
5
|
+
* Pure: a function of the source text, its root-relative path (`source.id`), the parsed
|
|
6
|
+
* sections and the extract context. Emits, all with `origin: 'extracted'`:
|
|
7
|
+
*
|
|
8
|
+
* • a `file` node, and a `section` node per heading breadcrumb, joined by `contains`;
|
|
9
|
+
* • `links` edges for relative markdown links, `[[wiki]]` links and HTML hrefs, resolved
|
|
10
|
+
* through `ctx.resolvePath` (an unresolvable target is skipped; a missing one stays an
|
|
11
|
+
* edge, so the graph can report it broken);
|
|
12
|
+
* • `mentions` edges where a section's normalized text contains a known entity key;
|
|
13
|
+
* • keys: path, basename, heading slugs and front-matter values.
|
|
14
|
+
*
|
|
15
|
+
* A section node carries `badges.section` (its index in the parser's output) so the review
|
|
16
|
+
* can hand exactly that text to judgement, and `badges.excerpt` for the index card.
|
|
17
|
+
*/
|
|
18
|
+
import type { ExtractContext, Extracted } from '../knowledge/model.js';
|
|
19
|
+
import { type ParsedSection, type SourceDocument } from './types.js';
|
|
20
|
+
export declare const DOCS_PACK_ID = "docs";
|
|
21
|
+
/** Badge: index of the section in the parser's output. */
|
|
22
|
+
export declare const SECTION_INDEX_BADGE = "section";
|
|
23
|
+
type Ctx = ExtractContext;
|
|
24
|
+
/** `key: value` and `- item` front matter between leading `---` fences. */
|
|
25
|
+
export declare function frontMatterOf(text: string): Record<string, string[]>;
|
|
26
|
+
export declare function extractMarkdown(source: SourceDocument, sections: ParsedSection[], ctx: Ctx): Extracted;
|
|
27
|
+
export declare function extractHtml(source: SourceDocument, sections: ParsedSection[], ctx: Ctx): Extracted;
|
|
28
|
+
export declare function extractPlainText(source: SourceDocument, sections: ParsedSection[], ctx: Ctx): Extracted;
|
|
29
|
+
export {};
|
|
30
|
+
//# sourceMappingURL=docsExtract.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"docsExtract.d.ts","sourceRoot":"","sources":["../../src/ingest/docsExtract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAyB,MAAM,uBAAuB,CAAC;AAG9F,OAAO,EAAsB,KAAK,aAAa,EAAE,KAAK,cAAc,EAAE,MAAM,YAAY,CAAC;AAEzF,eAAO,MAAM,YAAY,SAAS,CAAC;AACnC,0DAA0D;AAC1D,eAAO,MAAM,mBAAmB,YAAY,CAAC;AAI7C,KAAK,GAAG,GAAG,cAAc,CAAC;AA6F1B,2EAA2E;AAC3E,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAgBpE;AAYD,wBAAgB,eAAe,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,GAAG,GAAG,SAAS,CA8CtG;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,GAAG,GAAG,SAAS,CASlG;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,GAAG,GAAG,SAAS,CAQvG"}
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest/docsExtract.ts — the knowledge-graph `extract` of the builtin text parsers
|
|
3
|
+
* (markdown, HTML, plain text). PRD 38 §4.5, the docs pack.
|
|
4
|
+
*
|
|
5
|
+
* Pure: a function of the source text, its root-relative path (`source.id`), the parsed
|
|
6
|
+
* sections and the extract context. Emits, all with `origin: 'extracted'`:
|
|
7
|
+
*
|
|
8
|
+
* • a `file` node, and a `section` node per heading breadcrumb, joined by `contains`;
|
|
9
|
+
* • `links` edges for relative markdown links, `[[wiki]]` links and HTML hrefs, resolved
|
|
10
|
+
* through `ctx.resolvePath` (an unresolvable target is skipped; a missing one stays an
|
|
11
|
+
* edge, so the graph can report it broken);
|
|
12
|
+
* • `mentions` edges where a section's normalized text contains a known entity key;
|
|
13
|
+
* • keys: path, basename, heading slugs and front-matter values.
|
|
14
|
+
*
|
|
15
|
+
* A section node carries `badges.section` (its index in the parser's output) so the review
|
|
16
|
+
* can hand exactly that text to judgement, and `badges.excerpt` for the index card.
|
|
17
|
+
*/
|
|
18
|
+
import { basenameOf, compareStr, normalizeKey, slugOf, srcRef } from '../knowledge/refs.js';
|
|
19
|
+
import { hashText } from './hash.js';
|
|
20
|
+
import { CONTENT_HASH_FIELD } from './types.js';
|
|
21
|
+
export const DOCS_PACK_ID = 'docs';
|
|
22
|
+
/** Badge: index of the section in the parser's output. */
|
|
23
|
+
export const SECTION_INDEX_BADGE = 'section';
|
|
24
|
+
const EXCERPT_CHARS = 400;
|
|
25
|
+
const BREADCRUMB_SEPARATOR = ' › ';
|
|
26
|
+
const textOf = (source) => (source.content.kind === 'text' ? source.content.text : '');
|
|
27
|
+
class ExtractBuilder {
|
|
28
|
+
ctx;
|
|
29
|
+
path;
|
|
30
|
+
hash;
|
|
31
|
+
fileRef;
|
|
32
|
+
nodes = new Map();
|
|
33
|
+
edges = new Map();
|
|
34
|
+
keys = new Map();
|
|
35
|
+
constructor(source, ctx, label) {
|
|
36
|
+
this.ctx = ctx;
|
|
37
|
+
this.path = source.id;
|
|
38
|
+
const stated = source.metadata?.[CONTENT_HASH_FIELD];
|
|
39
|
+
this.hash = typeof stated === 'string' ? stated : hashText(textOf(source));
|
|
40
|
+
this.fileRef = srcRef(ctx.rootId, this.path);
|
|
41
|
+
const base = basenameOf(this.path);
|
|
42
|
+
this.node(this.fileRef, 'file', label || base);
|
|
43
|
+
this.key(this.fileRef, this.path);
|
|
44
|
+
this.key(this.fileRef, base.toLowerCase());
|
|
45
|
+
this.key(this.fileRef, normalizeKey(base.replace(/\.[^.]+$/, '')));
|
|
46
|
+
}
|
|
47
|
+
node(ref, kind, label, badges) {
|
|
48
|
+
const node = {
|
|
49
|
+
ref, kind, label, pack: DOCS_PACK_ID, importance: 0, origin: 'extracted', sourceHash: this.hash,
|
|
50
|
+
...(badges ? { badges } : {}),
|
|
51
|
+
};
|
|
52
|
+
this.nodes.set(ref, node);
|
|
53
|
+
return node;
|
|
54
|
+
}
|
|
55
|
+
edge(from, to, kind) {
|
|
56
|
+
if (from === to)
|
|
57
|
+
return;
|
|
58
|
+
const id = `${from}\u0000${to}\u0000${kind}`;
|
|
59
|
+
const prior = this.edges.get(id);
|
|
60
|
+
if (prior)
|
|
61
|
+
prior.weight += 1;
|
|
62
|
+
else
|
|
63
|
+
this.edges.set(id, { from, to, kind, weight: 1, origin: 'extracted' });
|
|
64
|
+
}
|
|
65
|
+
key(ref, key) {
|
|
66
|
+
if (key)
|
|
67
|
+
this.keys.set(`${ref}\u0000${key}`, { ref, key });
|
|
68
|
+
}
|
|
69
|
+
/** `links` edges to each resolvable target. */
|
|
70
|
+
link(from, target) {
|
|
71
|
+
const clean = target.split('#')[0].split('?')[0].trim();
|
|
72
|
+
if (!clean || /^[a-z][a-z0-9+.-]*:/i.test(clean))
|
|
73
|
+
return;
|
|
74
|
+
let decoded = clean;
|
|
75
|
+
try {
|
|
76
|
+
decoded = decodeURI(clean);
|
|
77
|
+
}
|
|
78
|
+
catch { /* keep the raw target */ }
|
|
79
|
+
const resolved = this.ctx.resolvePath(this.path, decoded);
|
|
80
|
+
if (resolved !== undefined && resolved !== this.path)
|
|
81
|
+
this.edge(from, srcRef(this.ctx.rootId, resolved), 'links');
|
|
82
|
+
}
|
|
83
|
+
wikiLinks(from, text) {
|
|
84
|
+
for (const m of text.matchAll(/\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|[^\]]*)?\]\]/g)) {
|
|
85
|
+
const name = m[1].trim();
|
|
86
|
+
if (name)
|
|
87
|
+
this.link(from, /\.[A-Za-z][A-Za-z0-9]{0,7}$/.test(name) ? name : `${name}.md`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
/** `mentions` edges where the normalized text contains a known key as whole words. */
|
|
91
|
+
mentions(from, text) {
|
|
92
|
+
if (this.ctx.knownKeys.size === 0)
|
|
93
|
+
return;
|
|
94
|
+
const padded = ` ${normalizeKey(text)} `;
|
|
95
|
+
const known = [...this.ctx.knownKeys.entries()].sort((a, b) => compareStr(a[0], b[0]));
|
|
96
|
+
for (const [key, ref] of known) {
|
|
97
|
+
if (key && padded.includes(` ${key} `))
|
|
98
|
+
this.edge(from, ref, 'mentions');
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/** A whole-body `section` (HTML, plain text): judged like a markdown section. */
|
|
102
|
+
bodySection(index, text) {
|
|
103
|
+
const ref = `${this.fileRef}#${index === 0 ? 'body' : `body-${index + 1}`}`;
|
|
104
|
+
this.node(ref, 'section', basenameOf(this.path), { [SECTION_INDEX_BADGE]: index, excerpt: excerpt(text) });
|
|
105
|
+
this.edge(this.fileRef, ref, 'contains');
|
|
106
|
+
return ref;
|
|
107
|
+
}
|
|
108
|
+
build() {
|
|
109
|
+
const byRef = (a, b) => compareStr(a.ref, b.ref);
|
|
110
|
+
return {
|
|
111
|
+
sourceRef: this.fileRef,
|
|
112
|
+
sourceHash: this.hash,
|
|
113
|
+
nodes: [...this.nodes.values()].sort(byRef),
|
|
114
|
+
edges: [...this.edges.values()].sort((a, b) => compareStr(a.from, b.from) || compareStr(a.to, b.to) || compareStr(a.kind, b.kind)),
|
|
115
|
+
keys: [...this.keys.values()].sort((a, b) => byRef(a, b) || compareStr(a.key, b.key)),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/** `key: value` and `- item` front matter between leading `---` fences. */
|
|
120
|
+
export function frontMatterOf(text) {
|
|
121
|
+
const out = {};
|
|
122
|
+
const m = /^---\r?\n([\s\S]*?)\r?\n---\s*(?:\r?\n|$)/.exec(text);
|
|
123
|
+
if (!m)
|
|
124
|
+
return out;
|
|
125
|
+
let current;
|
|
126
|
+
for (const line of m[1].split(/\r?\n/)) {
|
|
127
|
+
const item = /^\s+-\s+(.*)$/.exec(line);
|
|
128
|
+
if (item && current) {
|
|
129
|
+
out[current].push(unquote(item[1]));
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
const kv = /^([A-Za-z0-9_.-]+)\s*:\s*(.*)$/.exec(line);
|
|
133
|
+
if (!kv)
|
|
134
|
+
continue;
|
|
135
|
+
current = kv[1];
|
|
136
|
+
const raw = kv[2].trim();
|
|
137
|
+
const list = /^\[(.*)\]$/.exec(raw);
|
|
138
|
+
out[current] = list ? list[1].split(',').map(unquote).filter(Boolean) : raw ? [unquote(raw)] : [];
|
|
139
|
+
}
|
|
140
|
+
return out;
|
|
141
|
+
}
|
|
142
|
+
function unquote(s) {
|
|
143
|
+
return s.trim().replace(/^["']|["']$/g, '').trim();
|
|
144
|
+
}
|
|
145
|
+
function excerpt(text) {
|
|
146
|
+
return text.replace(/\s+/g, ' ').trim().slice(0, EXCERPT_CHARS);
|
|
147
|
+
}
|
|
148
|
+
const MD_LINK = /(?<!!)\[[^\]]*\]\(\s*<?([^)\s>]+)>?(?:\s+"[^"]*")?\s*\)/g;
|
|
149
|
+
export function extractMarkdown(source, sections, ctx) {
|
|
150
|
+
const b = new ExtractBuilder(source, ctx);
|
|
151
|
+
const fm = frontMatterOf(textOf(source));
|
|
152
|
+
const fileBadges = {};
|
|
153
|
+
for (const k of Object.keys(fm).sort(compareStr)) {
|
|
154
|
+
fileBadges[`fm.${k}`] = fm[k].join(', ');
|
|
155
|
+
for (const v of fm[k])
|
|
156
|
+
b.key(b.fileRef, normalizeKey(v));
|
|
157
|
+
}
|
|
158
|
+
if (Object.keys(fileBadges).length)
|
|
159
|
+
b.node(b.fileRef, 'file', fm['title']?.[0] || basenameOf(b.path), fileBadges);
|
|
160
|
+
const refByCrumb = new Map();
|
|
161
|
+
const occurrences = new Map();
|
|
162
|
+
const sectionNode = (crumbs, index, text) => {
|
|
163
|
+
const slugPath = crumbs.map(slugOf).join('/');
|
|
164
|
+
const n = (occurrences.get(slugPath) ?? 0) + 1;
|
|
165
|
+
occurrences.set(slugPath, n);
|
|
166
|
+
const ref = `${b.fileRef}#${n > 1 ? `${slugPath}-${n}` : slugPath}`;
|
|
167
|
+
const badges = { depth: crumbs.length, breadcrumb: crumbs.join(BREADCRUMB_SEPARATOR) };
|
|
168
|
+
if (index !== undefined)
|
|
169
|
+
badges[SECTION_INDEX_BADGE] = index;
|
|
170
|
+
if (text)
|
|
171
|
+
badges['excerpt'] = excerpt(text);
|
|
172
|
+
b.node(ref, 'section', crumbs[crumbs.length - 1], badges);
|
|
173
|
+
b.key(ref, slugOf(crumbs[crumbs.length - 1]));
|
|
174
|
+
const parent = crumbs.length > 1 ? refByCrumb.get(crumbs.slice(0, -1).join('\u0000')) : b.fileRef;
|
|
175
|
+
b.edge(parent ?? b.fileRef, ref, 'contains');
|
|
176
|
+
refByCrumb.set(crumbs.join('\u0000'), ref);
|
|
177
|
+
return ref;
|
|
178
|
+
};
|
|
179
|
+
sections.forEach((section, index) => {
|
|
180
|
+
const headingPath = section.metadata?.['headingPath'];
|
|
181
|
+
let from = b.fileRef;
|
|
182
|
+
if (typeof headingPath === 'string' && headingPath) {
|
|
183
|
+
const crumbs = headingPath.split(BREADCRUMB_SEPARATOR);
|
|
184
|
+
// Headings with no body of their own are not emitted by the parser; recover them.
|
|
185
|
+
for (let k = 1; k < crumbs.length; k++) {
|
|
186
|
+
if (!refByCrumb.has(crumbs.slice(0, k).join('\u0000')))
|
|
187
|
+
sectionNode(crumbs.slice(0, k));
|
|
188
|
+
}
|
|
189
|
+
from = sectionNode(crumbs, index, section.text);
|
|
190
|
+
}
|
|
191
|
+
for (const m of section.text.matchAll(MD_LINK)) {
|
|
192
|
+
if (!m[1].startsWith('#'))
|
|
193
|
+
b.link(from, m[1]);
|
|
194
|
+
}
|
|
195
|
+
b.wikiLinks(from, section.text);
|
|
196
|
+
b.mentions(from, section.text);
|
|
197
|
+
});
|
|
198
|
+
return b.build();
|
|
199
|
+
}
|
|
200
|
+
export function extractHtml(source, sections, ctx) {
|
|
201
|
+
const title = sections[0]?.metadata?.['htmlTitle'];
|
|
202
|
+
const b = new ExtractBuilder(source, ctx, typeof title === 'string' ? title : undefined);
|
|
203
|
+
if (typeof title === 'string' && title)
|
|
204
|
+
b.key(b.fileRef, normalizeKey(title));
|
|
205
|
+
for (const m of textOf(source).matchAll(/<a\s[^>]*href\s*=\s*["']([^"']+)["']/gi)) {
|
|
206
|
+
if (!m[1].startsWith('#'))
|
|
207
|
+
b.link(b.fileRef, m[1]);
|
|
208
|
+
}
|
|
209
|
+
sections.forEach((section, index) => b.mentions(b.bodySection(index, section.text), section.text));
|
|
210
|
+
return b.build();
|
|
211
|
+
}
|
|
212
|
+
export function extractPlainText(source, sections, ctx) {
|
|
213
|
+
const b = new ExtractBuilder(source, ctx);
|
|
214
|
+
sections.forEach((section, index) => {
|
|
215
|
+
const from = b.bodySection(index, section.text);
|
|
216
|
+
b.wikiLinks(from, section.text);
|
|
217
|
+
b.mentions(from, section.text);
|
|
218
|
+
});
|
|
219
|
+
return b.build();
|
|
220
|
+
}
|
|
221
|
+
//# sourceMappingURL=docsExtract.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"docsExtract.js","sourceRoot":"","sources":["../../src/ingest/docsExtract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC5F,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,kBAAkB,EAA2C,MAAM,YAAY,CAAC;AAEzF,MAAM,CAAC,MAAM,YAAY,GAAG,MAAM,CAAC;AACnC,0DAA0D;AAC1D,MAAM,CAAC,MAAM,mBAAmB,GAAG,SAAS,CAAC;AAC7C,MAAM,aAAa,GAAG,GAAG,CAAC;AAC1B,MAAM,oBAAoB,GAAG,KAAK,CAAC;AAInC,MAAM,MAAM,GAAG,CAAC,MAAsB,EAAU,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAE/G,MAAM,cAAc;IAQqC;IAP5C,IAAI,CAAS;IACb,IAAI,CAAS;IACb,OAAO,CAAU;IACT,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IAClC,KAAK,GAAG,IAAI,GAAG,EAAiB,CAAC;IACjC,IAAI,GAAG,IAAI,GAAG,EAAyC,CAAC;IAEzE,YAAY,MAAsB,EAAmB,GAAQ,EAAE,KAAc;QAAxB,QAAG,GAAH,GAAG,CAAK;QACzD,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC;QACtB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,kBAAkB,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,IAAI,CAAC,GAAY,EAAE,IAAY,EAAE,KAAa,EAAE,MAAwC;QACpF,MAAM,IAAI,GAAU;YAChB,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,YAAY,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI;YAC/F,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChC,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,IAAI,CAAC,IAAa,EAAE,EAAW,EAAE,IAAY;QACzC,IAAI,IAAI,KAAK,EAAE;YAAE,OAAO;QACxB,MAAM,EAAE,GAAG,GAAG,IAAI,SAAS,EAAE,SAAS,IAAI,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACjC,IAAI,KAAK;YAAE,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;;YACxB,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,GAAG,CAAC,GAAY,EAAE,GAAW;QACzB,IAAI,GAAG;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,SAAS,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;IAC/D,CAAC;IAED,+CAA+C;IAC/C,IAAI,CAAC,IAAa,EAAE,MAAc;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;QAC1D,IAAI,CAAC,KAAK,IAAI,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO;QACzD,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC;YAAC,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,yBAAyB,CAAC,CAAC;QACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1D,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,CAAC,IAAI;YAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;IACtH,CAAC;IAED,SAAS,CAAC,IAAa,EAAE,IAAY;QACjC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,+CAA+C,CAAC,EAAE,CAAC;YAC7E,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,IAAI;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC;QAC9F,CAAC;IACL,CAAC;IAED,sFAAsF;IACtF,QAAQ,CAAC,IAAa,EAAE,IAAY;QAChC,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC;YAAE,OAAO;QAC1C,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QACzC,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvF,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,GAAG,GAAG,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;QAC7E,CAAC;IACL,CAAC;IAED,iFAAiF;IACjF,WAAW,CAAC,KAAa,EAAE,IAAY;QACnC,MAAM,GAAG,GAAY,GAAG,IAAI,CAAC,OAAO,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,KAAK,GAAG,CAAC,EAAE,EAAE,CAAC;QACrF,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,mBAAmB,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC3G,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;QACzC,OAAO,GAAG,CAAC;IACf,CAAC;IAED,KAAK;QACD,MAAM,KAAK,GAAG,CAA6B,CAAI,EAAE,CAAI,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QACnF,OAAO;YACH,SAAS,EAAE,IAAI,CAAC,OAAO;YACvB,UAAU,EAAE,IAAI,CAAC,IAAI;YACrB,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YAC3C,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC1C,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YACvF,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;SACxF,CAAC;IACN,CAAC;CACJ;AAED,2EAA2E;AAC3E,MAAM,UAAU,aAAa,CAAC,IAAY;IACtC,MAAM,GAAG,GAA6B,EAAE,CAAC;IACzC,MAAM,CAAC,GAAG,2CAA2C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjE,IAAI,CAAC,CAAC;QAAE,OAAO,GAAG,CAAC;IACnB,IAAI,OAA2B,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,IAAI,IAAI,OAAO,EAAE,CAAC;YAAC,GAAG,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC;YAAC,SAAS;QAAC,CAAC;QACzE,MAAM,EAAE,GAAG,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,IAAI,CAAC,EAAE;YAAE,SAAS;QAClB,OAAO,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC;QACjB,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,GAAG,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACvG,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACtB,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;AACvD,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IACzB,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,OAAO,GAAG,0DAA0D,CAAC;AAE3E,MAAM,UAAU,eAAe,CAAC,MAAsB,EAAE,QAAyB,EAAE,GAAQ;IACvF,MAAM,CAAC,GAAG,IAAI,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1C,MAAM,EAAE,GAAG,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACzC,MAAM,UAAU,GAA2B,EAAE,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/C,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAE;YAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM;QAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;IAElH,MAAM,UAAU,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC9C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,MAAM,WAAW,GAAG,CAAC,MAAgB,EAAE,KAAc,EAAE,IAAa,EAAW,EAAE;QAC7E,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/C,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;QAC7B,MAAM,GAAG,GAAY,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;QAC7E,MAAM,MAAM,GAAoC,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,EAAE,CAAC;QACxH,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC;QAC7D,IAAI,IAAI;YAAE,MAAM,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC5C,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,EAAE,MAAM,CAAC,CAAC;QAC3D,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QAClG,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC;QAC7C,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC;QAC3C,OAAO,GAAG,CAAC;IACf,CAAC,CAAC;IAEF,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAChC,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,aAAa,CAAC,CAAC;QACtD,IAAI,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC;QACrB,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,WAAW,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACvD,kFAAkF;YAClF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;oBAAE,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAC5F,CAAC;YACD,IAAI,GAAG,WAAW,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;QACpD,CAAC;QACD,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,MAAsB,EAAE,QAAyB,EAAE,GAAQ;IACnF,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,WAAW,CAAC,CAAC;IACnD,MAAM,CAAC,GAAG,IAAI,cAAc,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACzF,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK;QAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9E,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,wCAAwC,CAAC,EAAE,CAAC;QAChF,IAAI,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IACzD,CAAC;IACD,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IACnG,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAsB,EAAE,QAAyB,EAAE,GAAQ;IACxF,MAAM,CAAC,GAAG,IAAI,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1C,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;QAChC,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAChD,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAChC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC;AACrB,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest/hash.ts — the content hash every ingest consumer shares (the pipeline's chunk
|
|
3
|
+
* hashes, the knowledge graph's extracted output and card hashes). A leaf module, so
|
|
4
|
+
* `ingest/docsExtract.ts` can use it without an import cycle through the pipeline.
|
|
5
|
+
* (SHA-256 digests live in `retrieval/knowledge-index/sha256.ts`.)
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* FNV-1a, 32-bit, rendered as hex.
|
|
9
|
+
*
|
|
10
|
+
* A content hash here answers "did this chunk change", not "was this chunk
|
|
11
|
+
* tampered with", so a fast non-cryptographic hash is the right tool — and it is
|
|
12
|
+
* synchronous, unlike WebCrypto, which keeps chunking a pure function.
|
|
13
|
+
*/
|
|
14
|
+
export declare function hashText(text: string): string;
|
|
15
|
+
//# sourceMappingURL=hash.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../../src/ingest/hash.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAO7C"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ingest/hash.ts — the content hash every ingest consumer shares (the pipeline's chunk
|
|
3
|
+
* hashes, the knowledge graph's extracted output and card hashes). A leaf module, so
|
|
4
|
+
* `ingest/docsExtract.ts` can use it without an import cycle through the pipeline.
|
|
5
|
+
* (SHA-256 digests live in `retrieval/knowledge-index/sha256.ts`.)
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* FNV-1a, 32-bit, rendered as hex.
|
|
9
|
+
*
|
|
10
|
+
* A content hash here answers "did this chunk change", not "was this chunk
|
|
11
|
+
* tampered with", so a fast non-cryptographic hash is the right tool — and it is
|
|
12
|
+
* synchronous, unlike WebCrypto, which keeps chunking a pure function.
|
|
13
|
+
*/
|
|
14
|
+
export function hashText(text) {
|
|
15
|
+
let hash = 0x811c9dc5;
|
|
16
|
+
for (let i = 0; i < text.length; i++) {
|
|
17
|
+
hash ^= text.charCodeAt(i);
|
|
18
|
+
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
19
|
+
}
|
|
20
|
+
return hash.toString(16).padStart(8, '0');
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=hash.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hash.js","sourceRoot":"","sources":["../../src/ingest/hash.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACjC,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parsers.d.ts","sourceRoot":"","sources":["../../src/ingest/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"parsers.d.ts","sourceRoot":"","sources":["../../src/ingest/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,cAAc,EAAiB,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAU/F;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,aAAa,GAAG,MAAM,CAKvD;AAED,eAAO,MAAM,UAAU,EAAE,cAexB,CAAC;AAIF;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,aAAa,EAAE,CA4CvE;AAmBD,eAAO,MAAM,SAAS,EAAE,cAWvB,CAAC;AAIF;;;GAGG;AACH,eAAO,MAAM,UAAU,EAAE,cA2BxB,CAAC;AAmBF,eAAO,MAAM,cAAc,EAAE,cA6C5B,CAAC;AAIF,eAAO,MAAM,UAAU,EAAE,cA8BxB,CAAC;AAIF,eAAO,MAAM,eAAe,EAAE,cAQ7B,CAAC;AAIF,wEAAwE;AACxE,eAAO,MAAM,eAAe,EAAE,SAAS,cAAc,EAOpD,CAAC;AAEF,qBAAa,cAAc;IACvB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmB;gBAEhC,OAAO,GAAE,SAAS,cAAc,EAAoB;IAIhE,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI;IAKtC,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,GAAG,SAAS;IAI3D,IAAI,IAAI,MAAM,EAAE;CAGnB"}
|
package/dist/ingest/parsers.js
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
* retriever to recover it statistically — the single most common reason a RAG
|
|
16
16
|
* pilot answers plausibly and cites nothing verifiable.
|
|
17
17
|
*/
|
|
18
|
+
import { extractHtml, extractMarkdown, extractPlainText } from './docsExtract.js';
|
|
18
19
|
const textOf = (source) => source.content.kind === 'text' ? source.content.text : '';
|
|
19
20
|
const mediaTypeOf = (source) => source.content.kind === 'text' ? (source.content.mediaType ?? 'text/plain').toLowerCase() : '';
|
|
20
21
|
// ── structured rows ───────────────────────────────────────────────────────────
|
|
@@ -235,6 +236,7 @@ export const markdownParser = {
|
|
|
235
236
|
flush();
|
|
236
237
|
return sections;
|
|
237
238
|
},
|
|
239
|
+
extract: extractMarkdown,
|
|
238
240
|
};
|
|
239
241
|
// ── HTML ──────────────────────────────────────────────────────────────────────
|
|
240
242
|
export const htmlParser = {
|
|
@@ -266,6 +268,7 @@ export const htmlParser = {
|
|
|
266
268
|
metadata['htmlTitle'] = titleMatch[1].trim();
|
|
267
269
|
return [{ text, metadata }];
|
|
268
270
|
},
|
|
271
|
+
extract: extractHtml,
|
|
269
272
|
};
|
|
270
273
|
// ── plain text ────────────────────────────────────────────────────────────────
|
|
271
274
|
export const plainTextParser = {
|
|
@@ -275,6 +278,7 @@ export const plainTextParser = {
|
|
|
275
278
|
const text = textOf(source).trim();
|
|
276
279
|
return text ? [{ text }] : [];
|
|
277
280
|
},
|
|
281
|
+
extract: extractPlainText,
|
|
278
282
|
};
|
|
279
283
|
// ── registry ──────────────────────────────────────────────────────────────────
|
|
280
284
|
/** Order matters: the first parser that accepts a source handles it. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../src/ingest/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../src/ingest/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGlF,MAAM,MAAM,GAAG,CAAC,MAAsB,EAAU,EAAE,CAC9C,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;AAE9D,MAAM,WAAW,GAAG,CAAC,MAAsB,EAAU,EAAE,CACnD,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,IAAI,YAAY,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AAEnG,iFAAiF;AAEjF;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,GAAkB;IAC3C,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC;SACrB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,EAAE,CAAC;SAC5D,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;SACvE,IAAI,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAmB;IACtC,EAAE,EAAE,MAAM;IACV,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;IACnD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,EAAE,CAAC;QAC9C,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;QAC5C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YAC3B,MAAM,QAAQ,GAAmB,EAAE,GAAG,GAAG,EAAE,CAAC;YAC5C,QAAQ,CAAC,UAAU,CAAC,GAAG,KAAK,CAAC;YAC7B,IAAI,UAAU,IAAI,GAAG,CAAC,UAAU,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC9C,QAAQ,CAAC,OAAO,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;YAChD,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,GAAG,CAAC,EAAE,QAAQ,EAAE,CAAC;QACjD,CAAC,CAAC,CAAC;IACP,CAAC;CACJ,CAAC;AAEF,iFAAiF;AAEjF;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY,EAAE,SAAS,GAAG,GAAG;IAClD,MAAM,IAAI,GAAe,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,IAAI,MAAM,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAW,CAAC;QAC7B,IAAI,MAAM,EAAE,CAAC;YACT,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBAAC,KAAK,IAAI,GAAG,CAAC;oBAAC,CAAC,EAAE,CAAC;gBAAC,CAAC;;oBAC1C,MAAM,GAAG,KAAK,CAAC;YACxB,CAAC;;gBAAM,KAAK,IAAI,EAAE,CAAC;YACnB,SAAS;QACb,CAAC;QACD,IAAI,EAAE,KAAK,GAAG,EAAE,CAAC;YAAC,MAAM,GAAG,IAAI,CAAC;YAAC,SAAS;QAAC,CAAC;QAC5C,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAAC,KAAK,GAAG,EAAE,CAAC;YAAC,SAAS;QAAC,CAAC;QACnE,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7B,IAAI,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,IAAI;gBAAE,CAAC,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAClB,MAAM,GAAG,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,CAAC;YACX,SAAS;QACb,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;IAChB,CAAC;IACD,IAAI,KAAK,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC5B,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,CAAC;IAEvB,OAAO,IAAI;SACN,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SACnD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACX,MAAM,GAAG,GAAkB,EAAE,CAAC;QAC9B,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;YACvB,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;QACH,OAAO,GAAG,CAAC;IACf,CAAC,CAAC,CAAC;AACX,CAAC;AAED;;;;GAIG;AACH,SAAS,MAAM,CAAC,GAAW;IACvB,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACzB,IAAI,KAAK,KAAK,EAAE;QAAE,OAAO,EAAE,CAAC;IAC5B,IAAI,KAAK,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAClC,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC;IACpC,IAAI,0BAA0B,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QACxB,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAmB;IACrC,EAAE,EAAE,KAAK;IACT,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE;QAChB,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC;IAClE,CAAC;IACD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;QAC7E,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,UAAU,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IAC5E,CAAC;CACJ,CAAC;AAEF,iFAAiF;AAEjF;;;GAGG;AACH,MAAM,CAAC,MAAM,UAAU,GAAmB;IACtC,EAAE,EAAE,MAAM;IACV,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;IACzD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACD,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACL,wEAAwE;YACxE,wEAAwE;YACxE,OAAO,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAChG,OAAO,UAAU,CAAC,KAAK,CAAC;gBACpB,GAAG,MAAM;gBACT,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAyB,EAAE;aAC7D,CAAC,CAAC;QACP,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACnC,OAAO,CAAC;gBACJ,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;gBACnE,QAAQ,EAAE,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE;aACzC,CAAC,CAAC;IACP,CAAC;CACJ,CAAC;AAEF,SAAS,OAAO,CAAC,KAAc,EAAE,IAAY,EAAE,MAA+B,EAAE;IAC5E,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9C,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACvC,OAAO,GAAG,CAAC;IACf,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC;QACzD,OAAO,GAAG,CAAC;IACf,CAAC;IACD,KAAK,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;QACtE,OAAO,CAAC,CAAC,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,GAAG,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC;AAED,iFAAiF;AAEjF,MAAM,CAAC,MAAM,cAAc,GAAmB;IAC1C,EAAE,EAAE,UAAU;IACd,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE;QAChB,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;QACjC,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC7D,CAAC;IACD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAoB,EAAE,CAAC;QACrC,4EAA4E;QAC5E,4BAA4B;QAC5B,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,IAAI,MAAM,GAAa,EAAE,CAAC;QAE1B,MAAM,KAAK,GAAG,GAAG,EAAE;YACf,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,GAAG,EAAE,CAAC;YACZ,IAAI,CAAC,IAAI;gBAAE,OAAO;YAClB,MAAM,QAAQ,GAAmB,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,QAAQ,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC3C,QAAQ,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAW,CAAC;gBACtD,QAAQ,CAAC,cAAc,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;YAC3C,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;QACtC,CAAC,CAAC;QAEF,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,GAAG,CAAC,OAAO,CAAC;YAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChE,IAAI,OAAO,EAAE,CAAC;gBACV,KAAK,EAAE,CAAC;gBACR,MAAM,KAAK,GAAI,OAAO,CAAC,CAAC,CAAY,CAAC,MAAM,CAAC;gBAC5C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;gBAC/C,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAI,OAAO,CAAC,CAAC,CAAY,CAAC,IAAI,EAAE,CAAC;gBAChD,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;gBACpB,SAAS;YACb,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QACD,KAAK,EAAE,CAAC;QACR,OAAO,QAAQ,CAAC;IACpB,CAAC;IACD,OAAO,EAAE,eAAe;CAC3B,CAAC;AAEF,iFAAiF;AAEjF,MAAM,CAAC,MAAM,UAAU,GAAmB;IACtC,EAAE,EAAE,MAAM;IACV,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC;IAChG,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3B,MAAM,UAAU,GAAG,kCAAkC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEhE,MAAM,IAAI,GAAG,GAAG;YACZ,wEAAwE;aACvE,OAAO,CAAC,uCAAuC,EAAE,GAAG,CAAC;aACrD,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC;aAChC,OAAO,CAAC,4CAA4C,EAAE,IAAI,CAAC;aAC3D,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC;aAC7B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;aACxB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;aACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;aACtB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aAC1B,IAAI,EAAE,CAAC;QAEZ,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,IAAI,UAAU,EAAE,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAClE,OAAO,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,EAAE,WAAW;CACvB,CAAC;AAEF,iFAAiF;AAEjF,MAAM,CAAC,MAAM,eAAe,GAAmB;IAC3C,EAAE,EAAE,MAAM;IACV,OAAO,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,KAAK,MAAM;IACnD,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACd,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClC,CAAC;IACD,OAAO,EAAE,gBAAgB;CAC5B,CAAC;AAEF,iFAAiF;AAEjF,wEAAwE;AACxE,MAAM,CAAC,MAAM,eAAe,GAA8B;IACtD,UAAU;IACV,SAAS;IACT,UAAU;IACV,cAAc;IACd,UAAU;IACV,eAAe;CAClB,CAAC;AAEF,MAAM,OAAO,cAAc;IACN,QAAQ,CAAmB;IAE5C,YAAY,UAAqC,eAAe;QAC5D,IAAI,CAAC,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;IACjC,CAAC;IAED,qEAAqE;IACrE,QAAQ,CAAC,MAAsB;QAC3B,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC9B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,OAAO,CAAC,MAAsB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,IAAI;QACA,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC;CACJ"}
|
package/dist/ingest/types.d.ts
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
* embedding has to recover.
|
|
11
11
|
*/
|
|
12
12
|
import type { MetadataRecord, MetadataValue } from '../vectorstore/types.js';
|
|
13
|
+
import type { ExtractContext, Extracted } from '../knowledge/model.js';
|
|
13
14
|
/** One row of a structured source (a DB result set, a CSV, a spreadsheet). */
|
|
14
15
|
export type StructuredRow = Record<string, MetadataValue>;
|
|
15
16
|
export type SourceContent = {
|
|
@@ -57,6 +58,12 @@ export interface DocumentParser {
|
|
|
57
58
|
/** True when this parser handles the source. First match in registry order wins. */
|
|
58
59
|
accepts(source: SourceDocument): boolean;
|
|
59
60
|
parse(source: SourceDocument): ParsedSection[];
|
|
61
|
+
/**
|
|
62
|
+
* Optional, pure: the knowledge-graph structure of a parsed source (file/section nodes,
|
|
63
|
+
* links, mentions, keys). Perception, never memory — recomputed from the source and
|
|
64
|
+
* cached by content hash. Parsers without it contribute a bare file node.
|
|
65
|
+
*/
|
|
66
|
+
extract?(source: SourceDocument, sections: ParsedSection[], ctx: ExtractContext): Extracted;
|
|
60
67
|
}
|
|
61
68
|
/** Identity of one stored chunk, for incremental sync. */
|
|
62
69
|
export interface ChunkFingerprint {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/ingest/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/ingest/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAC7E,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEvE,8EAA8E;AAC9E,MAAM,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;AAE1D,MAAM,MAAM,aAAa,GACnB;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAClD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,aAAa,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnE,MAAM,WAAW,cAAc;IAC3B,oFAAoF;IACpF,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,aAAa,CAAC;IACvB,wEAAwE;IACxE,QAAQ,EAAE,MAAM,CAAC;IACjB;;;;OAIG;IACH,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,yEAAyE;IACzE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,cAAc,CAAC;IAC1B,6CAA6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,GAAG,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8EAA8E;AAC9E,MAAM,WAAW,aAAa;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,sEAAsE;IACtE,QAAQ,CAAC,EAAE,cAAc,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC3B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,oFAAoF;IACpF,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC;IACzC,KAAK,CAAC,MAAM,EAAE,cAAc,GAAG,aAAa,EAAE,CAAC;IAC/C;;;;OAIG;IACH,OAAO,CAAC,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,cAAc,GAAG,SAAS,CAAC;CAC/F;AAED,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC3B,GAAG,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,GAAG,SAAS,CAAC,CAAC;IAC/D,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvE,MAAM,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3C;AAED,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;AAEzE,MAAM,MAAM,YAAY;AACpB,iFAAiF;AAC/E,MAAM;AACR,mEAAmE;GACjE,SAAS;AACX,iEAAiE;GAC/D,QAAQ,CAAC;AAEf,MAAM,WAAW,eAAe;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,gBAAgB,EAAE,MAAM,CAAC;IACzB,iEAAiE;IACjE,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACrD,UAAU,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,oEAAoE;AACpE,eAAO,MAAM,eAAe,aAAa,CAAC;AAC1C,eAAO,MAAM,iBAAiB,eAAe,CAAC;AAC9C,eAAO,MAAM,kBAAkB,gBAAgB,CAAC;AAChD,eAAO,MAAM,WAAW,UAAU,CAAC;AACnC,eAAO,MAAM,SAAS,QAAQ,CAAC;AAC/B,eAAO,MAAM,YAAY,WAAW,CAAC"}
|
package/dist/ingest/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/ingest/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/ingest/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AA0GH,oEAAoE;AACpE,MAAM,CAAC,MAAM,eAAe,GAAG,UAAU,CAAC;AAC1C,MAAM,CAAC,MAAM,iBAAiB,GAAG,YAAY,CAAC;AAC9C,MAAM,CAAC,MAAM,kBAAkB,GAAG,aAAa,CAAC;AAChD,MAAM,CAAC,MAAM,WAAW,GAAG,OAAO,CAAC;AACnC,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,CAAC;AAC/B,MAAM,CAAC,MAAM,YAAY,GAAG,QAAQ,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* knowledge/build.ts — `buildGraph`: join perception (extracted, computed) with memory
|
|
3
|
+
* (stored facts and relations) into one bounded {@link KnowledgeGraph} (PRD 38 §4.1).
|
|
4
|
+
*
|
|
5
|
+
* 1. extracted nodes/edges (analyzer output merges its badges into the same refs);
|
|
6
|
+
* 2. facts become nodes, joined to sources by `fileAnchors.path`; a model fact whose
|
|
7
|
+
* anchor hash differs from the source's current hash is `stale`;
|
|
8
|
+
* 3. stored relations become edges; near-duplicate pairs become `near-duplicate` edges
|
|
9
|
+
* (an exact `duplicate` between the same pair wins); live changes badge nodes and
|
|
10
|
+
* `runFiles` become `co-changed` edges;
|
|
11
|
+
* 4. partitions, areas, verdicts; PageRank importance; findings; then truncation.
|
|
12
|
+
*
|
|
13
|
+
* Deterministic: equal input ⇒ byte-identical JSON. Every list is sorted, every sort ends
|
|
14
|
+
* on ref, objects are built with a fixed key order.
|
|
15
|
+
*/
|
|
16
|
+
import type { BuildGraphInput, BuildGraphOptions, KEdge, KnowledgeGraph } from './model.js';
|
|
17
|
+
export declare const DEFAULT_MAX_NODES = 400;
|
|
18
|
+
export declare const MAX_NODES_CAP = 2000;
|
|
19
|
+
export declare const compareEdges: (a: KEdge, b: KEdge) => number;
|
|
20
|
+
/** sha-256 over the canonical input: equal inputs (in any order) ⇒ equal digests. */
|
|
21
|
+
export declare function graphDigest(input: BuildGraphInput): string;
|
|
22
|
+
export declare function buildGraph(input: BuildGraphInput, opts?: BuildGraphOptions): KnowledgeGraph;
|
|
23
|
+
export declare function finalEdge(e: KEdge): KEdge;
|
|
24
|
+
//# sourceMappingURL=build.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/knowledge/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAIH,OAAO,KAAK,EACR,eAAe,EAAE,iBAAiB,EAAa,KAAK,EAAS,cAAc,EAE9E,MAAM,YAAY,CAAC;AAUpB,eAAO,MAAM,iBAAiB,MAAM,CAAC;AACrC,eAAO,MAAM,aAAa,OAAO,CAAC;AAIlC,eAAO,MAAM,YAAY,GAAI,GAAG,KAAK,EAAE,GAAG,KAAK,KAAG,MACoC,CAAC;AAEvF,qFAAqF;AACrF,wBAAgB,WAAW,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CAa1D;AAyED,wBAAgB,UAAU,CAAC,KAAK,EAAE,eAAe,EAAE,IAAI,GAAE,iBAAsB,GAAG,cAAc,CA6D/F;AAuED,wBAAgB,SAAS,CAAC,CAAC,EAAE,KAAK,GAAG,KAAK,CAOzC"}
|