@undefineds.co/xpod 0.3.75 → 0.4.0-rc.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/xpod.js +43 -8
- package/config/bun.json +45 -48
- package/config/cloud.json +13 -90
- package/config/components-ignore.json +0 -1
- package/config/local.json +111 -63
- package/config/resolver.json +4 -2
- package/config/terminal.json +4 -1
- package/config/xpod.base.json +16 -8
- package/config/xpod.json +51 -54
- package/dist/acceptance/PublicCloudSemanticConformance.d.ts +4 -0
- package/dist/acceptance/PublicCloudSemanticConformance.js +214 -0
- package/dist/acceptance/QleverSearchConformance.d.ts +39 -0
- package/dist/acceptance/QleverSearchConformance.js +686 -0
- package/dist/acceptance/QleverSemanticConformance.d.ts +3 -0
- package/dist/acceptance/QleverSemanticConformance.js +178 -0
- package/dist/acceptance/RdfSemanticConformance.d.ts +114 -0
- package/dist/acceptance/RdfSemanticConformance.js +307 -0
- package/dist/acceptance/run-installed-qlever-conformance.d.ts +12 -0
- package/dist/acceptance/run-installed-qlever-conformance.js +87 -0
- package/dist/acceptance/run-installed-rdf-conformance.d.ts +12 -0
- package/dist/acceptance/run-installed-rdf-conformance.js +88 -0
- package/dist/ai/service/EmbeddingProviderError.d.ts +7 -0
- package/dist/ai/service/EmbeddingProviderError.js +42 -0
- package/dist/api/ai-gateway/connect/index.js +5 -2
- package/dist/api/auth/AuthContext.d.ts +5 -0
- package/dist/api/auth/AuthContext.js +10 -0
- package/dist/api/container/common.js +20 -1
- package/dist/api/container/rdf.d.ts +1 -0
- package/dist/api/container/rdf.js +43 -4
- package/dist/api/container/types.d.ts +4 -0
- package/dist/api/runs/InngestRunExecutionBackend.d.ts +2 -2
- package/dist/api/runs/PiAgentRuntimeDriver.d.ts +2 -0
- package/dist/api/runs/PiAgentRuntimeDriver.js +1 -0
- package/dist/api/runs/RdfRunContextRetriever.d.ts +8 -0
- package/dist/api/runs/RdfRunContextRetriever.js +23 -1
- package/dist/api/runtime.js +20 -6
- package/dist/api/service/RdfSearchIndexingService.d.ts +37 -3
- package/dist/api/service/RdfSearchIndexingService.js +130 -18
- package/dist/api/service/RdfSearchIndexingSolidFsSyncer.d.ts +8 -0
- package/dist/api/service/RdfSearchIndexingSolidFsSyncer.js +102 -15
- package/dist/api/service/RdfSearchReconciliationWorker.d.ts +76 -0
- package/dist/api/service/RdfSearchReconciliationWorker.js +294 -0
- package/dist/api/tasks/InngestTaskScheduler.d.ts +4 -4
- package/dist/components/components.jsonld +9 -12
- package/dist/components/context.jsonld +227 -348
- package/dist/document/MarkdownRetrievalPointProjector.d.ts +33 -0
- package/dist/document/MarkdownRetrievalPointProjector.js +196 -0
- package/dist/document/MarkdownRetrievalPointProjector.jsonld +135 -0
- package/dist/document/ReaderMaterialization.d.ts +34 -0
- package/dist/document/ReaderMaterialization.js +3 -0
- package/dist/document/ReaderMaterialization.jsonld +151 -0
- package/dist/document/ReaderMaterializationRepository.d.ts +27 -0
- package/dist/document/ReaderMaterializationRepository.js +400 -0
- package/dist/document/ReaderMaterializationRepository.jsonld +84 -0
- package/dist/document/index.d.ts +3 -1
- package/dist/document/index.js +3 -1
- package/dist/http/SubgraphSparqlHttpHandler.js +34 -54
- package/dist/http/search/SearchHttpHandler.js +14 -12
- package/dist/http/terminal/TerminalHttpHandler.d.ts +3 -0
- package/dist/http/terminal/TerminalHttpHandler.js +1 -0
- package/dist/http/terminal/TerminalHttpHandler.jsonld +11 -0
- package/dist/identity/drizzle/DrizzleIndexedStorage.js +19 -39
- package/dist/identity/drizzle/db.d.ts +1 -0
- package/dist/identity/drizzle/db.js +35 -0
- package/dist/index.d.ts +9 -17
- package/dist/index.js +14 -22
- package/dist/provision/ProvisionPodCreator.d.ts +1 -1
- package/dist/provision/ProvisionPodCreator.js +4 -3
- package/dist/search/RdfSearchIntentSink.d.ts +17 -0
- package/dist/search/RdfSearchIntentSink.js +46 -0
- package/dist/search/RdfSearchIntentSink.jsonld +47 -0
- package/dist/search/RdfSearchReconciliationRepository.d.ts +81 -0
- package/dist/search/RdfSearchReconciliationRepository.js +771 -0
- package/dist/solidfs/LocalRdfAuthorityRecoveryInitializer.d.ts +15 -0
- package/dist/solidfs/LocalRdfAuthorityRecoveryInitializer.js +48 -0
- package/dist/solidfs/LocalRdfAuthorityRecoveryInitializer.jsonld +99 -0
- package/dist/solidfs/RdfIndexSolidFsSyncer.js +22 -4
- package/dist/solidfs/SolidFsSyncJournal.d.ts +7 -1
- package/dist/solidfs/SolidFsSyncJournal.js +20 -5
- package/dist/solidfs/SolidFsSyncJournal.jsonld +43 -0
- package/dist/solidfs/index.d.ts +1 -0
- package/dist/solidfs/index.js +1 -0
- package/dist/storage/SparqlUpdateResourceStore.js +3 -3
- package/dist/storage/accessors/MixDataAccessor.d.ts +21 -29
- package/dist/storage/accessors/MixDataAccessor.js +182 -399
- package/dist/storage/accessors/MixDataAccessor.jsonld +109 -87
- package/dist/storage/accessors/SolidRdfDataAccessor.d.ts +12 -8
- package/dist/storage/accessors/SolidRdfDataAccessor.js +189 -59
- package/dist/storage/accessors/SolidRdfDataAccessor.jsonld +14 -29
- package/dist/storage/rdf/LocalQleverNativeSparqlClient.d.ts +55 -0
- package/dist/storage/rdf/LocalQleverNativeSparqlClient.js +519 -0
- package/dist/storage/rdf/LocalQleverNativeSparqlClient.jsonld +314 -0
- package/dist/storage/rdf/PostgresRdfEngine.d.ts +3 -1
- package/dist/storage/rdf/PostgresRdfEngine.js +111 -74
- package/dist/storage/rdf/PostgresRdfEngine.jsonld +12 -22
- package/dist/storage/rdf/PostgresRdfSchemaValidation.d.ts +8 -0
- package/dist/storage/rdf/PostgresRdfSchemaValidation.js +82 -0
- package/dist/storage/rdf/PostgresRdfTextIndex.d.ts +6 -3
- package/dist/storage/rdf/PostgresRdfTextIndex.js +158 -80
- package/dist/storage/rdf/PostgresRdfTextIndex.jsonld +16 -4
- package/dist/storage/rdf/PostgresRdfVectorIndex.d.ts +4 -5
- package/dist/storage/rdf/PostgresRdfVectorIndex.js +124 -120
- package/dist/storage/rdf/PostgresRdfVectorIndex.jsonld +14 -18
- package/dist/storage/rdf/QleverSparqlEngine.d.ts +35 -0
- package/dist/storage/rdf/QleverSparqlEngine.js +185 -0
- package/dist/storage/rdf/QleverSparqlEngine.jsonld +70 -0
- package/dist/storage/rdf/Rdf3xIndex.js +20 -16
- package/dist/storage/rdf/RdfAccessScope.d.ts +4 -0
- package/dist/storage/rdf/RdfAccessScope.js +74 -9
- package/dist/storage/rdf/RdfAccessScope.jsonld +16 -0
- package/dist/storage/rdf/RdfEngineRdfJsSource.d.ts +14 -0
- package/dist/storage/rdf/RdfEngineRdfJsSource.js +179 -0
- package/dist/storage/rdf/RdfQuadIndex.d.ts +13 -14
- package/dist/storage/rdf/RdfQuadIndex.js +286 -99
- package/dist/storage/rdf/RdfQuadIndex.jsonld +10 -22
- package/dist/storage/rdf/RdfQueryExecutor.js +10 -4
- package/dist/storage/rdf/RdfQuerySparqlEngine.d.ts +33 -0
- package/dist/storage/rdf/RdfQuerySparqlEngine.js +215 -0
- package/dist/storage/rdf/RdfQuerySparqlEngine.jsonld +78 -0
- package/dist/storage/rdf/RdfSparqlAdapter.d.ts +0 -4
- package/dist/storage/rdf/RdfSparqlAdapter.js +75 -173
- package/dist/storage/rdf/RdfSparqlBoundary.d.ts +37 -0
- package/dist/storage/rdf/RdfSparqlBoundary.js +247 -0
- package/dist/storage/rdf/RdfSparqlSerialization.d.ts +3 -0
- package/dist/storage/rdf/RdfSparqlSerialization.js +19 -0
- package/dist/storage/rdf/RdfSqliteConnection.d.ts +7 -0
- package/dist/storage/rdf/RdfSqliteConnection.js +23 -0
- package/dist/storage/rdf/RdfTermDictionary.d.ts +0 -8
- package/dist/storage/rdf/RdfTermDictionary.js +0 -136
- package/dist/storage/rdf/RdfTermDictionary.jsonld +0 -32
- package/dist/storage/rdf/RdfTermSemantics.d.ts +6 -0
- package/dist/storage/rdf/RdfTermSemantics.js +29 -0
- package/dist/storage/rdf/RdfTextIndex.d.ts +5 -6
- package/dist/storage/rdf/RdfTextIndex.js +187 -115
- package/dist/storage/rdf/RdfTextIndex.jsonld +10 -14
- package/dist/storage/rdf/RdfVectorIndex.d.ts +4 -4
- package/dist/storage/rdf/RdfVectorIndex.js +156 -233
- package/dist/storage/rdf/RdfVectorIndex.jsonld +10 -14
- package/dist/storage/rdf/SolidRdfEngine.d.ts +16 -21
- package/dist/storage/rdf/SolidRdfEngine.js +28 -308
- package/dist/storage/rdf/SolidRdfEngine.jsonld +26 -148
- package/dist/storage/rdf/XpodComunicaQueryEngine.d.ts +10 -0
- package/dist/storage/rdf/XpodComunicaQueryEngine.js +72 -0
- package/dist/storage/rdf/index.d.ts +7 -6
- package/dist/storage/rdf/index.js +17 -12
- package/dist/storage/rdf/models-benchmark.d.ts +5 -1
- package/dist/storage/rdf/models-benchmark.js +17 -19
- package/dist/storage/rdf/types.d.ts +68 -6
- package/dist/storage/rdf/types.jsonld +69 -0
- package/dist/storage/sparql/SubgraphQueryEngine.d.ts +13 -7
- package/dist/storage/sparql/SubgraphQueryEngine.js +6 -6
- package/dist/storage/vector/index.d.ts +0 -1
- package/dist/storage/vector/index.js +0 -1
- package/dist/terminal/AclPermissionService.d.ts +4 -5
- package/dist/terminal/AclPermissionService.js +55 -75
- package/dist/terminal/TerminalSessionManager.d.ts +3 -2
- package/dist/terminal/TerminalSessionManager.js +3 -3
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/execution/ldp-executor.d.ts +1 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/execution/ldp-executor.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/execution/ldp-executor.js +29 -26
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/execution/ldp-executor.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/execution/sparql-strategy.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/execution/sparql-strategy.js +9 -4
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/execution/sparql-strategy.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/notifications/notifications-client.d.ts +1 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/notifications/notifications-client.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/notifications/notifications-client.js +6 -3
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/notifications/notifications-client.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/pod-database.js +3 -3
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/pod-database.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/query-builders/select-query-builder.d.ts +6 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/query-builders/select-query-builder.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/query-builders/select-query-builder.js +13 -3
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/query-builders/select-query-builder.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/query-where-policy.d.ts +1 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/query-where-policy.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/query-where-policy.js +31 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/query-where-policy.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/sparql/builder/expression-builder.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/sparql/builder/expression-builder.js +5 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/sparql/builder/expression-builder.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/sparql-executor.d.ts +1 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/sparql-executor.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/sparql-executor.js +45 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/core/sparql-executor.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/execution/ldp-executor.d.ts +1 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/execution/ldp-executor.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/execution/ldp-executor.js +29 -26
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/execution/ldp-executor.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/execution/sparql-strategy.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/execution/sparql-strategy.js +9 -4
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/execution/sparql-strategy.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/notifications/notifications-client.d.ts +1 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/notifications/notifications-client.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/notifications/notifications-client.js +6 -3
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/notifications/notifications-client.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/pod-database.js +3 -3
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/pod-database.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/query-builders/select-query-builder.d.ts +6 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/query-builders/select-query-builder.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/query-builders/select-query-builder.js +14 -4
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/query-builders/select-query-builder.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/query-where-policy.d.ts +1 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/query-where-policy.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/query-where-policy.js +30 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/query-where-policy.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/sparql/builder/expression-builder.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/sparql/builder/expression-builder.js +5 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/sparql/builder/expression-builder.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/sparql-executor.d.ts +1 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/sparql-executor.d.ts.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/sparql-executor.js +45 -0
- package/node_modules/@undefineds.co/drizzle-solid/dist/esm/core/sparql-executor.js.map +1 -1
- package/node_modules/@undefineds.co/drizzle-solid/package.json +5 -4
- package/package.json +22 -8
- package/static/app/assets/{index-36TJKN-3.js → index-BikOyGPu.js} +34 -34
- package/static/app/assets/inrupt-smoke.js +2 -2
- package/static/app/assets/main.css +1 -1
- package/static/app/assets/main.js +11 -11
- package/static/auth-callback/assets/{AiConfigContext-BvBdu_A3.js → AiConfigContext-iQ74JmRZ.js} +1 -1
- package/static/auth-callback/assets/{AiConfigPage-BpDw4jco.js → AiConfigPage-BBDKE1gJ.js} +1 -1
- package/static/auth-callback/assets/{DocumentProcessingPanel-BmZ_jzRr.js → DocumentProcessingPanel-D9NntIEK.js} +1 -1
- package/static/auth-callback/assets/{IndexLifecyclePanel-BfQBjN4z.js → IndexLifecyclePanel-BG07yBdA.js} +1 -1
- package/static/auth-callback/assets/{IndexSubjectPanel-BlsnatPt.js → IndexSubjectPanel-B-bUnU3G.js} +1 -1
- package/static/auth-callback/assets/{ModelAssignmentsPanel-KA6Mdndz.js → ModelAssignmentsPanel-C7I-Nqy-.js} +1 -1
- package/static/auth-callback/assets/{ModelsPage-C7jymyIY.js → ModelsPage-C2hxyrmI.js} +1 -1
- package/static/auth-callback/assets/{NetworkPage-k2ShktNm.js → NetworkPage-IGIc2q1Y.js} +1 -1
- package/static/auth-callback/assets/{PaneListHeader-mLMe_dvU.js → PaneListHeader-CebuPl_g.js} +1 -1
- package/static/auth-callback/assets/{SearchIndexingPanel-CqV9um_t.js → SearchIndexingPanel-6HnZSg00.js} +1 -1
- package/static/auth-callback/assets/{StatusSubjectPanel-BgMBTbr2.js → StatusSubjectPanel-votxBkr5.js} +1 -1
- package/static/auth-callback/assets/{StatusWorkspace-zbSyhdG2.js → StatusWorkspace-Cksl61zX.js} +1 -1
- package/static/auth-callback/assets/{SystemSettingsPage-CtW79nW6.js → SystemSettingsPage-CKZFUaZT.js} +1 -1
- package/static/auth-callback/assets/{SystemSettingsSubjectPanel-ZOPKYUtZ.js → SystemSettingsSubjectPanel-pzfUcCf4.js} +1 -1
- package/static/auth-callback/assets/{activity-BQ-CQE3O.js → activity-DQL6kZHZ.js} +1 -1
- package/static/auth-callback/assets/auth-callback-B_s8Ye5Z.css +1 -0
- package/static/auth-callback/assets/{auth-callback-Htr_CB_g.js → auth-callback-BxQMd_Mc.js} +39 -39
- package/static/auth-callback/assets/{badge-D5SGTJeO.js → badge-DvW4LUHH.js} +1 -1
- package/static/auth-callback/assets/{download-DK183Ldo.js → download-Cx_uaCR1.js} +1 -1
- package/static/auth-callback/assets/{external-link-LDHy_4bn.js → external-link-D6L6ExH1.js} +1 -1
- package/static/auth-callback/assets/{index-BWrvgHqK.js → index-BAS_hxGu.js} +1 -1
- package/static/auth-callback/assets/{index-2WmlePQ9.js → index-BmBjpCoD.js} +1 -1
- package/static/auth-callback/assets/{index-browser-DUd4E4-x.js → index-browser-pdn4D3M4.js} +1 -1
- package/static/auth-callback/assets/{key-round-ibX79Kr-.js → key-round-BTBzOggq.js} +1 -1
- package/static/auth-callback/assets/{rotate-ccw-B2_VMaiX.js → rotate-ccw-eV7Cx8sB.js} +1 -1
- package/static/auth-callback/assets/{skeleton-F4nrpycc.js → skeleton-Bg4GYgBM.js} +1 -1
- package/static/auth-callback/assets/{waypoints-DImIq93f.js → waypoints-CJbydf0B.js} +1 -1
- package/static/auth-callback/assets/{workspace-layout-IVCSYU21.js → workspace-layout-C6Hh8Xf-.js} +1 -1
- package/static/auth-callback/auth-callback.html +2 -2
- package/static/dashboard/assets/{AiConfigContext-cjjfYTo9.js → AiConfigContext-BkJj28vK.js} +1 -1
- package/static/dashboard/assets/{AiConfigPage-CO0DSD_j.js → AiConfigPage-Cs9B4uyp.js} +1 -1
- package/static/dashboard/assets/{DocumentProcessingPanel-DMGnH8Oj.js → DocumentProcessingPanel-BBWmM7mS.js} +1 -1
- package/static/dashboard/assets/{IndexLifecyclePanel-DSkIOCL5.js → IndexLifecyclePanel-D5xGTT_i.js} +1 -1
- package/static/dashboard/assets/{IndexSubjectPanel-qWN_DbAp.js → IndexSubjectPanel-d-mjDthn.js} +1 -1
- package/static/dashboard/assets/{ModelAssignmentsPanel-DCf83uM3.js → ModelAssignmentsPanel-C-OLNbqf.js} +1 -1
- package/static/dashboard/assets/{ModelsPage-CMnvSrl5.js → ModelsPage-CBrLjuUn.js} +1 -1
- package/static/dashboard/assets/{NetworkPage-C8Gkwclb.js → NetworkPage-BlYk8L1z.js} +1 -1
- package/static/dashboard/assets/{PaneListHeader-BtU_8cDK.js → PaneListHeader-A5oZJQC0.js} +1 -1
- package/static/dashboard/assets/{SearchIndexingPanel-DEn12pNU.js → SearchIndexingPanel-Bs1j8ZP2.js} +1 -1
- package/static/dashboard/assets/{StatusSubjectPanel-B1_ILg2d.js → StatusSubjectPanel-Qao3orvi.js} +1 -1
- package/static/dashboard/assets/{StatusWorkspace-BG6nMNCo.js → StatusWorkspace-rDM0yDtk.js} +1 -1
- package/static/dashboard/assets/{SystemSettingsPage-BnWYIXaM.js → SystemSettingsPage-FPRPkKaR.js} +1 -1
- package/static/dashboard/assets/{SystemSettingsSubjectPanel-C_mYVPWb.js → SystemSettingsSubjectPanel-CjlJAxnX.js} +1 -1
- package/static/dashboard/assets/{activity-DYJ7PdRl.js → activity-CJ5K6hYC.js} +1 -1
- package/static/dashboard/assets/{badge-BIaZco1P.js → badge-L-NMOi1P.js} +1 -1
- package/static/dashboard/assets/dashboard-B_s8Ye5Z.css +1 -0
- package/static/dashboard/assets/{dashboard-Dl40_MEa.js → dashboard-DyjAyYxx.js} +34 -34
- package/static/dashboard/assets/{download-By16k9VK.js → download-Bi4u5K5B.js} +1 -1
- package/static/dashboard/assets/{external-link-B92xOG_d.js → external-link-jmhLDfvT.js} +1 -1
- package/static/dashboard/assets/{index-BajZpJ48.js → index-BoaGJBxf.js} +1 -1
- package/static/dashboard/assets/{index-wsVPom_x.js → index-C22TAEi4.js} +1 -1
- package/static/dashboard/assets/{key-round-Clo5FOwA.js → key-round-Dk1VmMrn.js} +1 -1
- package/static/dashboard/assets/{rotate-ccw-smWSfi1L.js → rotate-ccw-DIfLsSwi.js} +1 -1
- package/static/dashboard/assets/{skeleton-BrTaGeJw.js → skeleton-BGPJMHb-.js} +1 -1
- package/static/dashboard/assets/{waypoints-DGPQInE7.js → waypoints-CY53egpu.js} +1 -1
- package/static/dashboard/assets/{workspace-layout-CPU4XRmb.js → workspace-layout-CoHkcx70.js} +1 -1
- package/static/dashboard/dashboard.html +2 -2
- package/static/settings/assets/{AiConfigContext-CT29_OyW.js → AiConfigContext-BcEPlEJe.js} +1 -1
- package/static/settings/assets/{AiConfigPage-WIE95S28.js → AiConfigPage-D-t3ZUhg.js} +1 -1
- package/static/settings/assets/{DocumentProcessingPanel-KG8Sq999.js → DocumentProcessingPanel-m2SasiMO.js} +1 -1
- package/static/settings/assets/{IndexLifecyclePanel-iKvtHDXY.js → IndexLifecyclePanel-BNmAvx8V.js} +1 -1
- package/static/settings/assets/{IndexSubjectPanel-BQdSCkk4.js → IndexSubjectPanel-C5-L2AcW.js} +1 -1
- package/static/settings/assets/{ModelAssignmentsPanel-P8QcVB0F.js → ModelAssignmentsPanel-Cfpkf02Y.js} +1 -1
- package/static/settings/assets/{ModelsPage-UZkZGqor.js → ModelsPage-DZrACwVN.js} +1 -1
- package/static/settings/assets/{NetworkPage-Dcpcvg15.js → NetworkPage-BlrGGfqL.js} +1 -1
- package/static/settings/assets/{PaneListHeader--gHtZws5.js → PaneListHeader-C3TCs1RI.js} +1 -1
- package/static/settings/assets/{SearchIndexingPanel-DdgJSJYQ.js → SearchIndexingPanel-Cyp3CUzX.js} +1 -1
- package/static/settings/assets/{StatusSubjectPanel-UOxp_XVL.js → StatusSubjectPanel-gRkb8WEX.js} +1 -1
- package/static/settings/assets/{StatusWorkspace-CPNXuD12.js → StatusWorkspace-Dptx4Ec2.js} +1 -1
- package/static/settings/assets/{SystemSettingsPage-B6ySeVPw.js → SystemSettingsPage-BWgGUQbs.js} +1 -1
- package/static/settings/assets/{SystemSettingsSubjectPanel-I7zWmpiI.js → SystemSettingsSubjectPanel-VflUvpgC.js} +1 -1
- package/static/settings/assets/{activity-CFhmzahP.js → activity-D7iNhTkQ.js} +1 -1
- package/static/settings/assets/{badge-BBdUc5B-.js → badge-DO0r_RT4.js} +1 -1
- package/static/settings/assets/{download-WJE9-yxQ.js → download-CsJlNSbd.js} +1 -1
- package/static/settings/assets/{external-link-RKoYyCp_.js → external-link-BCR7SHme.js} +1 -1
- package/static/settings/assets/{index-Bq2o0Bif.js → index-Cgq16ekN.js} +1 -1
- package/static/settings/assets/{index-DvxneI5O.js → index-SrhQvcrU.js} +1 -1
- package/static/settings/assets/{index-browser-Qmo6EsFT.js → index-browser-THpcMCnz.js} +1 -1
- package/static/settings/assets/{key-round-r-D9sQUK.js → key-round-Dd3jk1BT.js} +1 -1
- package/static/settings/assets/{rotate-ccw-D3IIhbPH.js → rotate-ccw-tiL14i1d.js} +1 -1
- package/static/settings/assets/settings-B_s8Ye5Z.css +1 -0
- package/static/settings/assets/{settings-D8oDBXWg.js → settings-CVLDPejo.js} +34 -34
- package/static/settings/assets/{skeleton-DrL4RQuw.js → skeleton-DBxLuNgU.js} +1 -1
- package/static/settings/assets/{waypoints-C5sRhpJw.js → waypoints-DS3e3hv7.js} +1 -1
- package/static/settings/assets/{workspace-layout-B4ALmWsM.js → workspace-layout-mucrxyTK.js} +1 -1
- package/static/settings/settings.html +2 -2
- package/config/search.json +0 -66
- package/dist/document/HeadingChunker.jsonld +0 -41
- package/dist/security/secret-cell/DeploymentRootKeyProvider.jsonld +0 -124
- package/dist/security/secret-cell/SecretCellVault.jsonld +0 -207
- package/dist/storage/PostgresDerivedIndexJournal.d.ts +0 -47
- package/dist/storage/PostgresDerivedIndexJournal.js +0 -408
- package/dist/storage/PostgresDerivedIndexJournal.jsonld +0 -303
- package/dist/storage/RdfDerivedIndexingListener.d.ts +0 -32
- package/dist/storage/RdfDerivedIndexingListener.js +0 -135
- package/dist/storage/RdfDerivedIndexingListener.jsonld +0 -228
- package/dist/storage/accessors/QuadstoreSparqlDataAccessor.d.ts +0 -148
- package/dist/storage/accessors/QuadstoreSparqlDataAccessor.js +0 -436
- package/dist/storage/accessors/QuadstoreSparqlDataAccessor.jsonld +0 -172
- package/dist/storage/accessors/QuintStoreSparqlDataAccessor.d.ts +0 -102
- package/dist/storage/accessors/QuintStoreSparqlDataAccessor.js +0 -474
- package/dist/storage/accessors/QuintStoreSparqlDataAccessor.jsonld +0 -192
- package/dist/storage/rdf/RdfSparqlAdapter.jsonld +0 -486
- package/dist/storage/rdf/ShadowRdfQuintStore.d.ts +0 -59
- package/dist/storage/rdf/ShadowRdfQuintStore.js +0 -216
- package/dist/storage/rdf/ShadowRdfQuintStore.jsonld +0 -312
- package/dist/storage/rdf/SolidRdfSparqlEngine.d.ts +0 -107
- package/dist/storage/rdf/SolidRdfSparqlEngine.js +0 -1117
- package/dist/storage/rdf/SolidRdfSparqlEngine.jsonld +0 -312
- package/dist/storage/sparql/AlgebraUtils.d.ts +0 -48
- package/dist/storage/sparql/AlgebraUtils.js +0 -118
- package/dist/storage/sparql/CompatibilitySparqlEngine.d.ts +0 -37
- package/dist/storage/sparql/CompatibilitySparqlEngine.js +0 -96
- package/dist/storage/sparql/CompatibilitySparqlEngine.jsonld +0 -123
- package/dist/storage/sparql/CompatibilitySparqlEngineImpl.d.ts +0 -36
- package/dist/storage/sparql/CompatibilitySparqlEngineImpl.js +0 -124
- package/dist/storage/sparql/ComunicaOptimizedEngine.d.ts +0 -59
- package/dist/storage/sparql/ComunicaOptimizedEngine.js +0 -254
- package/dist/storage/sparql/ComunicaQuintEngine.d.ts +0 -143
- package/dist/storage/sparql/ComunicaQuintEngine.js +0 -781
- package/dist/storage/sparql/ExpressionEvaluator.d.ts +0 -54
- package/dist/storage/sparql/ExpressionEvaluator.js +0 -340
- package/dist/storage/sparql/FilterPushdownExtractor.d.ts +0 -74
- package/dist/storage/sparql/FilterPushdownExtractor.js +0 -421
- package/dist/storage/sparql/OptimizedQuadstoreEngine.d.ts +0 -65
- package/dist/storage/sparql/OptimizedQuadstoreEngine.js +0 -327
- package/dist/storage/sparql/OptimizedQuadstoreSource.d.ts +0 -46
- package/dist/storage/sparql/OptimizedQuadstoreSource.js +0 -118
- package/dist/storage/sparql/PatternBuilder.d.ts +0 -41
- package/dist/storage/sparql/PatternBuilder.js +0 -118
- package/dist/storage/sparql/QueryOptimizer.d.ts +0 -125
- package/dist/storage/sparql/QueryOptimizer.js +0 -375
- package/dist/storage/sparql/QuintEngine.d.ts +0 -92
- package/dist/storage/sparql/QuintEngine.js +0 -150
- package/dist/storage/sparql/QuintQuerySource.d.ts +0 -241
- package/dist/storage/sparql/QuintQuerySource.js +0 -1069
- package/dist/storage/sparql/SimpleSparqlExecutor.d.ts +0 -40
- package/dist/storage/sparql/SimpleSparqlExecutor.js +0 -131
- package/dist/storage/vector/VectorIndexingListener.d.ts +0 -114
- package/dist/storage/vector/VectorIndexingListener.js +0 -346
- package/static/auth-callback/assets/auth-callback-rWdEQY5f.css +0 -1
- package/static/dashboard/assets/dashboard-rWdEQY5f.css +0 -1
- package/static/settings/assets/settings-rWdEQY5f.css +0 -1
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import type { EmbeddingService } from '../../ai/service';
|
|
2
2
|
import type { RdfEngineLike, RdfTextChunkInput, RdfTextSourceInput, RdfVectorSourceInput } from '../../storage/rdf/types';
|
|
3
3
|
import type { StoreContext } from '../chatkit/store';
|
|
4
|
-
|
|
4
|
+
import { type EmbeddingProviderFailureCategory } from '../../ai/service/EmbeddingProviderError';
|
|
5
|
+
export type RdfVectorIndexingSkippedReason = 'rdf_engine_unavailable' | 'vector_index_unavailable' | 'ai_config_unavailable' | 'embedding_model_unavailable' | 'text_source_unavailable' | EmbeddingProviderFailureCategory;
|
|
5
6
|
export interface RdfSearchIndexingServiceOptions {
|
|
6
|
-
rdfEngine?: Pick<RdfEngineLike, 'indexVectorSource' | 'deleteVectorSource'>;
|
|
7
|
+
rdfEngine?: Pick<RdfEngineLike, 'indexVectorSource' | 'deleteVectorSource' | 'listTextSourceChunks'>;
|
|
7
8
|
store: {
|
|
8
9
|
getAiConfig(context: StoreContext): Promise<RdfSearchAiConfig | undefined> | RdfSearchAiConfig | undefined;
|
|
9
10
|
};
|
|
@@ -26,7 +27,8 @@ export interface RdfSearchAiConfig {
|
|
|
26
27
|
credentialId: string;
|
|
27
28
|
}
|
|
28
29
|
export interface IndexRdfVectorSourceInput {
|
|
29
|
-
context
|
|
30
|
+
context?: StoreContext;
|
|
31
|
+
embeddingConfig?: RdfSearchAiConfig;
|
|
30
32
|
source: RdfTextSourceInput & RdfVectorSourceInput;
|
|
31
33
|
text?: string;
|
|
32
34
|
chunks?: RdfTextChunkInput[];
|
|
@@ -34,17 +36,44 @@ export interface IndexRdfVectorSourceInput {
|
|
|
34
36
|
export interface DeleteRdfVectorSourceInput {
|
|
35
37
|
source: string;
|
|
36
38
|
}
|
|
39
|
+
export interface RebuildRdfVectorSourceInput {
|
|
40
|
+
context?: StoreContext;
|
|
41
|
+
embeddingConfig?: RdfSearchAiConfig;
|
|
42
|
+
sourceKey: string;
|
|
43
|
+
}
|
|
37
44
|
export type RdfVectorIndexingResult = {
|
|
38
45
|
status: 'indexed';
|
|
39
46
|
source: string;
|
|
47
|
+
sourceHash?: string;
|
|
48
|
+
sourceVersion?: string;
|
|
49
|
+
providerId?: string;
|
|
40
50
|
model?: string;
|
|
51
|
+
modelVersion?: string;
|
|
52
|
+
configFingerprint?: string;
|
|
41
53
|
chunkCount: number;
|
|
42
54
|
skippedInputs?: RdfSkippedEmbeddingInput[];
|
|
43
55
|
summarizedInputs?: RdfSummarizedEmbeddingInput[];
|
|
44
56
|
} | {
|
|
45
57
|
status: 'skipped';
|
|
46
58
|
source: string;
|
|
59
|
+
sourceHash?: string;
|
|
60
|
+
sourceVersion?: string;
|
|
47
61
|
reason: RdfVectorIndexingSkippedReason;
|
|
62
|
+
providerId?: string;
|
|
63
|
+
model?: string;
|
|
64
|
+
modelVersion?: string;
|
|
65
|
+
configFingerprint?: string;
|
|
66
|
+
message?: string;
|
|
67
|
+
} | {
|
|
68
|
+
status: 'retryable';
|
|
69
|
+
source: string;
|
|
70
|
+
sourceHash?: string;
|
|
71
|
+
sourceVersion?: string;
|
|
72
|
+
reason: EmbeddingProviderFailureCategory;
|
|
73
|
+
providerId: string;
|
|
74
|
+
model: string;
|
|
75
|
+
modelVersion?: string;
|
|
76
|
+
configFingerprint: string;
|
|
48
77
|
message?: string;
|
|
49
78
|
};
|
|
50
79
|
export interface RdfVectorDeleteResult {
|
|
@@ -88,6 +117,9 @@ export interface RdfSummarizedEmbeddingInput {
|
|
|
88
117
|
summaryChars: number;
|
|
89
118
|
rounds: number;
|
|
90
119
|
}
|
|
120
|
+
export declare const DEFAULT_RDF_VECTOR_MODEL_VERSION = "unversioned";
|
|
121
|
+
export declare const DEFAULT_RDF_VECTOR_PROJECTION_POLICY_VERSION = "rdf-vector-projection-v1";
|
|
122
|
+
export declare function normalizeRdfVectorModelVersion(value: string | undefined): string;
|
|
91
123
|
/**
|
|
92
124
|
* Product-level bridge from Pod-owned AI credentials to the RDF vector index.
|
|
93
125
|
*
|
|
@@ -98,6 +130,7 @@ export interface RdfSummarizedEmbeddingInput {
|
|
|
98
130
|
export declare class RdfSearchIndexingService {
|
|
99
131
|
private readonly options;
|
|
100
132
|
constructor(options: RdfSearchIndexingServiceOptions);
|
|
133
|
+
rebuildVectorSource(input: RebuildRdfVectorSourceInput): Promise<RdfVectorIndexingResult>;
|
|
101
134
|
indexVectorSource(input: IndexRdfVectorSourceInput): Promise<RdfVectorIndexingResult>;
|
|
102
135
|
deleteVectorSource(input: DeleteRdfVectorSourceInput): Promise<RdfVectorDeleteResult>;
|
|
103
136
|
private sourceWithHash;
|
|
@@ -110,4 +143,5 @@ export declare class RdfSearchIndexingService {
|
|
|
110
143
|
private summarizeEmbeddingInput;
|
|
111
144
|
private skippedEmbeddingInput;
|
|
112
145
|
private indexedResult;
|
|
146
|
+
private skippedResult;
|
|
113
147
|
}
|
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.RdfSearchIndexingService = void 0;
|
|
3
|
+
exports.RdfSearchIndexingService = exports.DEFAULT_RDF_VECTOR_PROJECTION_POLICY_VERSION = exports.DEFAULT_RDF_VECTOR_MODEL_VERSION = void 0;
|
|
4
|
+
exports.normalizeRdfVectorModelVersion = normalizeRdfVectorModelVersion;
|
|
4
5
|
const node_crypto_1 = require("node:crypto");
|
|
5
6
|
const RdfTextIndex_1 = require("../../storage/rdf/RdfTextIndex");
|
|
7
|
+
const EmbeddingProviderError_1 = require("../../ai/service/EmbeddingProviderError");
|
|
6
8
|
const DEFAULT_EMBEDDING_INPUT_KINDS = ['semantic'];
|
|
7
|
-
|
|
9
|
+
exports.DEFAULT_RDF_VECTOR_MODEL_VERSION = 'unversioned';
|
|
10
|
+
exports.DEFAULT_RDF_VECTOR_PROJECTION_POLICY_VERSION = 'rdf-vector-projection-v1';
|
|
8
11
|
const DEFAULT_SUMMARY_PROMPT_VERSION = 'rdf-embedding-summary-v1';
|
|
12
|
+
function normalizeRdfVectorModelVersion(value) {
|
|
13
|
+
return value?.trim() || exports.DEFAULT_RDF_VECTOR_MODEL_VERSION;
|
|
14
|
+
}
|
|
9
15
|
/**
|
|
10
16
|
* Product-level bridge from Pod-owned AI credentials to the RDF vector index.
|
|
11
17
|
*
|
|
@@ -17,6 +23,48 @@ class RdfSearchIndexingService {
|
|
|
17
23
|
constructor(options) {
|
|
18
24
|
this.options = options;
|
|
19
25
|
}
|
|
26
|
+
async rebuildVectorSource(input) {
|
|
27
|
+
if (!this.options.rdfEngine) {
|
|
28
|
+
return { status: 'skipped', source: input.sourceKey, reason: 'rdf_engine_unavailable' };
|
|
29
|
+
}
|
|
30
|
+
if (!this.options.rdfEngine.listTextSourceChunks) {
|
|
31
|
+
return { status: 'skipped', source: input.sourceKey, reason: 'vector_index_unavailable' };
|
|
32
|
+
}
|
|
33
|
+
const chunks = await this.options.rdfEngine.listTextSourceChunks(input.sourceKey);
|
|
34
|
+
if (chunks.length === 0) {
|
|
35
|
+
if (this.options.rdfEngine.deleteVectorSource) {
|
|
36
|
+
await this.options.rdfEngine.deleteVectorSource(input.sourceKey);
|
|
37
|
+
}
|
|
38
|
+
return { status: 'skipped', source: input.sourceKey, reason: 'text_source_unavailable' };
|
|
39
|
+
}
|
|
40
|
+
const first = chunks[0];
|
|
41
|
+
return this.indexVectorSource({
|
|
42
|
+
context: input.context,
|
|
43
|
+
embeddingConfig: input.embeddingConfig,
|
|
44
|
+
source: {
|
|
45
|
+
sourceKey: first.sourceKey,
|
|
46
|
+
source: first.source,
|
|
47
|
+
workspace: first.workspace,
|
|
48
|
+
localPath: first.localPath,
|
|
49
|
+
contentType: first.contentType,
|
|
50
|
+
sourceVersion: first.sourceVersion,
|
|
51
|
+
sourceHash: first.sourceHash,
|
|
52
|
+
},
|
|
53
|
+
chunks: chunks.map((chunk) => ({
|
|
54
|
+
chunkKey: chunk.retrievalPointKey,
|
|
55
|
+
retrievalPointKey: chunk.retrievalPointKey,
|
|
56
|
+
retrievalKind: chunk.retrievalKind,
|
|
57
|
+
ordinal: chunk.ordinal,
|
|
58
|
+
level: chunk.level,
|
|
59
|
+
heading: chunk.heading,
|
|
60
|
+
path: chunk.path,
|
|
61
|
+
content: chunk.content,
|
|
62
|
+
startOffset: chunk.startOffset,
|
|
63
|
+
endOffset: chunk.endOffset,
|
|
64
|
+
entities: chunk.entities,
|
|
65
|
+
})),
|
|
66
|
+
});
|
|
67
|
+
}
|
|
20
68
|
async indexVectorSource(input) {
|
|
21
69
|
if (!this.options.rdfEngine) {
|
|
22
70
|
return { status: 'skipped', source: input.source.source, reason: 'rdf_engine_unavailable' };
|
|
@@ -28,20 +76,22 @@ class RdfSearchIndexingService {
|
|
|
28
76
|
const textChunks = this.textChunks(source, input);
|
|
29
77
|
if (textChunks.length === 0) {
|
|
30
78
|
await this.options.rdfEngine.indexVectorSource(source, []);
|
|
31
|
-
return
|
|
79
|
+
return this.indexedResult(source, undefined, 0, [], []);
|
|
32
80
|
}
|
|
33
81
|
const { accepted: embeddingInputs, skipped: skippedInputs, summarized: summarizedInputs } = await this.resolveEmbeddingInputBudget(source, this.embeddingInputs(source, textChunks));
|
|
34
82
|
if (embeddingInputs.length === 0) {
|
|
35
83
|
await this.options.rdfEngine.indexVectorSource(source, []);
|
|
36
|
-
return this.indexedResult(source
|
|
84
|
+
return this.indexedResult(source, undefined, 0, skippedInputs, summarizedInputs);
|
|
37
85
|
}
|
|
38
|
-
const config = await this.options.store.getAiConfig(input.context);
|
|
86
|
+
const config = input.embeddingConfig ?? (input.context ? await this.options.store.getAiConfig(input.context) : undefined);
|
|
39
87
|
if (!config?.apiKey) {
|
|
40
|
-
return
|
|
88
|
+
return this.skippedResult(source, 'ai_config_unavailable');
|
|
41
89
|
}
|
|
42
90
|
if (!config.embeddingModel) {
|
|
43
|
-
return
|
|
91
|
+
return this.skippedResult(source, 'embedding_model_unavailable');
|
|
44
92
|
}
|
|
93
|
+
const embeddingModel = config.embeddingModel;
|
|
94
|
+
const embeddingModelVersion = normalizeRdfVectorModelVersion(config.embeddingModelVersion);
|
|
45
95
|
const contents = embeddingInputs.map((embeddingInput) => embeddingInput.content);
|
|
46
96
|
let embeddings;
|
|
47
97
|
try {
|
|
@@ -50,13 +100,42 @@ class RdfSearchIndexingService {
|
|
|
50
100
|
apiKey: config.apiKey,
|
|
51
101
|
baseUrl: config.baseUrl,
|
|
52
102
|
proxyUrl: config.proxyUrl,
|
|
53
|
-
},
|
|
103
|
+
}, embeddingModel);
|
|
54
104
|
}
|
|
55
105
|
catch (error) {
|
|
106
|
+
const failure = (0, EmbeddingProviderError_1.classifyEmbeddingProviderFailure)(error);
|
|
107
|
+
if (!failure.retryable) {
|
|
108
|
+
return {
|
|
109
|
+
status: 'skipped',
|
|
110
|
+
source: source.source,
|
|
111
|
+
...(source.sourceHash ? { sourceHash: source.sourceHash } : {}),
|
|
112
|
+
...(source.sourceVersion ? { sourceVersion: source.sourceVersion } : {}),
|
|
113
|
+
reason: failure.category,
|
|
114
|
+
providerId: config.providerId,
|
|
115
|
+
model: embeddingModel,
|
|
116
|
+
modelVersion: embeddingModelVersion,
|
|
117
|
+
configFingerprint: embeddingConfigFingerprint({
|
|
118
|
+
...config,
|
|
119
|
+
embeddingModel,
|
|
120
|
+
embeddingModelVersion,
|
|
121
|
+
}),
|
|
122
|
+
message: error instanceof Error ? error.message : String(error),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
56
125
|
return {
|
|
57
|
-
status: '
|
|
126
|
+
status: 'retryable',
|
|
58
127
|
source: source.source,
|
|
59
|
-
|
|
128
|
+
...(source.sourceHash ? { sourceHash: source.sourceHash } : {}),
|
|
129
|
+
...(source.sourceVersion ? { sourceVersion: source.sourceVersion } : {}),
|
|
130
|
+
reason: failure.category,
|
|
131
|
+
providerId: config.providerId,
|
|
132
|
+
model: embeddingModel,
|
|
133
|
+
modelVersion: embeddingModelVersion,
|
|
134
|
+
configFingerprint: embeddingConfigFingerprint({
|
|
135
|
+
...config,
|
|
136
|
+
embeddingModel,
|
|
137
|
+
embeddingModelVersion,
|
|
138
|
+
}),
|
|
60
139
|
message: error instanceof Error ? error.message : String(error),
|
|
61
140
|
};
|
|
62
141
|
}
|
|
@@ -70,8 +149,8 @@ class RdfSearchIndexingService {
|
|
|
70
149
|
level: chunk.level,
|
|
71
150
|
embedding: embeddings[index],
|
|
72
151
|
provider: config.providerId,
|
|
73
|
-
model:
|
|
74
|
-
modelVersion:
|
|
152
|
+
model: embeddingModel,
|
|
153
|
+
modelVersion: embeddingModelVersion,
|
|
75
154
|
inputKind: kind,
|
|
76
155
|
inputHash,
|
|
77
156
|
projectionPolicyVersion,
|
|
@@ -82,7 +161,16 @@ class RdfSearchIndexingService {
|
|
|
82
161
|
endOffset: chunk.endOffset,
|
|
83
162
|
summaryMetadata,
|
|
84
163
|
})));
|
|
85
|
-
return this.indexedResult(source
|
|
164
|
+
return this.indexedResult(source, {
|
|
165
|
+
providerId: config.providerId,
|
|
166
|
+
model: embeddingModel,
|
|
167
|
+
modelVersion: embeddingModelVersion,
|
|
168
|
+
configFingerprint: embeddingConfigFingerprint({
|
|
169
|
+
...config,
|
|
170
|
+
embeddingModel,
|
|
171
|
+
embeddingModelVersion,
|
|
172
|
+
}),
|
|
173
|
+
}, embeddingInputs.length, skippedInputs, summarizedInputs);
|
|
86
174
|
}
|
|
87
175
|
async deleteVectorSource(input) {
|
|
88
176
|
if (!this.options.rdfEngine) {
|
|
@@ -114,7 +202,7 @@ class RdfSearchIndexingService {
|
|
|
114
202
|
}
|
|
115
203
|
return {
|
|
116
204
|
...source,
|
|
117
|
-
sourceHash: (0, node_crypto_1.createHash)('sha256').update(text).digest('hex')
|
|
205
|
+
sourceHash: `sha256:${(0, node_crypto_1.createHash)('sha256').update(text).digest('hex')}`,
|
|
118
206
|
};
|
|
119
207
|
}
|
|
120
208
|
textChunks(source, input) {
|
|
@@ -149,7 +237,7 @@ class RdfSearchIndexingService {
|
|
|
149
237
|
return [...new Set(configured)];
|
|
150
238
|
}
|
|
151
239
|
projectionPolicyVersion() {
|
|
152
|
-
return this.options.projectionPolicyVersion ??
|
|
240
|
+
return this.options.projectionPolicyVersion ?? exports.DEFAULT_RDF_VECTOR_PROJECTION_POLICY_VERSION;
|
|
153
241
|
}
|
|
154
242
|
embeddingInputContent(source, chunk, kind) {
|
|
155
243
|
if (kind === 'semantic') {
|
|
@@ -257,16 +345,40 @@ class RdfSearchIndexingService {
|
|
|
257
345
|
...(message ? { message } : {}),
|
|
258
346
|
};
|
|
259
347
|
}
|
|
260
|
-
indexedResult(source,
|
|
348
|
+
indexedResult(source, profile, chunkCount, skippedInputs, summarizedInputs = []) {
|
|
261
349
|
return {
|
|
262
350
|
status: 'indexed',
|
|
263
|
-
source,
|
|
264
|
-
...(
|
|
351
|
+
source: source.source,
|
|
352
|
+
...(source.sourceHash ? { sourceHash: source.sourceHash } : {}),
|
|
353
|
+
...(source.sourceVersion ? { sourceVersion: source.sourceVersion } : {}),
|
|
354
|
+
...(profile?.providerId ? { providerId: profile.providerId } : {}),
|
|
355
|
+
...(profile?.model ? { model: profile.model } : {}),
|
|
356
|
+
...(profile?.modelVersion ? { modelVersion: profile.modelVersion } : {}),
|
|
357
|
+
...(profile?.configFingerprint ? { configFingerprint: profile.configFingerprint } : {}),
|
|
265
358
|
chunkCount,
|
|
266
359
|
...(skippedInputs.length > 0 ? { skippedInputs } : {}),
|
|
267
360
|
...(summarizedInputs.length > 0 ? { summarizedInputs } : {}),
|
|
268
361
|
};
|
|
269
362
|
}
|
|
363
|
+
skippedResult(source, reason) {
|
|
364
|
+
return {
|
|
365
|
+
status: 'skipped',
|
|
366
|
+
source: source.source,
|
|
367
|
+
...(source.sourceHash ? { sourceHash: source.sourceHash } : {}),
|
|
368
|
+
...(source.sourceVersion ? { sourceVersion: source.sourceVersion } : {}),
|
|
369
|
+
reason,
|
|
370
|
+
};
|
|
371
|
+
}
|
|
270
372
|
}
|
|
271
373
|
exports.RdfSearchIndexingService = RdfSearchIndexingService;
|
|
374
|
+
function embeddingConfigFingerprint(input) {
|
|
375
|
+
return `sha256:${(0, node_crypto_1.createHash)('sha256')
|
|
376
|
+
.update([
|
|
377
|
+
input.providerId,
|
|
378
|
+
input.embeddingModel,
|
|
379
|
+
normalizeRdfVectorModelVersion(input.embeddingModelVersion),
|
|
380
|
+
input.credentialId,
|
|
381
|
+
].join('\0'))
|
|
382
|
+
.digest('hex')}`;
|
|
383
|
+
}
|
|
272
384
|
//# sourceMappingURL=RdfSearchIndexingService.js.map
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { SolidFsChange, SolidFsManifest, SolidFsPrepareInput, SolidFsSyncer } from '../../solidfs';
|
|
2
2
|
import type { RdfSearchIndexingService, RdfVectorDeleteResult, RdfVectorIndexingResult } from './RdfSearchIndexingService';
|
|
3
|
+
import type { RdfSearchReconciliationRepository } from '../../search/RdfSearchReconciliationRepository';
|
|
3
4
|
export interface RdfSearchIndexingSolidFsSyncerOptions {
|
|
4
5
|
service: RdfSearchIndexingService;
|
|
5
6
|
/**
|
|
@@ -9,6 +10,9 @@ export interface RdfSearchIndexingSolidFsSyncerOptions {
|
|
|
9
10
|
failOnError?: boolean;
|
|
10
11
|
onError?: (error: unknown, input: RdfSearchIndexingSolidFsSyncerErrorInput) => void;
|
|
11
12
|
onResult?: (result: RdfVectorIndexingResult | RdfVectorDeleteResult) => void;
|
|
13
|
+
reconciliationRepository?: Pick<RdfSearchReconciliationRepository, 'upsertRetryable' | 'upsertBlockedConfig' | 'waitForConfig' | 'upsertApplied' | 'deleteSource'>;
|
|
14
|
+
now?: () => number;
|
|
15
|
+
retryDelayMs?: number;
|
|
12
16
|
}
|
|
13
17
|
export interface RdfSearchIndexingSolidFsSyncerErrorInput {
|
|
14
18
|
change: SolidFsChange;
|
|
@@ -20,8 +24,12 @@ export declare class RdfSearchIndexingSolidFsSyncer implements SolidFsSyncer {
|
|
|
20
24
|
private readonly failOnError;
|
|
21
25
|
private readonly onError?;
|
|
22
26
|
private readonly onResult?;
|
|
27
|
+
private readonly reconciliationRepository?;
|
|
28
|
+
private readonly now;
|
|
29
|
+
private readonly retryDelayMs;
|
|
23
30
|
constructor(options: RdfSearchIndexingSolidFsSyncerOptions);
|
|
24
31
|
shouldTrack(input: SolidFsPrepareInput): boolean;
|
|
25
32
|
shouldTrackPath(relativePath: string): boolean;
|
|
26
33
|
sync(change: SolidFsChange, workspace: SolidFsManifest, context?: unknown): Promise<void>;
|
|
34
|
+
private recordReconciliationResult;
|
|
27
35
|
}
|
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
3
|
exports.RdfSearchIndexingSolidFsSyncer = void 0;
|
|
7
|
-
const promises_1 = require("node:fs/promises");
|
|
8
|
-
const node_path_1 = __importDefault(require("node:path"));
|
|
9
4
|
const solidfs_1 = require("../../solidfs");
|
|
10
5
|
const RdfContentTypes_1 = require("../../storage/rdf/RdfContentTypes");
|
|
6
|
+
const BLOCKED_CONFIG_FAILURES = new Set([
|
|
7
|
+
'embedding_authentication_failed',
|
|
8
|
+
'embedding_authorization_failed',
|
|
9
|
+
'embedding_model_invalid',
|
|
10
|
+
'embedding_request_invalid',
|
|
11
|
+
]);
|
|
11
12
|
class RdfSearchIndexingSolidFsSyncer {
|
|
12
13
|
constructor(options) {
|
|
13
14
|
this.service = options.service;
|
|
14
15
|
this.failOnError = options.failOnError === true;
|
|
15
16
|
this.onError = options.onError;
|
|
16
17
|
this.onResult = options.onResult;
|
|
18
|
+
this.reconciliationRepository = options.reconciliationRepository;
|
|
19
|
+
this.now = options.now ?? Date.now;
|
|
20
|
+
this.retryDelayMs = options.retryDelayMs ?? 60_000;
|
|
17
21
|
}
|
|
18
22
|
shouldTrack(input) {
|
|
19
23
|
try {
|
|
@@ -38,21 +42,15 @@ class RdfSearchIndexingSolidFsSyncer {
|
|
|
38
42
|
try {
|
|
39
43
|
if (change.type === 'deleted') {
|
|
40
44
|
const result = await this.service.deleteVectorSource({ source });
|
|
45
|
+
await this.reconciliationRepository?.deleteSource(source);
|
|
41
46
|
this.onResult?.(result);
|
|
42
47
|
return;
|
|
43
48
|
}
|
|
44
|
-
const
|
|
45
|
-
const result = await this.service.indexVectorSource({
|
|
49
|
+
const result = await this.service.rebuildVectorSource({
|
|
46
50
|
context,
|
|
47
|
-
|
|
48
|
-
source,
|
|
49
|
-
workspace: workspace.workspace,
|
|
50
|
-
localPath: change.path.split(node_path_1.default.sep).join('/'),
|
|
51
|
-
contentType: change.contentType,
|
|
52
|
-
sourceVersion: change.sourceVersion,
|
|
53
|
-
},
|
|
54
|
-
text,
|
|
51
|
+
sourceKey: source,
|
|
55
52
|
});
|
|
53
|
+
await this.recordReconciliationResult(result, source, workspace.workspace);
|
|
56
54
|
this.onResult?.(result);
|
|
57
55
|
}
|
|
58
56
|
catch (error) {
|
|
@@ -62,8 +60,97 @@ class RdfSearchIndexingSolidFsSyncer {
|
|
|
62
60
|
}
|
|
63
61
|
}
|
|
64
62
|
}
|
|
63
|
+
async recordReconciliationResult(result, source, podRoot) {
|
|
64
|
+
const identity = {
|
|
65
|
+
sourceHash: result.sourceHash,
|
|
66
|
+
sourceVersion: result.sourceVersion,
|
|
67
|
+
};
|
|
68
|
+
if (result.status !== 'skipped') {
|
|
69
|
+
if (result.status !== 'retryable') {
|
|
70
|
+
if (hasAppliedEmbeddingProfile(result)) {
|
|
71
|
+
await this.reconciliationRepository?.upsertApplied({
|
|
72
|
+
sourceKey: source,
|
|
73
|
+
sourceUri: source,
|
|
74
|
+
podRoot,
|
|
75
|
+
providerId: result.providerId,
|
|
76
|
+
model: result.model,
|
|
77
|
+
modelVersion: result.modelVersion,
|
|
78
|
+
configFingerprint: result.configFingerprint,
|
|
79
|
+
sourceHash: identity.sourceHash,
|
|
80
|
+
sourceVersion: identity.sourceVersion,
|
|
81
|
+
reason: 'source-indexed',
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
await this.reconciliationRepository?.deleteSource(source);
|
|
86
|
+
}
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
await this.reconciliationRepository?.upsertRetryable({
|
|
90
|
+
sourceKey: source,
|
|
91
|
+
sourceUri: source,
|
|
92
|
+
podRoot,
|
|
93
|
+
providerId: result.providerId,
|
|
94
|
+
model: result.model,
|
|
95
|
+
modelVersion: result.modelVersion,
|
|
96
|
+
configFingerprint: result.configFingerprint,
|
|
97
|
+
sourceHash: identity.sourceHash,
|
|
98
|
+
sourceVersion: identity.sourceVersion,
|
|
99
|
+
reason: result.reason,
|
|
100
|
+
failureCategory: result.reason,
|
|
101
|
+
nextAttemptAt: new Date(this.now() + this.retryDelayMs),
|
|
102
|
+
});
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
if (result.reason === 'text_source_unavailable') {
|
|
106
|
+
await this.reconciliationRepository?.deleteSource(source);
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
if (BLOCKED_CONFIG_FAILURES.has(result.reason)) {
|
|
110
|
+
if (!hasBlockedEmbeddingProfile(result)) {
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
await this.reconciliationRepository?.upsertBlockedConfig({
|
|
114
|
+
sourceKey: source,
|
|
115
|
+
sourceUri: source,
|
|
116
|
+
podRoot,
|
|
117
|
+
providerId: result.providerId,
|
|
118
|
+
model: result.model,
|
|
119
|
+
modelVersion: result.modelVersion,
|
|
120
|
+
configFingerprint: result.configFingerprint,
|
|
121
|
+
sourceHash: identity.sourceHash,
|
|
122
|
+
sourceVersion: identity.sourceVersion,
|
|
123
|
+
reason: result.reason,
|
|
124
|
+
failureCategory: result.reason,
|
|
125
|
+
});
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
if (result.reason !== 'ai_config_unavailable' && result.reason !== 'embedding_model_unavailable') {
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
await this.reconciliationRepository?.waitForConfig({
|
|
132
|
+
sourceKey: source,
|
|
133
|
+
sourceUri: source,
|
|
134
|
+
podRoot,
|
|
135
|
+
sourceHash: identity.sourceHash,
|
|
136
|
+
sourceVersion: identity.sourceVersion,
|
|
137
|
+
reason: result.reason,
|
|
138
|
+
failureCategory: result.reason,
|
|
139
|
+
});
|
|
140
|
+
}
|
|
65
141
|
}
|
|
66
142
|
exports.RdfSearchIndexingSolidFsSyncer = RdfSearchIndexingSolidFsSyncer;
|
|
143
|
+
function hasBlockedEmbeddingProfile(result) {
|
|
144
|
+
return typeof result.providerId === 'string'
|
|
145
|
+
&& typeof result.model === 'string'
|
|
146
|
+
&& typeof result.configFingerprint === 'string';
|
|
147
|
+
}
|
|
148
|
+
function hasAppliedEmbeddingProfile(result) {
|
|
149
|
+
return result.status === 'indexed'
|
|
150
|
+
&& typeof result.providerId === 'string'
|
|
151
|
+
&& typeof result.model === 'string'
|
|
152
|
+
&& typeof result.configFingerprint === 'string';
|
|
153
|
+
}
|
|
67
154
|
function isSearchIndexableChange(change) {
|
|
68
155
|
return (0, RdfContentTypes_1.isLineAddressableRdf)(change.contentType, change.path)
|
|
69
156
|
|| isTextContentType(change.contentType)
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { RdfEngineLike } from '../../storage/rdf';
|
|
2
|
+
import type { StoreContext } from '../chatkit/store';
|
|
3
|
+
import type { RunAuthContextRegistry } from '../runs/RunAuthContextRegistry';
|
|
4
|
+
import { type RdfSearchIndexingService } from './RdfSearchIndexingService';
|
|
5
|
+
import type { RdfSearchReconciliationRepository } from '../../search/RdfSearchReconciliationRepository';
|
|
6
|
+
export interface RdfSearchReconciliationWorkerOptions {
|
|
7
|
+
repository: RdfSearchReconciliationRepository;
|
|
8
|
+
indexingService?: RdfSearchIndexingService;
|
|
9
|
+
contextRegistry: RunAuthContextRegistry;
|
|
10
|
+
store?: {
|
|
11
|
+
getAiConfig(context: StoreContext): Promise<RdfSearchEmbeddingConfig | undefined> | RdfSearchEmbeddingConfig | undefined;
|
|
12
|
+
};
|
|
13
|
+
rdfEngine?: Pick<RdfEngineLike, 'listTextSources'>;
|
|
14
|
+
workerId?: string;
|
|
15
|
+
now?: () => number;
|
|
16
|
+
leaseDurationMs?: number;
|
|
17
|
+
maxBatchSize?: number;
|
|
18
|
+
intervalMs?: number;
|
|
19
|
+
reconcileIntervalMs?: number;
|
|
20
|
+
retryBaseDelayMs?: number;
|
|
21
|
+
retryMaxDelayMs?: number;
|
|
22
|
+
sourcePageSize?: number;
|
|
23
|
+
onError?: (error: unknown, input: RdfSearchReconciliationWorkerErrorInput) => void;
|
|
24
|
+
}
|
|
25
|
+
export interface RdfSearchEmbeddingConfig {
|
|
26
|
+
providerId: string;
|
|
27
|
+
baseUrl: string;
|
|
28
|
+
proxyUrl?: string;
|
|
29
|
+
defaultModel?: string;
|
|
30
|
+
embeddingModel?: string;
|
|
31
|
+
embeddingModelVersion?: string;
|
|
32
|
+
apiKey: string;
|
|
33
|
+
credentialId: string;
|
|
34
|
+
}
|
|
35
|
+
export interface RdfSearchReconciliationDrainResult {
|
|
36
|
+
processed: number;
|
|
37
|
+
}
|
|
38
|
+
export interface RdfSearchRememberedContextReconciliationResult {
|
|
39
|
+
contexts: number;
|
|
40
|
+
sources: number;
|
|
41
|
+
}
|
|
42
|
+
export interface RdfSearchReconciliationWorkerErrorInput {
|
|
43
|
+
phase: 'tick' | 'drain' | 'reconcile';
|
|
44
|
+
sourceKey?: string;
|
|
45
|
+
sourceUri?: string;
|
|
46
|
+
}
|
|
47
|
+
export declare class RdfSearchReconciliationWorker {
|
|
48
|
+
private readonly repository;
|
|
49
|
+
private readonly indexingService?;
|
|
50
|
+
private readonly contextRegistry;
|
|
51
|
+
private readonly store?;
|
|
52
|
+
private readonly rdfEngine?;
|
|
53
|
+
private readonly workerId;
|
|
54
|
+
private readonly now;
|
|
55
|
+
private readonly leaseDurationMs;
|
|
56
|
+
private readonly maxBatchSize;
|
|
57
|
+
private readonly intervalMs;
|
|
58
|
+
private readonly reconcileIntervalMs;
|
|
59
|
+
private readonly retryBaseDelayMs;
|
|
60
|
+
private readonly retryMaxDelayMs;
|
|
61
|
+
private readonly sourcePageSize;
|
|
62
|
+
private readonly onError?;
|
|
63
|
+
private timer;
|
|
64
|
+
private running;
|
|
65
|
+
constructor(options: RdfSearchReconciliationWorkerOptions);
|
|
66
|
+
start(): void;
|
|
67
|
+
stop(): void;
|
|
68
|
+
drain(): Promise<RdfSearchReconciliationDrainResult>;
|
|
69
|
+
reconcileRememberedContexts(): Promise<RdfSearchRememberedContextReconciliationResult>;
|
|
70
|
+
private process;
|
|
71
|
+
private recordResult;
|
|
72
|
+
private contextForSource;
|
|
73
|
+
private desiredProfile;
|
|
74
|
+
private nextAttemptAt;
|
|
75
|
+
private reportError;
|
|
76
|
+
}
|