@undefineds.co/xpod 0.3.74 → 0.4.0-rc.139

package/dist/api/service/RdfSearchReconciliationWorker.js

@@ -0,0 +1,294 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RdfSearchReconciliationWorker = void 0;
+const node_crypto_1 = require("node:crypto");
+const AuthContext_1 = require("../auth/AuthContext");
+const RdfSearchIndexingService_1 = require("./RdfSearchIndexingService");
+const CONFIG_FAILURES = new Set([
+    'ai_config_unavailable',
+    'embedding_model_unavailable',
+    'embedding_authentication_failed',
+    'embedding_authorization_failed',
+    'embedding_model_invalid',
+    'embedding_request_invalid',
+]);
+class RdfSearchReconciliationWorker {
+    constructor(options) {
+        this.running = false;
+        this.repository = options.repository;
+        this.indexingService = options.indexingService;
+        this.contextRegistry = options.contextRegistry;
+        this.store = options.store;
+        this.rdfEngine = options.rdfEngine;
+        this.workerId = options.workerId ?? `rdf-search-reconciler-${(0, node_crypto_1.randomUUID)()}`;
+        this.now = options.now ?? Date.now;
+        this.leaseDurationMs = options.leaseDurationMs ?? 60_000;
+        this.maxBatchSize = options.maxBatchSize ?? 10;
+        this.intervalMs = options.intervalMs ?? 30_000;
+        this.reconcileIntervalMs = options.reconcileIntervalMs ?? 120_000;
+        this.retryBaseDelayMs = options.retryBaseDelayMs ?? 60_000;
+        this.retryMaxDelayMs = options.retryMaxDelayMs ?? 15 * 60_000;
+        this.sourcePageSize = options.sourcePageSize ?? 1_000;
+        this.onError = options.onError;
+    }
+    start() {
+        if (this.timer) {
+            return;
+        }
+        let lastReconcileAt = 0;
+        const tick = async () => {
+            if (this.running) {
+                return;
+            }
+            this.running = true;
+            try {
+                const now = this.now();
+                if (now - lastReconcileAt >= this.reconcileIntervalMs) {
+                    lastReconcileAt = now;
+                    await this.reconcileRememberedContexts();
+                }
+                await this.drain();
+            }
+            finally {
+                this.running = false;
+            }
+        };
+        this.timer = setInterval(() => {
+            void tick().catch((error) => this.reportError(error, { phase: 'tick' }));
+        }, this.intervalMs);
+        void tick().catch((error) => this.reportError(error, { phase: 'tick' }));
+    }
+    stop() {
+        if (!this.timer) {
+            return;
+        }
+        clearInterval(this.timer);
+        this.timer = undefined;
+    }
+    async drain() {
+        if (!this.indexingService) {
+            return { processed: 0 };
+        }
+        let processed = 0;
+        for (let index = 0; index < this.maxBatchSize; index += 1) {
+            const now = new Date(this.now());
+            const row = await this.repository.claimNext(this.workerId, now, this.leaseDurationMs);
+            if (!row) {
+                break;
+            }
+            try {
+                await this.process(row, now);
+            }
+            catch (error) {
+                this.reportError(error, { phase: 'drain', sourceKey: row.sourceKey, sourceUri: row.sourceUri });
+                try {
+                    await this.repository.markRetryable(row.sourceKey, this.workerId, 'embedding_reconciliation_failed', this.nextAttemptAt(row, now), now);
+                }
+                catch (retryError) {
+                    this.reportError(retryError, { phase: 'drain', sourceKey: row.sourceKey, sourceUri: row.sourceUri });
+                }
+            }
+            processed += 1;
+        }
+        return { processed };
+    }
+    async reconcileRememberedContexts() {
+        if (!this.rdfEngine?.listTextSources || !this.store) {
+            return { contexts: 0, sources: 0 };
+        }
+        let contexts = 0;
+        let sources = 0;
+        const plannedScopes = new Map();
+        if (this.store) {
+            for (const context of this.contextRegistry.list()) {
+                const scope = sourcePrefixForContext(context);
+                if (!scope) {
+                    continue;
+                }
+                try {
+                    const profile = await this.desiredProfile(context);
+                    if (profile) {
+                        plannedScopes.set(scope, profile);
+                    }
+                }
+                catch (error) {
+                    this.reportError(error, { phase: 'reconcile' });
+                }
+            }
+        }
+        for (const [scope, profile] of plannedScopes) {
+            contexts += 1;
+            let offset = 0;
+            while (true) {
+                let page;
+                try {
+                    page = await this.rdfEngine.listTextSources({
+                        sourcePrefix: scope,
+                        limit: this.sourcePageSize,
+                        offset,
+                    });
+                }
+                catch (error) {
+                    this.reportError(error, { phase: 'reconcile' });
+                    break;
+                }
+                if (page.length === 0) {
+                    break;
+                }
+                for (const source of page) {
+                    try {
+                        await this.repository.upsertDesired({
+                            sourceKey: source.sourceKey ?? source.source,
+                            sourceUri: source.source,
+                            podRoot: source.workspace,
+                            sourceHash: source.sourceHash,
+                            sourceVersion: source.sourceVersion,
+                            ...profile,
+                            reason: 'embedding-config-observed',
+                        }, new Date(this.now()));
+                    }
+                    catch (error) {
+                        this.reportError(error, { phase: 'reconcile', sourceKey: source.sourceKey ?? source.source, sourceUri: source.source });
+                        continue;
+                    }
+                    sources += 1;
+                }
+                if (page.length < this.sourcePageSize) {
+                    break;
+                }
+                offset += page.length;
+            }
+        }
+        return { contexts, sources };
+    }
+    async process(row, now) {
+        const context = this.contextForSource(row.sourceUri);
+        if (context) {
+            const result = await this.indexingService.rebuildVectorSource({
+                context,
+                sourceKey: row.sourceKey,
+            });
+            await this.recordResult(row, result, now);
+            return;
+        }
+        await this.repository.markRetryable(row.sourceKey, this.workerId, 'auth_context_unavailable', this.nextAttemptAt(row, now), now);
+    }
+    async recordResult(row, result, now) {
+        if (result.status === 'indexed') {
+            await this.repository.complete(row.sourceKey, this.workerId, now);
+            return;
+        }
+        if (result.status === 'skipped') {
+            if (result.reason === 'text_source_unavailable') {
+                await this.repository.deleteSource(row.sourceKey);
+                return;
+            }
+            if (CONFIG_FAILURES.has(result.reason)) {
+                await this.repository.markBlockedConfig(row.sourceKey, this.workerId, result.reason, now);
+            }
+            else {
+                await this.repository.markRetryable(row.sourceKey, this.workerId, result.reason, this.nextAttemptAt(row, now), now);
+            }
+            return;
+        }
+        await this.repository.markRetryable(row.sourceKey, this.workerId, result.reason, this.nextAttemptAt(row, now), now);
+    }
+    contextForSource(sourceUri) {
+        return this.contextRegistry.list().find((context) => {
+            const prefix = sourcePrefixForContext(context);
+            return !!prefix && isUrlWithinPrefix(sourceUri, prefix);
+        });
+    }
+    async desiredProfile(context) {
+        const config = await this.store?.getAiConfig(context);
+        return config ? desiredProfileFromConfig(config) : undefined;
+    }
+    nextAttemptAt(row, now) {
+        const exponent = Math.max(0, row.attemptCount);
+        const delay = Math.min(this.retryMaxDelayMs, this.retryBaseDelayMs * (2 ** exponent));
+        return new Date(now.getTime() + delay);
+    }
+    reportError(error, input) {
+        this.onError?.(error, input);
+    }
+}
+exports.RdfSearchReconciliationWorker = RdfSearchReconciliationWorker;
+function desiredProfileFromConfig(config) {
+    if (!config.apiKey || !config.embeddingModel) {
+        return undefined;
+    }
+    const modelVersion = (0, RdfSearchIndexingService_1.normalizeRdfVectorModelVersion)(config.embeddingModelVersion);
+    return {
+        providerId: config.providerId,
+        model: config.embeddingModel,
+        modelVersion,
+        configFingerprint: embeddingProfileFingerprint({
+            providerId: config.providerId,
+            embeddingModel: config.embeddingModel,
+            embeddingModelVersion: modelVersion,
+            credentialId: config.credentialId,
+        }),
+    };
+}
+function sourcePrefixForContext(context) {
+    const rdfAccessScope = context.rdfAccessScope;
+    if (typeof rdfAccessScope?.basePath === 'string' && rdfAccessScope.basePath) {
+        return rdfAccessScope.basePath;
+    }
+    const auth = context.auth;
+    if (!(0, AuthContext_1.hasSolidClientCredentialsAuthority)(auth)) {
+        return undefined;
+    }
+    return solidPodRootFromWebId(auth.webId);
+}
+function solidPodRootFromWebId(webId) {
+    try {
+        const url = new URL(webId);
+        const marker = '/profile/card';
+        const index = url.pathname.indexOf(marker);
+        if (index < 0) {
+            return undefined;
+        }
+        url.pathname = `${url.pathname.slice(0, index)}/`;
+        url.hash = '';
+        url.search = '';
+        return url.href;
+    }
+    catch {
+        return undefined;
+    }
+}
+function isUrlWithinPrefix(sourceUri, rawPrefix) {
+    try {
+        const source = new URL(sourceUri);
+        const prefix = new URL(rawPrefix);
+        if (source.origin !== prefix.origin) {
+            return false;
+        }
+        const sourcePath = normalizeSourcePath(source.pathname);
+        const prefixPath = normalizeDirectoryPath(prefix.pathname);
+        return sourcePath === prefixPath.slice(0, -1) || sourcePath.startsWith(prefixPath);
+    }
+    catch {
+        const prefix = rawPrefix.endsWith('/') ? rawPrefix : `${rawPrefix}/`;
+        return sourceUri === rawPrefix || sourceUri.startsWith(prefix);
+    }
+}
+function normalizeDirectoryPath(pathname) {
+    const normalized = normalizeSourcePath(pathname);
+    return normalized.endsWith('/') ? normalized : `${normalized}/`;
+}
+function normalizeSourcePath(pathname) {
+    return pathname.startsWith('/') ? pathname : `/${pathname}`;
+}
+function embeddingProfileFingerprint(input) {
+    return `sha256:${(0, node_crypto_1.createHash)('sha256')
+        .update([
+        input.providerId,
+        input.embeddingModel,
+        (0, RdfSearchIndexingService_1.normalizeRdfVectorModelVersion)(input.embeddingModelVersion),
+        input.credentialId,
+    ].join('\0'))
+        .digest('hex')}`;
+}
+//# sourceMappingURL=RdfSearchReconciliationWorker.js.map

package/dist/api/tasks/InngestTaskScheduler.d.ts

@@ -135,7 +135,7 @@ export declare class InngestTaskScheduler<TContext = StoreContext> {
         readonly event: string;
     }], Omit<Omit<import("inngest/types").BaseContext<import("inngest").Inngest.Any>, never> & Record<never, never> & {
         logger: import("inngest").Logger;
-    }, "group" | "requestId" | "event" | "attempt" | "events" | "runId" | "step" | "defer" | "jobId" | "maxAttempts"> & import("inngest/types").FailureEventArgs<EventPayload<any>> & {
+    }, "group" | "step" | "requestId" | "event" | "attempt" | "events" | "runId" | "defer" | "jobId" | "maxAttempts"> & import("inngest/types").FailureEventArgs<EventPayload<any>> & {
         step: {
             sendEvent: (idOrOptions: import("inngest/types").StepOptionsOrId, payload: import("inngest").SendEventPayload) => Promise<import("inngest/types").SendEventOutput<import("inngest").ClientOptionsFromInngest<import("inngest").Inngest<import("inngest/types").ClientOptions>>>>;
             waitForSignal: <TData>(idOrOptions: import("inngest/types").StepOptionsOrId, opts: {
@@ -313,7 +313,7 @@ export declare class InngestTaskScheduler<TContext = StoreContext> {
         readonly event: string;
     }], Omit<Omit<import("inngest/types").BaseContext<import("inngest").Inngest.Any>, never> & Record<never, never> & {
         logger: import("inngest").Logger;
-    }, "group" | "requestId" | "event" | "attempt" | "events" | "runId" | "step" | "defer" | "jobId" | "maxAttempts"> & import("inngest/types").FailureEventArgs<EventPayload<any>> & {
+    }, "group" | "step" | "requestId" | "event" | "attempt" | "events" | "runId" | "defer" | "jobId" | "maxAttempts"> & import("inngest/types").FailureEventArgs<EventPayload<any>> & {
         step: {
             sendEvent: (idOrOptions: import("inngest/types").StepOptionsOrId, payload: import("inngest").SendEventPayload) => Promise<import("inngest/types").SendEventOutput<import("inngest").ClientOptionsFromInngest<import("inngest").Inngest<import("inngest/types").ClientOptions>>>>;
             waitForSignal: <TData>(idOrOptions: import("inngest/types").StepOptionsOrId, opts: {
@@ -496,7 +496,7 @@ export declare class InngestTaskScheduler<TContext = StoreContext> {
         readonly event: string;
     }], Omit<Omit<import("inngest/types").BaseContext<import("inngest").Inngest.Any>, never> & Record<never, never> & {
         logger: import("inngest").Logger;
-    }, "group" | "requestId" | "event" | "attempt" | "events" | "runId" | "step" | "defer" | "jobId" | "maxAttempts"> & import("inngest/types").FailureEventArgs<EventPayload<any>> & {
+    }, "group" | "step" | "requestId" | "event" | "attempt" | "events" | "runId" | "defer" | "jobId" | "maxAttempts"> & import("inngest/types").FailureEventArgs<EventPayload<any>> & {
         step: {
             sendEvent: (idOrOptions: import("inngest/types").StepOptionsOrId, payload: import("inngest").SendEventPayload) => Promise<import("inngest/types").SendEventOutput<import("inngest").ClientOptionsFromInngest<import("inngest").Inngest<import("inngest/types").ClientOptions>>>>;
             waitForSignal: <TData>(idOrOptions: import("inngest/types").StepOptionsOrId, opts: {
@@ -672,7 +672,7 @@ export declare class InngestTaskScheduler<TContext = StoreContext> {
         readonly event: string;
     }], Omit<Omit<import("inngest/types").BaseContext<import("inngest").Inngest.Any>, never> & Record<never, never> & {
         logger: import("inngest").Logger;
-    }, "group" | "requestId" | "event" | "attempt" | "events" | "runId" | "step" | "defer" | "jobId" | "maxAttempts"> & import("inngest/types").FailureEventArgs<EventPayload<any>> & {
+    }, "group" | "step" | "requestId" | "event" | "attempt" | "events" | "runId" | "defer" | "jobId" | "maxAttempts"> & import("inngest/types").FailureEventArgs<EventPayload<any>> & {
         step: {
             sendEvent: (idOrOptions: import("inngest/types").StepOptionsOrId, payload: import("inngest").SendEventPayload) => Promise<import("inngest/types").SendEventOutput<import("inngest").ClientOptionsFromInngest<import("inngest").Inngest<import("inngest/types").ClientOptions>>>>;
             waitForSignal: <TData>(idOrOptions: import("inngest/types").StepOptionsOrId, opts: {

package/dist/components/components.jsonld

@@ -6,12 +6,10 @@
   "@type": "Module",
   "requireName": "@undefineds.co/xpod",
   "import": [
+    "undefineds:dist/storage/accessors/MixDataAccessor.jsonld",
     "undefineds:dist/storage/sparql/SubgraphQueryEngine.jsonld",
     "undefineds:dist/storage/rdf/types.jsonld",
-    "undefineds:dist/storage/rdf/RdfSparqlAdapter.jsonld",
     "undefineds:dist/storage/rdf/RdfAccessScope.jsonld",
-    "undefineds:dist/storage/rdf/ShadowRdfQuintStore.jsonld",
-    "undefineds:dist/storage/rdf/SolidRdfSparqlEngine.jsonld",
     "undefineds:dist/storage/rdf/PostgresRdfEngine.jsonld",
     "undefineds:dist/storage/rdf/PostgresRdfTextIndex.jsonld",
     "undefineds:dist/storage/rdf/PostgresRdfVectorIndex.jsonld",
@@ -25,18 +23,12 @@
     "undefineds:dist/tunnel/TunnelProvider.jsonld",
     "undefineds:dist/subdomain/SubdomainService.jsonld",
     "undefineds:dist/http/AppStaticAssetHandler.jsonld",
-    "undefineds:dist/storage/PostgresDerivedIndexJournal.jsonld",
     "undefineds:dist/storage/ObservableResourceStore.jsonld",
-    "undefineds:dist/storage/RdfDerivedIndexingListener.jsonld",
     "undefineds:dist/storage/RepresentationPartialConvertingStore.jsonld",
     "undefineds:dist/storage/accessors/MinioDataAccessor.jsonld",
-    "undefineds:dist/storage/accessors/QuadstoreSparqlDataAccessor.jsonld",
-    "undefineds:dist/storage/accessors/QuintStoreSparqlDataAccessor.jsonld",
     "undefineds:dist/storage/accessors/SolidRdfDataAccessor.jsonld",
-    "undefineds:dist/storage/accessors/MixDataAccessor.jsonld",
     "undefineds:dist/logging/ConfigurableLoggerFactory.jsonld",
     "undefineds:dist/storage/SparqlUpdateResourceStore.jsonld",
-    "undefineds:dist/storage/sparql/CompatibilitySparqlEngine.jsonld",
     "undefineds:dist/http/SubgraphSparqlHttpHandler.jsonld",
     "undefineds:dist/http/InternalPodDataHttpHandler.jsonld",
     "undefineds:dist/http/quota/QuotaAdminHttpHandler.jsonld",
@@ -87,6 +79,9 @@
     "undefineds:dist/storage/rdf/RdfTextIndex.jsonld",
     "undefineds:dist/storage/rdf/RdfVectorIndex.jsonld",
     "undefineds:dist/storage/rdf/SolidRdfEngine.jsonld",
+    "undefineds:dist/storage/rdf/LocalQleverNativeSparqlClient.jsonld",
+    "undefineds:dist/storage/rdf/QleverSparqlEngine.jsonld",
+    "undefineds:dist/storage/rdf/RdfQuerySparqlEngine.jsonld",
     "undefineds:dist/storage/vector/VectorStore.jsonld",
     "undefineds:dist/storage/vector/SqliteVectorStore.jsonld",
     "undefineds:dist/storage/vector/PostgresVectorStore.jsonld",
@@ -116,16 +111,18 @@
     "undefineds:dist/provision/ProvisionCodeCodec.jsonld",
     "undefineds:dist/provision/LocalPodProvisioningService.jsonld",
     "undefineds:dist/solidfs/SolidFsSyncJournal.jsonld",
+    "undefineds:dist/solidfs/LocalRdfAuthorityRecoveryInitializer.jsonld",
+    "undefineds:dist/search/RdfSearchIntentSink.jsonld",
     "undefineds:dist/document/DocumentReader.jsonld",
     "undefineds:dist/document/Chunker.jsonld",
     "undefineds:dist/document/JinaReader.jsonld",
-    "undefineds:dist/document/HeadingChunker.jsonld",
+    "undefineds:dist/document/MarkdownRetrievalPointProjector.jsonld",
     "undefineds:dist/document/ReaderPolicy.jsonld",
     "undefineds:dist/document/ReaderAiConfig.jsonld",
     "undefineds:dist/document/L0SourceSummary.jsonld",
     "undefineds:dist/document/PaddleOcrReader.jsonld",
-    "undefineds:dist/security/secret-cell/DeploymentRootKeyProvider.jsonld",
-    "undefineds:dist/security/secret-cell/SecretCellVault.jsonld",
+    "undefineds:dist/document/ReaderMaterialization.jsonld",
+    "undefineds:dist/document/ReaderMaterializationRepository.jsonld",
     "undefineds:dist/api/reconciler/coordination.jsonld",
     "undefineds:dist/api/reconciler/ClientReconcilerCoordinator.jsonld",
     "undefineds:dist/api/reconciler/WakeAgentQueue.jsonld",