@undefineds.co/xpod 0.3.74 → 0.4.0-rc.139

package/config/resolver.json

@@ -381,14 +381,16 @@
           "CombinedShorthandResolver:_resolvers_key": "urn:solid-server:default:variable:acmeRenewBeforeDays",
           "CombinedShorthandResolver:_resolvers_value": {
             "@type": "KeyExtractor",
-            "key": "acmeRenewBeforeDays"
+            "key": "acmeRenewBeforeDays",
+            "defaultValue": 15
           }
         },
         {
           "CombinedShorthandResolver:_resolvers_key": "urn:solid-server:default:variable:acmePropagationDelayMs",
           "CombinedShorthandResolver:_resolvers_value": {
             "@type": "KeyExtractor",
-            "key": "acmePropagationDelayMs"
+            "key": "acmePropagationDelayMs",
+            "defaultValue": 0
           }
         },
         {

package/config/terminal.json

@@ -16,7 +16,10 @@
       "maxTotalSessions": 100,
       "defaultTimeout": 3600,
       "maxTimeout": 86400,
-      "defaultWorkdir": "/workspace"
+      "defaultWorkdir": "/workspace",
+      "sparqlEngine": {
+        "@id": "urn:undefineds:xpod:DefaultSparqlEngine"
+      }
     }
   ]
 }

package/config/xpod.base.json

@@ -9,6 +9,17 @@
     "./resolver.json"
   ],
   "@graph": [
+    {
+      "comment": "Close the profile-owned RDF authority after writes stop.",
+      "@id": "urn:solid-server:default:CleanupFinalizer",
+      "@type": "SequenceHandler",
+      "handlers": [
+        {
+          "@type": "FinalizableHandler",
+          "finalizable": { "@id": "urn:undefineds:xpod:SolidRdfDataAccessor" }
+        }
+      ]
+    },
     {
       "comment": "Core mode: server or local",
       "@id": "urn:solid-server:default:variable:edition",
@@ -20,15 +31,12 @@
       "@type": "Variable"
     },
     {
-      "comment": "An accessor for a Quadstore",
-      "@type": "QuadstoreSparqlDataAccessor",
-      "@id": "urn:undefineds:xpod:QuadstoreSparqlDataAccessor",
-      "endpoint": {
-        "@id": "urn:solid-server:default:variable:sparqlEndpoint",
+      "comment": "Durable search intent sink for committed RDF/text source changes.",
+      "@id": "urn:undefineds:xpod:RdfSearchIntentSink",
+      "@type": "RdfSearchReconciliationIntentSink",
+      "identityDbUrl": {
+        "@id": "urn:solid-server:default:variable:identityDbUrl",
         "@type": "Variable"
-      },
-      "identifierStrategy": {
-        "@id": "urn:solid-server:default:IdentifierStrategy"
       }
     },
     {

package/config/xpod.json

@@ -111,34 +111,12 @@
       "@value": "./data"
     },
     {
-      "comment": "Compatibility QuintStore instance (SQLite TEXT quints for migration/oracle paths).",
-      "@id": "urn:undefineds:xpod:CompatibilityQuintStore",
-      "@type": "SqliteQuintStore",
-      "options_path": {
-        "@id": "urn:solid-server:default:variable:sparqlEndpoint",
-        "@type": "Variable"
-      }
-    },
-    {
-      "comment": "Compatibility write bridge that mirrors legacy quints into the term-id RDF index.",
-      "@id": "urn:undefineds:xpod:QuintStore",
-      "@type": "ShadowRdfQuintStore",
-      "compatibilityStore": { "@id": "urn:undefineds:xpod:CompatibilityQuintStore" },
-      "index_path": {
-        "@id": "urn:solid-server:default:variable:rdfIndexPath",
-        "@type": "Variable"
-      },
-      "autoOpen": true,
-      "autoBackfill_enabled": true,
-      "autoBackfill_clear": true,
-      "autoBackfill_batchSize": 1000
-    },
-    {
-      "comment": "VectorStore using SQLite",
+      "comment": "VectorStore using the canonical Local RDF SQLite file.",
       "@id": "urn:undefineds:xpod:VectorStore",
       "@type": "SqliteVectorStore",
       "options_connectionString": {
-        "@id": "urn:solid-server:default:variable:sparqlEndpoint"
+        "@id": "urn:solid-server:default:variable:rdfIndexPath",
+        "@type": "Variable"
       }
     },
     {
@@ -151,34 +129,69 @@
       "authorizer": { "@id": "urn:solid-server:default:Authorizer" }
     },
     {
-      "comment": "RDF term-id query engine backed by the shared local RDF index.",
+      "comment": "Local FTS index sharing the canonical RDF SQLite file.",
+      "@id": "urn:undefineds:xpod:RdfTextIndex",
+      "@type": "RdfTextIndex",
+      "options_path": {
+        "@id": "urn:solid-server:default:variable:rdfIndexPath",
+        "@type": "Variable"
+      }
+    },
+    {
+      "comment": "Local vector index sharing the canonical RDF SQLite file.",
+      "@id": "urn:undefineds:xpod:RdfVectorIndex",
+      "@type": "RdfVectorIndex",
+      "options_path": {
+        "@id": "urn:solid-server:default:variable:rdfIndexPath",
+        "@type": "Variable"
+      },
+      "options_defaultMetric": "cosine"
+    },
+    {
+      "comment": "Persistent native-only QLever process backed by the canonical RDF SQLite file.",
+      "@id": "urn:undefineds:xpod:LocalQleverNativeSparqlClient",
+      "@type": "LocalQleverNativeSparqlClient",
+      "options_args": [
+        "--sqlite-path",
+        {
+          "@id": "urn:solid-server:default:variable:rdfIndexPath",
+          "@type": "Variable"
+        }
+      ]
+    },
+    {
+      "comment": "RDF facts, FTS, VEC, and native QLever share one Local SQLite file.",
       "@id": "urn:undefineds:xpod:SolidRdfEngine",
       "@type": "SolidRdfEngine",
       "index_path": {
         "@id": "urn:solid-server:default:variable:rdfIndexPath",
         "@type": "Variable"
       },
+      "textIndex": {
+        "@id": "urn:undefineds:xpod:RdfTextIndex"
+      },
+      "vectorIndex": {
+        "@id": "urn:undefineds:xpod:RdfVectorIndex"
+      },
+      "nativeSparqlClient": {
+        "@id": "urn:undefineds:xpod:LocalQleverNativeSparqlClient"
+      },
       "autoOpen": true
     },
     {
-      "comment": "Compatibility SPARQL engine backed by the shadow store; not wired into DefaultSparqlEngine by default.",
-      "@id": "urn:undefineds:xpod:CompatibilitySparqlEngine",
-      "@type": "QuintstoreSparqlEngine",
-      "store": { "@id": "urn:undefineds:xpod:QuintStore" }
-    },
-    {
-      "comment": "Default SPARQL engine: server-owned Pod queries use SolidRdfEngine; compatibility fallback is opt-in.",
+      "comment": "Default SPARQL engine: every server-owned Pod query requires native QLever.",
       "@id": "urn:undefineds:xpod:DefaultSparqlEngine",
-      "@type": "SolidRdfSparqlEngine",
-      "rdfEngine": { "@id": "urn:undefineds:xpod:SolidRdfEngine" },
-      "enablePrimary": true
+      "@type": "QleverSparqlEngine",
+      "rdfEngine": { "@id": "urn:undefineds:xpod:SolidRdfEngine" }
     },
     {
       "comment": "MixDataAccessor for RDF and binary data.",
       "@id": "urn:undefineds:xpod:MixDataAccessor",
       "@type": "MixDataAccessor",
       "structuredDataAccessor": { "@id": "urn:undefineds:xpod:SolidRdfDataAccessor" },
-      "unstructuredDataAccessor": { "@id": "urn:solid-server:default:FileDataAccessor" }
+      "unstructuredDataAccessor": { "@id": "urn:solid-server:default:FileDataAccessor" },
+      "textSearchIndexingEnabled": true,
+      "rdfSearchIntentSink": { "@id": "urn:undefineds:xpod:RdfSearchIntentSink" }
     },
     {
       "comment": "Default structured RDF data accessor backed directly by SolidRdfEngine.",
@@ -188,23 +201,7 @@
       "identifierStrategy": { "@id": "urn:solid-server:default:IdentifierStrategy" }
     },
     {
-      "comment": "Compatibility SPARQL data accessor using the legacy QuintStore bridge.",
-      "@id": "urn:undefineds:xpod:QuintStoreSparqlDataAccessor",
-      "@type": "QuintStoreSparqlDataAccessor",
-      "store": { "@id": "urn:undefineds:xpod:QuintStore" },
-      "identifierStrategy": { "@id": "urn:solid-server:default:IdentifierStrategy" }
-    },
-    {
-      "comment": "SPARQL-capable store for quadstore-backed resources",
-      "@id": "urn:undefineds:xpod:SparqlQuadstoreResourceStore",
-      "@type": "SparqlUpdateResourceStore",
-      "identifierStrategy": { "@id": "urn:solid-server:default:IdentifierStrategy" },
-      "auxiliaryStrategy": { "@id": "urn:solid-server:default:AuxiliaryStrategy" },
-      "metadataStrategy": { "@id": "urn:solid-server:default:MetadataStrategy" },
-      "accessor": { "@id": "urn:undefineds:xpod:MixDataAccessor" }
-    },
-    {
-      "comment": "Override: ResourceStore 使用 QuintStore",
+      "comment": "Override: ResourceStore 使用 RDF authority accessor",
       "@type": "Override",
       "overrideInstance": { "@id": "urn:solid-server:default:ResourceStore_Backend" },
       "overrideParameters": {

package/dist/acceptance/PublicCloudSemanticConformance.d.ts

@@ -0,0 +1,4 @@
+import type { RdfEngineLike } from '../storage/rdf/types';
+import { type PostgresPublicSemanticHarnessOptions, type SemanticConformanceReport, type SemanticFixtureCase } from './RdfSemanticConformance';
+export declare function runPostgresPublicSemanticConformance(options: PostgresPublicSemanticHarnessOptions): Promise<SemanticConformanceReport>;
+export declare function runPublicRdfQuerySemanticCase(engine: RdfEngineLike, testCase: SemanticFixtureCase, timeoutMs: number | undefined, authority: string): Promise<SemanticConformanceReport['results'][number]>;

package/dist/acceptance/PublicCloudSemanticConformance.js

@@ -0,0 +1,214 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.runPostgresPublicSemanticConformance = runPostgresPublicSemanticConformance;
+exports.runPublicRdfQuerySemanticCase = runPublicRdfQuerySemanticCase;
+const node_crypto_1 = require("node:crypto");
+const node_fs_1 = require("node:fs");
+const node_path_1 = __importDefault(require("node:path"));
+const pg_1 = require("pg");
+const sparqljs_1 = require("sparqljs");
+const PostgresRdfEngine_1 = require("../storage/rdf/PostgresRdfEngine");
+const RdfQuerySparqlEngine_1 = require("../storage/rdf/RdfQuerySparqlEngine");
+const RdfSparqlAdapter_1 = require("../storage/rdf/RdfSparqlAdapter");
+const RdfAccessScope_1 = require("../storage/rdf/RdfAccessScope");
+const RdfSemanticConformance_1 = require("./RdfSemanticConformance");
+async function runPostgresPublicSemanticConformance(options) {
+    (0, RdfSemanticConformance_1.assertPath)('fixturePath', options.fixturePath);
+    (0, RdfSemanticConformance_1.assertPath)('artifactPath', options.artifactPath);
+    if (!options.connectionString) {
+        throw new Error('connectionString is required');
+    }
+    const fixture = require(node_path_1.default.resolve(options.fixturePath));
+    const admin = new pg_1.Pool({ connectionString: options.connectionString, max: 1 });
+    const results = [];
+    const failed = [];
+    try {
+        for (let index = 0; index < fixture.semanticConformanceCases.length; index += 1) {
+            const testCase = fixture.semanticConformanceCases[index];
+            const schema = `xpod_semantic_public_${process.pid}_${index}_${(0, node_crypto_1.randomUUID)().replaceAll('-', '')}`;
+            try {
+                await admin.query(`CREATE SCHEMA ${(0, RdfSemanticConformance_1.quoteIdentifier)(schema)}`);
+                const connectionString = (0, RdfSemanticConformance_1.connectionStringWithSearchPath)(options.connectionString, schema);
+                const engine = new PostgresRdfEngine_1.PostgresRdfEngine({
+                    driver: 'pg',
+                    connectionString,
+                    rdfAccelerationProfile: 'pg-hot-operators',
+                    maintenanceIntervalMs: 0,
+                });
+                results.push(await runPublicRdfQuerySemanticCase(engine, testCase, options.timeoutMs, `postgres-public-schema:${schema}`));
+            }
+            catch (error) {
+                failed.push({
+                    caseId: testCase.id,
+                    message: error instanceof Error ? error.message : String(error),
+                });
+            }
+            finally {
+                await admin.query(`DROP SCHEMA IF EXISTS ${(0, RdfSemanticConformance_1.quoteIdentifier)(schema)} CASCADE`);
+            }
+        }
+    }
+    finally {
+        await admin.end();
+    }
+    const report = (0, RdfSemanticConformance_1.buildSemanticReport)({
+        backend: 'pg',
+        engine: 'pg-public-rdf-query-authority',
+        caseIds: fixture.REQUIRED_CASES,
+        failed,
+        results,
+        sourceDeniedValidatedBy: 'allowedSourceUrls/deniedSourceUrls resolved by PG RDF query authority',
+    });
+    (0, node_fs_1.mkdirSync)(node_path_1.default.dirname(options.artifactPath), { recursive: true });
+    (0, node_fs_1.writeFileSync)(options.artifactPath, `${JSON.stringify(report, null, 2)}\n`);
+    return report;
+}
+async function runPublicRdfQuerySemanticCase(engine, testCase, timeoutMs, authority) {
+    return (0, RdfSemanticConformance_1.runCaseWithEngine)(engine, testCase, timeoutMs, authority, true, publicRdfQueryExecutor);
+}
+const publicRdfQueryExecutor = {
+    prepareAndApplyUpdate: prepareAndApplyPublicRdfQueryUpdate,
+    query: queryPublicRdfQueryEngine,
+};
+async function prepareAndApplyPublicRdfQueryUpdate(engine, sparql, sourceUri, _timeoutMs) {
+    const adapter = new RdfSparqlAdapter_1.RdfSparqlAdapter();
+    const delta = adapter.compileUpdateDelta(sparql, RdfSemanticConformance_1.DEFAULT_BASE_PATH, {
+        defaultGraph: sourceUri,
+    });
+    const writeScope = writeAccessScope(sourceUri);
+    const graphDeltas = new Map();
+    const appendDeletes = (quads) => {
+        for (const quad of quads) {
+            graphDeltaForPublicUpdate(graphDeltas, quad).deletes.push(quad);
+        }
+    };
+    const appendInserts = (quads) => {
+        for (const quad of quads) {
+            graphDeltaForPublicUpdate(graphDeltas, quad).inserts.push(quad);
+        }
+    };
+    for (const operation of delta.operations) {
+        if (operation.type === 'insert') {
+            appendInserts(operation.quads ?? []);
+        }
+        else if (operation.type === 'delete') {
+            appendDeletes(operation.quads ?? []);
+        }
+        else if (operation.type === 'insertDeleteWhere') {
+            const result = await engine.query((0, RdfAccessScope_1.applyRdfAccessScope)(assertPublicUpdateQuery(operation.query), writeScope));
+            appendDeletes(adapter.materializeDeleteWhere(operation.deletes, result.bindings));
+            appendInserts(adapter.materializeDeleteWhere(operation.inserts, result.bindings));
+        }
+        else if (operation.type === 'insertWhere') {
+            const result = await engine.query((0, RdfAccessScope_1.applyRdfAccessScope)(assertPublicUpdateQuery(operation.query), writeScope));
+            appendInserts(adapter.materializeDeleteWhere(operation.inserts, result.bindings));
+        }
+        else {
+            const result = await engine.query((0, RdfAccessScope_1.applyRdfAccessScope)(assertPublicUpdateQuery(operation.query), writeScope));
+            appendDeletes(adapter.materializeDeleteWhere(operation.template, result.bindings));
+        }
+    }
+    const summary = { deletedRows: 0, insertedRows: 0 };
+    for (const graph of graphDeltas.values()) {
+        const applied = await engine.applyDelta(graph.deletes.map(RdfSemanticConformance_1.exactQuadPattern), graph.inserts, {
+            source: {
+                source: graph.sourceUri,
+                workspace: 'semantic',
+                localPath: `/semantic/${(0, RdfSemanticConformance_1.slug)(graph.sourceUri)}`,
+                contentType: 'text/turtle',
+            },
+        });
+        summary.deletedRows += applied.deletedRows;
+        summary.insertedRows += applied.insertedRows;
+    }
+    await engine.refreshDerivedIndexes({ mode: 'full' });
+    return summary;
+}
+function graphDeltaForPublicUpdate(deltas, quad) {
+    if (quad.graph.termType !== 'NamedNode') {
+        throw new Error('Public semantic conformance updates must resolve to a named source graph');
+    }
+    const graphIri = quad.graph.value;
+    let delta = deltas.get(graphIri);
+    if (!delta) {
+        delta = {
+            graphIri,
+            sourceUri: graphIri,
+            deletes: [],
+            inserts: [],
+        };
+        deltas.set(graphIri, delta);
+    }
+    return delta;
+}
+async function queryPublicRdfQueryEngine(engine, testCase, timeoutMs) {
+    const sparql = new RdfQuerySparqlEngine_1.RdfQuerySparqlEngine(engine);
+    const accessScope = (0, RdfSemanticConformance_1.makeAccessScope)(testCase);
+    if (testCase.acceptMediaType === 'application/n-triples') {
+        const quads = await sparql.queryQuads(testCase.query, RdfSemanticConformance_1.DEFAULT_BASE_PATH, accessScope, { timeoutMs });
+        return {
+            status: 'ok',
+            mediaType: 'application/n-triples',
+            body: await (0, RdfSemanticConformance_1.serializeQuads)(quads),
+            queryStatus: 0,
+        };
+    }
+    if (queryType(testCase.query) === 'ASK') {
+        const result = await sparql.queryBoolean(testCase.query, RdfSemanticConformance_1.DEFAULT_BASE_PATH, accessScope, { timeoutMs });
+        return {
+            status: 'ok',
+            mediaType: 'application/sparql-results+json',
+            body: JSON.stringify({ head: {}, boolean: result }),
+            queryStatus: 0,
+        };
+    }
+    const bindings = await sparql.queryBindings(testCase.query, RdfSemanticConformance_1.DEFAULT_BASE_PATH, accessScope, { timeoutMs });
+    const metadata = typeof bindings.metadata === 'function' ? await bindings.metadata() : undefined;
+    const variables = metadata?.variables?.map((variable) => variable.value) ?? [];
+    const rows = [];
+    const seenVariables = new Set();
+    for await (const binding of bindings) {
+        const row = {};
+        for (const [variable, term] of binding) {
+            const name = typeof variable === 'string' ? variable : variable.value;
+            row[name] = (0, RdfSemanticConformance_1.termToSparqlJson)(term);
+            seenVariables.add(name);
+        }
+        rows.push(row);
+    }
+    return {
+        status: 'ok',
+        mediaType: 'application/sparql-results+json',
+        body: JSON.stringify({
+            head: { vars: variables.length > 0 ? variables : [...seenVariables] },
+            results: { bindings: rows },
+        }),
+        queryStatus: 0,
+    };
+}
+function queryType(query) {
+    const parsed = new sparqljs_1.Parser({ baseIRI: RdfSemanticConformance_1.DEFAULT_BASE_PATH }).parse(query);
+    if (parsed.type !== 'query') {
+        throw new Error(`Expected SPARQL query, received ${parsed.type}`);
+    }
+    return parsed.queryType;
+}
+function assertPublicUpdateQuery(query) {
+    if (!query) {
+        throw new Error('Public semantic conformance update operation is missing its RDF query');
+    }
+    return query;
+}
+function writeAccessScope(sourceUri) {
+    return {
+        basePath: RdfSemanticConformance_1.DEFAULT_BASE_PATH,
+        mode: 'write',
+        resolved: true,
+        principal: 'urn:xpod:semantic-writer',
+        version: `semantic-write-${sourceUri}`,
+    };
+}
+//# sourceMappingURL=PublicCloudSemanticConformance.js.map

package/dist/acceptance/QleverSearchConformance.d.ts

@@ -0,0 +1,39 @@
+import { PostgresRdfEngine } from '../storage/rdf/PostgresRdfEngine';
+import { SolidRdfEngine } from '../storage/rdf/SolidRdfEngine';
+import type { RdfEngineLike } from '../storage/rdf/types';
+export declare const QLEVER_ACCEPTANCE_GATE_ENV = "XPOD_QLEVER_ACCEPTANCE_GATE";
+export declare function qleverAcceptanceGateEnabled(): boolean;
+export declare function requireAcceptanceEnv(name: string): string;
+export declare function createAcceptanceTempDir(name: string): string;
+export interface SearchCanonicalRow {
+    source: string;
+    sourceKey: string;
+    retrievalPointKey: string;
+    textSnippet: string;
+    vectorSnippet: string;
+}
+export declare function runLocalSearchFusionAcceptance(engine?: SolidRdfEngine): Promise<SearchCanonicalRow[]>;
+export interface NativeSearchConformanceOptions {
+    workspace?: string;
+}
+export interface NativeSearchConformanceReport {
+    textOnlyBeforeVector: NativeSearchRow[];
+    fusedBeforeVector: NativeSearchRow[];
+    fusedAfterVector: NativeSearchRow[];
+    fusedAfterVectorExact: NativeSearchRow[];
+    fusedDuringMove: NativeSearchRow[];
+    fusedDuringMoveExact: NativeSearchRow[];
+    fusedAfterMove: NativeSearchRow[];
+    oldSourceAfterMove: NativeSearchRow[];
+    deniedSource: NativeSearchRow[];
+}
+export interface NativeSearchRow {
+    retrieval: string;
+    source?: string;
+}
+export declare function createLocalNativeSearchEngine(runtimeCommand: string, databasePath: string): SolidRdfEngine;
+export declare function createPostgresSearchEngine(connectionString: string, nativeSparqlEnabled: boolean): PostgresRdfEngine;
+export declare function runPostgresNativeSearchFusionAcceptance(connectionString: string): Promise<NativeSearchConformanceReport>;
+export declare function runPostgresPublicSearchFusionAcceptance(connectionString: string): Promise<NativeSearchConformanceReport>;
+export declare function runNativeSearchFusionAcceptance(engine: RdfEngineLike, options?: NativeSearchConformanceOptions): Promise<NativeSearchConformanceReport>;
+export declare function runPublicSearchFusionAcceptance(engine: RdfEngineLike, options?: NativeSearchConformanceOptions): Promise<NativeSearchConformanceReport>;