@undefineds.co/xpod 0.3.58 → 0.3.61

package/dist/storage/rdf/Rdf3xIndex.js

@@ -7,6 +7,7 @@ const n3_1 = require("n3");
 const SqliteRuntime_1 = require("../SqliteRuntime");
 const RdfTermDictionary_1 = require("./RdfTermDictionary");
 const RdfTermSemantics_1 = require("./RdfTermSemantics");
+const RdfJoinShape_1 = require("./RdfJoinShape");
 const Rdf3xSchema_1 = require("./Rdf3xSchema");
 const XSD_INTEGER = 'http://www.w3.org/2001/XMLSchema#integer';
 const XSD_DECIMAL = 'http://www.w3.org/2001/XMLSchema#decimal';
@@ -79,12 +80,25 @@ class Rdf3xIndex {
     }
     clear() {
         this.clearRdf3xTables();
+        this.clearDirtyProjectionTables();
         this.setFactsDataVersion(0);
     }
-    rebuildFromCurrentQuads() {
+    rebuildFromCurrentQuads(options) {
         const start = Date.now();
-        const db = this.requireDb();
         const factsDataVersion = this.currentFactsDataVersion();
+        if (options?.mode === 'auto') {
+            const dirty = this.dirtyProjectionStats();
+            if (dirty.graphs > 0 || dirty.pairs > 0 || dirty.terms > 0) {
+                return this.refreshDirtyProjectionTables(start, factsDataVersion, dirty);
+            }
+            if (this.factsDataVersion() !== factsDataVersion) {
+                return this.rebuildAllProjectionTables(start, factsDataVersion);
+            }
+        }
+        return this.rebuildAllProjectionTables(start, factsDataVersion);
+    }
+    rebuildAllProjectionTables(start, factsDataVersion) {
+        const db = this.requireDb();
         const scannedQuads = db.prepare('SELECT COUNT(*) AS count FROM rdf_quads').get()?.count ?? 0;
         db.transaction(() => {
             this.clearRdf3xTables();
@@ -95,16 +109,45 @@ class Rdf3xIndex {
                 this.rebuildTermProjection(projection);
             }
             this.rebuildGraphProjection();
+            this.clearDirtyProjectionTables();
+            this.setFactsDataVersion(factsDataVersion);
+        })();
+        const stats = this.stats();
+        return {
+            mode: 'full',
+            scannedQuads,
+            uniqueTriples: stats.uniqueTriples,
+            memberships: stats.membershipCount,
+            projectionRows: pairProjectionRowTotal(stats.pairProjectionRows) + termProjectionRowTotal(stats.termProjectionRows),
+            factsDataVersion,
+            durationMs: Date.now() - start,
+        };
+    }
+    refreshDirtyProjectionTables(start, factsDataVersion, dirty) {
+        const scannedQuads = this.countDirtyProjectionQuads();
+        this.requireDb().transaction(() => {
+            for (const projection of PAIR_PROJECTIONS) {
+                this.refreshDirtyPairProjection(projection);
+            }
+            for (const projection of TERM_PROJECTIONS) {
+                this.refreshDirtyTermProjection(projection);
+            }
+            this.refreshDirtyGraphProjection();
+            this.clearDirtyProjectionTables();
             this.setFactsDataVersion(factsDataVersion);
         })();
         const stats = this.stats();
         return {
+            mode: 'incremental',
             scannedQuads,
             uniqueTriples: stats.uniqueTriples,
             memberships: stats.membershipCount,
             projectionRows: pairProjectionRowTotal(stats.pairProjectionRows) + termProjectionRowTotal(stats.termProjectionRows),
             factsDataVersion,
             durationMs: Date.now() - start,
+            dirtyGraphs: dirty.graphs,
+            dirtyPairs: dirty.pairs,
+            dirtyTerms: dirty.terms,
         };
     }
     factsDataVersion() {
@@ -604,6 +647,23 @@ class Rdf3xIndex {
         membership_count INTEGER NOT NULL
       ) WITHOUT ROWID;
 
+      CREATE TABLE IF NOT EXISTS ${Rdf3xSchema_1.RDF3X_DIRTY_GRAPH_TABLE} (
+        graph_id INTEGER NOT NULL PRIMARY KEY
+      ) WITHOUT ROWID;
+
+      CREATE TABLE IF NOT EXISTS ${Rdf3xSchema_1.RDF3X_DIRTY_PAIR_TABLE} (
+        name TEXT NOT NULL,
+        left_id INTEGER NOT NULL,
+        right_id INTEGER NOT NULL,
+        PRIMARY KEY (name, left_id, right_id)
+      ) WITHOUT ROWID;
+
+      CREATE TABLE IF NOT EXISTS ${Rdf3xSchema_1.RDF3X_DIRTY_TERM_TABLE} (
+        name TEXT NOT NULL,
+        term_id INTEGER NOT NULL,
+        PRIMARY KEY (name, term_id)
+      ) WITHOUT ROWID;
+
       CREATE TABLE IF NOT EXISTS rdf3x_metadata (
         key TEXT PRIMARY KEY,
         value TEXT NOT NULL
@@ -612,8 +672,36 @@ class Rdf3xIndex {
       ${pairProjectionTables}
       ${termProjectionTables}
     `);
+        this.initializeDirtyTrackingTriggers();
         this.setMetadataValue('schema_version', String(RDF3X_INDEX_SCHEMA_VERSION));
     }
+    initializeDirtyTrackingTriggers() {
+        const db = this.requireDb();
+        const factsTable = db.prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = 'rdf_quads'").get();
+        if (!factsTable) {
+            return;
+        }
+        db.exec(`
+      CREATE TRIGGER IF NOT EXISTS rdf3x_dirty_quads_insert
+      AFTER INSERT ON rdf_quads
+      BEGIN
+        ${dirtyProjectionStatements('NEW')}
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS rdf3x_dirty_quads_delete
+      AFTER DELETE ON rdf_quads
+      BEGIN
+        ${dirtyProjectionStatements('OLD')}
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS rdf3x_dirty_quads_update
+      AFTER UPDATE OF graph_id, subject_id, predicate_id, object_id ON rdf_quads
+      BEGIN
+        ${dirtyProjectionStatements('OLD')}
+        ${dirtyProjectionStatements('NEW')}
+      END;
+    `);
+    }
     dropMaterializedFactCopies() {
         (0, Rdf3xSchema_1.dropRdf3xMaterializedFactCopies)(this.requireDb());
     }
@@ -645,6 +733,13 @@ class Rdf3xIndex {
             `DELETE FROM ${GRAPH_PROJECTION_TABLE};`,
         ].join('\n'));
     }
+    clearDirtyProjectionTables() {
+        this.requireDb().exec(`
+      DELETE FROM ${Rdf3xSchema_1.RDF3X_DIRTY_GRAPH_TABLE};
+      DELETE FROM ${Rdf3xSchema_1.RDF3X_DIRTY_PAIR_TABLE};
+      DELETE FROM ${Rdf3xSchema_1.RDF3X_DIRTY_TERM_TABLE};
+    `);
+    }
     prepareSchemaVersion() {
         this.ensureMetadataTable();
         const row = this.requireDb()
@@ -694,6 +789,41 @@ class Rdf3xIndex {
       DO UPDATE SET value = excluded.value
     `).run(key, value);
     }
+    dirtyProjectionStats() {
+        return {
+            graphs: this.rowCount(Rdf3xSchema_1.RDF3X_DIRTY_GRAPH_TABLE),
+            pairs: this.rowCount(Rdf3xSchema_1.RDF3X_DIRTY_PAIR_TABLE),
+            terms: this.rowCount(Rdf3xSchema_1.RDF3X_DIRTY_TERM_TABLE),
+        };
+    }
+    countDirtyProjectionQuads() {
+        return this.requireDb().prepare(`
+      SELECT COUNT(*) AS count
+      FROM ${RDF_FACTS_TABLE} q
+      WHERE EXISTS (
+        SELECT 1
+        FROM ${Rdf3xSchema_1.RDF3X_DIRTY_GRAPH_TABLE} dirty_graph
+        WHERE dirty_graph.graph_id = q.graph_id
+      )
+      OR EXISTS (
+        SELECT 1
+        FROM ${Rdf3xSchema_1.RDF3X_DIRTY_PAIR_TABLE} dirty_pair
+        WHERE (dirty_pair.name = 'SP' AND dirty_pair.left_id = q.subject_id AND dirty_pair.right_id = q.predicate_id)
+           OR (dirty_pair.name = 'SO' AND dirty_pair.left_id = q.subject_id AND dirty_pair.right_id = q.object_id)
+           OR (dirty_pair.name = 'PS' AND dirty_pair.left_id = q.predicate_id AND dirty_pair.right_id = q.subject_id)
+           OR (dirty_pair.name = 'PO' AND dirty_pair.left_id = q.predicate_id AND dirty_pair.right_id = q.object_id)
+           OR (dirty_pair.name = 'OS' AND dirty_pair.left_id = q.object_id AND dirty_pair.right_id = q.subject_id)
+           OR (dirty_pair.name = 'OP' AND dirty_pair.left_id = q.object_id AND dirty_pair.right_id = q.predicate_id)
+      )
+      OR EXISTS (
+        SELECT 1
+        FROM ${Rdf3xSchema_1.RDF3X_DIRTY_TERM_TABLE} dirty_term
+        WHERE (dirty_term.name = 'S' AND dirty_term.term_id = q.subject_id)
+           OR (dirty_term.name = 'P' AND dirty_term.term_id = q.predicate_id)
+           OR (dirty_term.name = 'O' AND dirty_term.term_id = q.object_id)
+      )
+    `).get()?.count ?? 0;
+    }
     rebuildPairProjection(projection) {
         const [left, right] = projection.columns;
         this.requireDb().prepare(`
@@ -734,6 +864,65 @@ class Rdf3xIndex {
        AND member.${right} = triple.${right}
     `).run();
     }
+    refreshDirtyPairProjection(projection) {
+        const [left, right] = projection.columns;
+        const db = this.requireDb();
+        db.prepare(`
+      DELETE FROM ${projection.table}
+      WHERE EXISTS (
+        SELECT 1
+        FROM ${Rdf3xSchema_1.RDF3X_DIRTY_PAIR_TABLE} dirty
+        WHERE dirty.name = ?
+          AND dirty.left_id = ${projection.table}.${left}
+          AND dirty.right_id = ${projection.table}.${right}
+      )
+    `).run(projection.name);
+        db.prepare(`
+      INSERT INTO ${projection.table} (
+        ${left},
+        ${right},
+        triple_count,
+        membership_count,
+        min_${projection.remainder},
+        max_${projection.remainder}
+      )
+      SELECT
+        triple.${left},
+        triple.${right},
+        triple.triple_count,
+        COALESCE(member.membership_count, 0) AS membership_count,
+        triple.min_remainder,
+        triple.max_remainder
+      FROM (
+        SELECT
+          q.${left},
+          q.${right},
+          COUNT(DISTINCT q.${projection.remainder}) AS triple_count,
+          MIN(q.${projection.remainder}) AS min_remainder,
+          MAX(q.${projection.remainder}) AS max_remainder
+        FROM ${RDF_FACTS_TABLE} q
+        JOIN ${Rdf3xSchema_1.RDF3X_DIRTY_PAIR_TABLE} dirty
+          ON dirty.name = ?
+         AND dirty.left_id = q.${left}
+         AND dirty.right_id = q.${right}
+        GROUP BY q.${left}, q.${right}
+      ) triple
+      LEFT JOIN (
+        SELECT
+          q.${left},
+          q.${right},
+          COUNT(*) AS membership_count
+        FROM ${RDF_FACTS_TABLE} q
+        JOIN ${Rdf3xSchema_1.RDF3X_DIRTY_PAIR_TABLE} dirty
+          ON dirty.name = ?
+         AND dirty.left_id = q.${left}
+         AND dirty.right_id = q.${right}
+        GROUP BY q.${left}, q.${right}
+      ) member
+        ON member.${left} = triple.${left}
+       AND member.${right} = triple.${right}
+    `).run(projection.name, projection.name);
+    }
     rebuildTermProjection(projection) {
         this.requireDb().prepare(`
       INSERT INTO ${projection.table} (
@@ -765,6 +954,56 @@ class Rdf3xIndex {
         ON member.${projection.column} = triple.${projection.column}
     `).run();
     }
+    refreshDirtyTermProjection(projection) {
+        const db = this.requireDb();
+        db.prepare(`
+      DELETE FROM ${projection.table}
+      WHERE EXISTS (
+        SELECT 1
+        FROM ${Rdf3xSchema_1.RDF3X_DIRTY_TERM_TABLE} dirty
+        WHERE dirty.name = ?
+          AND dirty.term_id = ${projection.table}.${projection.column}
+      )
+    `).run(projection.name);
+        db.prepare(`
+      INSERT INTO ${projection.table} (
+        ${projection.column},
+        triple_count,
+        membership_count
+      )
+      SELECT
+        triple.${projection.column},
+        triple.triple_count,
+        COALESCE(member.membership_count, 0) AS membership_count
+      FROM (
+        SELECT
+          distinct_triples.${projection.column},
+          COUNT(*) AS triple_count
+        FROM (
+          SELECT DISTINCT
+            q.subject_id,
+            q.predicate_id,
+            q.object_id
+          FROM ${RDF_FACTS_TABLE} q
+          JOIN ${Rdf3xSchema_1.RDF3X_DIRTY_TERM_TABLE} dirty
+            ON dirty.name = ?
+           AND dirty.term_id = q.${projection.column}
+        ) distinct_triples
+        GROUP BY distinct_triples.${projection.column}
+      ) triple
+      LEFT JOIN (
+        SELECT
+          q.${projection.column},
+          COUNT(*) AS membership_count
+        FROM ${RDF_FACTS_TABLE} q
+        JOIN ${Rdf3xSchema_1.RDF3X_DIRTY_TERM_TABLE} dirty
+          ON dirty.name = ?
+         AND dirty.term_id = q.${projection.column}
+        GROUP BY q.${projection.column}
+      ) member
+        ON member.${projection.column} = triple.${projection.column}
+    `).run(projection.name, projection.name);
+    }
     rebuildGraphProjection() {
         this.requireDb().prepare(`
       INSERT INTO ${GRAPH_PROJECTION_TABLE} (
@@ -778,6 +1017,29 @@ class Rdf3xIndex {
       GROUP BY graph_id
     `).run();
     }
+    refreshDirtyGraphProjection() {
+        const db = this.requireDb();
+        db.exec(`
+      DELETE FROM ${GRAPH_PROJECTION_TABLE}
+      WHERE graph_id IN (
+        SELECT graph_id
+        FROM ${Rdf3xSchema_1.RDF3X_DIRTY_GRAPH_TABLE}
+      );
+    `);
+        db.prepare(`
+      INSERT INTO ${GRAPH_PROJECTION_TABLE} (
+        graph_id,
+        membership_count
+      )
+      SELECT
+        q.graph_id,
+        COUNT(*) AS membership_count
+      FROM ${RDF_FACTS_TABLE} q
+      JOIN ${Rdf3xSchema_1.RDF3X_DIRTY_GRAPH_TABLE} dirty
+        ON dirty.graph_id = q.graph_id
+      GROUP BY q.graph_id
+    `).run();
+    }
     compileScanSql(permutation, resolved, options, tupleValues) {
         const conditions = [];
         const params = [];
@@ -979,6 +1241,7 @@ class Rdf3xIndex {
         const indexOnly = this.canUseIndexOnlyJoin(sources, options);
         const queryPlan = [
             `Rdf3xJoinBGP(${patterns.length})`,
+            ...(0, RdfJoinShape_1.rdfSubjectStarJoinPlanMarker)('SubjectStarJoin', patterns),
             `Rdf3xJoinOrder(${orderedSources.map((source) => `?${source.inputIndex}:${source.estimate.indexChoice}`).join('>')})`,
             ...(indexOnly ? ['Rdf3xIndexOnlyJoin'] : []),
         ];
@@ -2504,6 +2767,27 @@ function requiredTerm(termMap, id) {
     }
     return term;
 }
+function dirtyProjectionStatements(prefix) {
+    return `
+    INSERT OR IGNORE INTO ${Rdf3xSchema_1.RDF3X_DIRTY_GRAPH_TABLE} (graph_id)
+    VALUES (${prefix}.graph_id);
+
+    INSERT OR IGNORE INTO ${Rdf3xSchema_1.RDF3X_DIRTY_PAIR_TABLE} (name, left_id, right_id)
+    VALUES
+      ('SP', ${prefix}.subject_id, ${prefix}.predicate_id),
+      ('SO', ${prefix}.subject_id, ${prefix}.object_id),
+      ('PS', ${prefix}.predicate_id, ${prefix}.subject_id),
+      ('PO', ${prefix}.predicate_id, ${prefix}.object_id),
+      ('OS', ${prefix}.object_id, ${prefix}.subject_id),
+      ('OP', ${prefix}.object_id, ${prefix}.predicate_id);
+
+    INSERT OR IGNORE INTO ${Rdf3xSchema_1.RDF3X_DIRTY_TERM_TABLE} (name, term_id)
+    VALUES
+      ('S', ${prefix}.subject_id),
+      ('P', ${prefix}.predicate_id),
+      ('O', ${prefix}.object_id);
+  `;
+}
 function isRdfTerm(value) {
     return value !== null && typeof value === 'object' && 'termType' in value;
 }