@undefineds.co/xpod 0.3.18 → 0.3.22

package/dist/storage/rdf/RdfQuadIndex.d.ts

@@ -15,9 +15,16 @@ export declare class RdfQuadIndex {
     put(quad: Quad, options?: RdfIndexPutOptions): void;
     replaceSource(quads: Quad[], source: RdfSourceInput): void;
     deleteSource(source: string): number;
+    private deleteSourceInternal;
     multiPut(quads: Quad[], options?: RdfIndexPutOptions): void;
     private insertQuads;
     delete(pattern: QuintPattern): number;
+    applyDelta(deletes: QuintPattern[], inserts: Quad[], options?: RdfIndexPutOptions): {
+        deletedRows: number;
+        insertedRows: number;
+    };
+    private deleteInternal;
+    dataVersion(): number;
     scan(pattern: QuintPattern, options?: RdfQuadScanOptions): RdfQuadIndexScanResult;
     scanWithTupleConstraints(pattern: QuintPattern, tupleSource: RdfQuadTupleConstraintSource, options?: RdfQuadScanOptions): RdfQuadIndexScanResult;
     joinPatterns(patterns: RdfQuadJoinPattern[], options?: RdfQuadJoinOptions): RdfQuadJoinScanResult;
@@ -30,7 +37,6 @@ export declare class RdfQuadIndex {
     private scanInternal;
     private compileJoinPatterns;
     private buildJoinOrderClause;
-    private joinRowKeyExpression;
     private buildJoinAggregateColumn;
     private buildGroupCountHavingClause;
     private buildGroupAggregateHavingClause;
@@ -57,6 +63,11 @@ export declare class RdfQuadIndex {
     private estimateSpaceObjectsFromSchema;
     private estimatePageSize;
     private initializeSchema;
+    private prepareSchemaVersion;
+    private ensureMetadataTable;
+    private dropIndexSchema;
+    private bumpDataVersion;
+    private setMetadataValue;
     private upsertSource;
     private buildWhereClause;
     private matchToCondition;

package/dist/storage/rdf/RdfQuadIndex.js

@@ -7,10 +7,12 @@ const n3_1 = require("n3");
 const SqliteRuntime_1 = require("../SqliteRuntime");
 const types_1 = require("../quint/types");
 const RdfTermDictionary_1 = require("./RdfTermDictionary");
+const Rdf3xSchema_1 = require("./Rdf3xSchema");
 const RdfTermSemantics_1 = require("./RdfTermSemantics");
 const XSD_INTEGER = 'http://www.w3.org/2001/XMLSchema#integer';
 const XSD_DECIMAL = 'http://www.w3.org/2001/XMLSchema#decimal';
 const XSD_STRING = 'http://www.w3.org/2001/XMLSchema#string';
+const RDF_QUAD_INDEX_SCHEMA_VERSION = 1;
 const TERM_COLUMN = {
     graph: 'graph_id',
     subject: 'subject_id',
@@ -38,6 +40,7 @@ class RdfQuadIndex {
             }
         }
         this.db = this.sqliteRuntime.openDatabase(this.options.path);
+        this.prepareSchemaVersion();
         this.dictionary = new RdfTermDictionary_1.RdfTermDictionary(this.db);
         this.dictionary.initialize();
         this.initializeSchema();
@@ -51,6 +54,7 @@ class RdfQuadIndex {
         const db = this.requireDb();
         db.exec('DELETE FROM rdf_quads; DELETE FROM rdf_sources; DELETE FROM rdf_terms;');
         this.cardinalityCache.clear();
+        this.bumpDataVersion();
     }
     put(quad, options) {
         this.multiPut([quad], options);
@@ -58,7 +62,7 @@ class RdfQuadIndex {
     replaceSource(quads, source) {
         const db = this.requireDb();
         db.transaction(() => {
-            this.deleteSource(source.source);
+            this.deleteSourceInternal(source.source);
             if (quads.length > 0) {
                 this.insertQuads(quads, { source });
             }
@@ -67,8 +71,17 @@ class RdfQuadIndex {
             }
         })();
         this.cardinalityCache.clear();
+        this.bumpDataVersion();
     }
     deleteSource(source) {
+        const result = this.deleteSourceInternal(source);
+        if (result > 0) {
+            this.cardinalityCache.clear();
+            this.bumpDataVersion();
+        }
+        return result;
+    }
+    deleteSourceInternal(source) {
         const db = this.requireDb();
         const row = db
             .prepare('SELECT id FROM rdf_sources WHERE source = ?')
@@ -78,9 +91,6 @@ class RdfQuadIndex {
         }
         const result = db.prepare('DELETE FROM rdf_quads WHERE source_file_id = ?').run(row.id);
         db.prepare('DELETE FROM rdf_sources WHERE id = ?').run(row.id);
-        if (result.changes > 0) {
-            this.cardinalityCache.clear();
-        }
         return result.changes;
     }
     multiPut(quads, options) {
@@ -92,6 +102,7 @@ class RdfQuadIndex {
             this.insertQuads(quads, options);
         })();
         this.cardinalityCache.clear();
+        this.bumpDataVersion();
     }
     insertQuads(quads, options) {
         const db = this.requireDb();
@@ -117,21 +128,53 @@ class RdfQuadIndex {
         }
     }
     delete(pattern) {
+        const changes = this.deleteInternal(pattern);
+        if (changes > 0) {
+            this.cardinalityCache.clear();
+            this.bumpDataVersion();
+        }
+        return changes;
+    }
+    applyDelta(deletes, inserts, options) {
+        if (deletes.length === 0 && inserts.length === 0) {
+            return { deletedRows: 0, insertedRows: 0 };
+        }
+        const db = this.requireDb();
+        let deletedRows = 0;
+        db.transaction(() => {
+            for (const pattern of deletes) {
+                deletedRows += this.deleteInternal(pattern);
+            }
+            if (inserts.length > 0) {
+                this.insertQuads(inserts, options);
+            }
+        })();
+        if (deletedRows > 0 || inserts.length > 0) {
+            this.cardinalityCache.clear();
+            this.bumpDataVersion();
+        }
+        return {
+            deletedRows,
+            insertedRows: inserts.length,
+        };
+    }
+    deleteInternal(pattern) {
         const db = this.requireDb();
         const { joins, whereClause, params } = this.buildWhereClause(pattern, false);
         if (!whereClause) {
             const result = db.prepare('DELETE FROM rdf_quads').run();
-            this.cardinalityCache.clear();
             return result.changes;
         }
         const sql = joins
             ? `DELETE FROM rdf_quads WHERE rowid IN (SELECT rdf_quads.rowid FROM rdf_quads${joins}${whereClause})`
             : `DELETE FROM rdf_quads${whereClause}`;
-        const changes = db.prepare(sql).run(...params).changes;
-        if (changes > 0) {
-            this.cardinalityCache.clear();
-        }
-        return changes;
+        return db.prepare(sql).run(...params).changes;
+    }
+    dataVersion() {
+        const row = this.requireDb()
+            .prepare("SELECT value FROM rdf_index_metadata WHERE key = 'data_version'")
+            .get();
+        return Number(row?.value ?? 0) || 0;
     }
     scan(pattern, options) {
         return this.scanInternal(pattern, options);
@@ -191,7 +234,7 @@ class RdfQuadIndex {
         const projection = options.aggregates.map((aggregate, index) => {
             const alias = `a${index}`;
             aggregateAliases.set(aggregate.as, alias);
-            return this.buildJoinAggregateColumn(aggregate, alias, compiled.variableColumns, aggregateTypes, numericJoins, numericJoinSql, 'RDF BGP', this.joinRowKeyExpression(patterns));
+            return this.buildJoinAggregateColumn(aggregate, alias, compiled.variableColumns, aggregateTypes, numericJoins, numericJoinSql, 'RDF BGP');
         }).join(', ');
         const aggregateJoins = numericJoinSql.join('');
         const sql = `SELECT ${projection} FROM ${compiled.from}${compiled.joins}${aggregateJoins}${compiled.whereClause}`;
@@ -246,7 +289,7 @@ class RdfQuadIndex {
             const alias = `a${index}`;
             aggregateAliases.set(aggregate.as, alias);
             aggregateSqlAliases.set(aggregate.as, alias);
-            return this.buildJoinAggregateColumn(aggregate, alias, compiled.variableColumns, aggregateTypes, numericJoins, numericJoinSql, 'RDF BGP group aggregate', '__row_key');
+            return this.buildJoinAggregateColumn(aggregate, alias, compiled.variableColumns, aggregateTypes, numericJoins, numericJoinSql, 'RDF BGP group aggregate');
         });
         const projection = [
             ...options.groupBy.map((variableName) => {
@@ -260,15 +303,12 @@ class RdfQuadIndex {
             ...aggregateColumns,
         ].join(', ');
         const groupBy = groupColumns.join(', ');
-        const rowKeyExpression = this.joinRowKeyExpression(patterns);
         const aggregateJoins = numericJoinSql.join('');
         const havingClause = this.buildGroupAggregateHavingClause(options.having, aggregateSqlAliases);
         const orderScope = this.buildGroupAggregateOrderScope(options, compiled.variableColumns, aggregateSqlAliases);
         const fromSql = `${compiled.from}${compiled.joins}${aggregateJoins}${compiled.whereClause}`;
         const sourceFromSql = `${compiled.from}${compiled.joins}${aggregateJoins}${orderScope.joins}${compiled.whereClause}`;
-        const sourceSql = aggregateColumns.some((entry) => entry.includes('__row_key'))
-            ? `SELECT ${projection.replace(/__row_key/g, rowKeyExpression)} FROM ${sourceFromSql} GROUP BY ${groupBy}${havingClause.sql}`
-            : `SELECT ${projection} FROM ${sourceFromSql} GROUP BY ${groupBy}${havingClause.sql}`;
+        const sourceSql = `SELECT ${projection} FROM ${sourceFromSql} GROUP BY ${groupBy}${havingClause.sql}`;
         const orderClause = orderScope.orderBy;
         let sql = `${sourceSql}${orderClause}`;
         const params = [...compiled.params, ...havingClause.params];
@@ -476,13 +516,10 @@ class RdfQuadIndex {
             orderBy: ` ORDER BY ${joins.map((entry) => entry.order).join(', ')}`,
         };
     }
-    joinRowKeyExpression(patterns) {
-        return patterns.map((_, index) => `q${index}.rowid`).join(` || ':' || `);
-    }
-    buildJoinAggregateColumn(aggregate, alias, variableColumns, aggregateTypes, numericJoins, numericJoinSql, errorPrefix, rowKeyExpression) {
+    buildJoinAggregateColumn(aggregate, alias, variableColumns, aggregateTypes, numericJoins, numericJoinSql, errorPrefix) {
         if (aggregate.type === 'count' && !aggregate.variable) {
             aggregateTypes.set(aggregate.as, 'integer');
-            return `${aggregate.distinct ? `COUNT(DISTINCT ${rowKeyExpression})` : 'COUNT(*)'} AS ${alias}`;
+            return `${aggregate.distinct ? `COUNT(DISTINCT ${joinSolutionMappingKeyExpression(variableColumns, aggregate.distinctVariables, errorPrefix)})` : 'COUNT(*)'} AS ${alias}`;
         }
         if (!aggregate.variable) {
             throw new Error(`${errorPrefix} ${aggregate.type} aggregate requires a bound variable`);
@@ -977,8 +1014,83 @@ class RdfQuadIndex {
       CREATE INDEX IF NOT EXISTS rdf_quads_gspo ON rdf_quads(graph_id, subject_id, predicate_id, object_id);
       CREATE INDEX IF NOT EXISTS rdf_quads_gpos ON rdf_quads(graph_id, predicate_id, object_id, subject_id);
       CREATE INDEX IF NOT EXISTS rdf_quads_source ON rdf_quads(source_file_id);
+
+      CREATE TABLE IF NOT EXISTS rdf_index_metadata (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+    `);
+        this.requireDb().prepare(`
+      INSERT OR IGNORE INTO rdf_index_metadata (key, value)
+      SELECT 'data_version', '1'
+      WHERE EXISTS (SELECT 1 FROM rdf_quads LIMIT 1)
+    `).run();
+    }
+    prepareSchemaVersion() {
+        this.ensureMetadataTable();
+        const db = this.requireDb();
+        const row = db
+            .prepare("SELECT value FROM rdf_index_metadata WHERE key = 'schema_version'")
+            .get();
+        if (row && row.value !== String(RDF_QUAD_INDEX_SCHEMA_VERSION)) {
+            this.dropIndexSchema();
+            this.ensureMetadataTable();
+        }
+        this.setMetadataValue('schema_version', String(RDF_QUAD_INDEX_SCHEMA_VERSION));
+    }
+    ensureMetadataTable() {
+        this.requireDb().exec(`
+      CREATE TABLE IF NOT EXISTS rdf_index_metadata (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
     `);
     }
+    dropIndexSchema() {
+        const db = this.requireDb();
+        const foreignKeys = db.prepare('PRAGMA foreign_keys').get()?.foreign_keys ?? 0;
+        db.exec('PRAGMA foreign_keys = OFF;');
+        try {
+            db.exec(`
+        DROP INDEX IF EXISTS rdf_quads_spog;
+        DROP INDEX IF EXISTS rdf_quads_sopg;
+        DROP INDEX IF EXISTS rdf_quads_psog;
+        DROP INDEX IF EXISTS rdf_quads_posg;
+        DROP INDEX IF EXISTS rdf_quads_ospg;
+        DROP INDEX IF EXISTS rdf_quads_opsg;
+        DROP INDEX IF EXISTS rdf_quads_gspo;
+        DROP INDEX IF EXISTS rdf_quads_gpos;
+        DROP INDEX IF EXISTS rdf_quads_source;
+        DROP TABLE IF EXISTS rdf_quads;
+        DROP TABLE IF EXISTS rdf_sources;
+        DROP TABLE IF EXISTS rdf_terms;
+        DROP TABLE IF EXISTS rdf_index_metadata;
+      `);
+            (0, Rdf3xSchema_1.dropRdf3xDerivedSchemaObjects)(db);
+        }
+        finally {
+            if (foreignKeys) {
+                db.exec('PRAGMA foreign_keys = ON;');
+            }
+        }
+        this.cardinalityCache.clear();
+    }
+    bumpDataVersion() {
+        this.requireDb().prepare(`
+      INSERT INTO rdf_index_metadata (key, value)
+      VALUES ('data_version', '1')
+      ON CONFLICT (key)
+      DO UPDATE SET value = CAST(value AS INTEGER) + 1
+    `).run();
+    }
+    setMetadataValue(key, value) {
+        this.requireDb().prepare(`
+      INSERT INTO rdf_index_metadata (key, value)
+      VALUES (?, ?)
+      ON CONFLICT (key)
+      DO UPDATE SET value = excluded.value
+    `).run(key, value);
+    }
     upsertSource(source) {
         const db = this.requireDb();
         db.prepare(`
@@ -1443,9 +1555,11 @@ class RdfQuadIndex {
         return {
             join: ` JOIN rdf_terms ${alias} ON ${alias}.id = ${this.scopedQuadColumn(column, scope)}`,
             sql: `${alias}.kind IN (${kind.map(() => '?').join(', ')})
+        AND ${alias}.value_head >= ?
+        AND ${alias}.value_head < ?
         AND ${alias}.value >= ?
         AND ${alias}.value < ?`,
-            params: [...kind, prefix, `${prefix}\uffff`],
+            params: [...kind, (0, RdfTermDictionary_1.rdfTermValueHead)(prefix), `${(0, RdfTermDictionary_1.rdfTermValueHead)(prefix)}\uffff`, prefix, `${prefix}\uffff`],
         };
     }
     textSearchConditionJoin(kind, column, operator, value, scope) {
@@ -1837,4 +1951,20 @@ function uniqueNumbers(values) {
 function uniquePatternKeys(values) {
     return TERM_KEYS.filter((key) => values.includes(key));
 }
+function uniqueVariableNames(values) {
+    return [...new Set(values)];
+}
+function joinSolutionMappingKeyExpression(variableColumns, variables, errorPrefix) {
+    const variableNames = uniqueVariableNames(variables ?? [...variableColumns.keys()]);
+    if (variableNames.length === 0) {
+        return '1';
+    }
+    return variableNames.map((variableName) => {
+        const column = variableColumns.get(variableName);
+        if (!column) {
+            throw new Error(`${errorPrefix} COUNT(DISTINCT *) cannot read unbound variable: ${variableName}`);
+        }
+        return column;
+    }).join(` || ':' || `);
+}
 //# sourceMappingURL=RdfQuadIndex.js.map