@malloy-publisher/server 0.0.239 → 0.0.240

package/dist/server.mjs

@@ -276144,9 +276144,10 @@ class IncrementalLedgerRepository {
   constructor(db) {
     this.db = db;
   }
-  async get(environmentId, packageName, sourceEntityId) {
+  async get(environmentId, connectionName, physicalTableName) {
     const row = await this.db.get(`SELECT * FROM incremental_ledger
-           WHERE environment_id = ? AND package_name = ? AND source_entity_id = ?`, [environmentId, packageName, sourceEntityId]);
+           WHERE environment_id = ? AND connection_name = ?
+             AND physical_table_name = ?`, [environmentId, connectionName, physicalTableName]);
     return row ? mapRow(row) : null;
   }
   async upsert(entry) {
@@ -276158,14 +276159,15 @@ class IncrementalLedgerRepository {
              physical_table_name, connection_name,
              advanced_by_materialization_id, advanced_at, created_at
           ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-          ON CONFLICT (environment_id, package_name, source_entity_id) DO UPDATE SET
+          ON CONFLICT (environment_id, connection_name, physical_table_name)
+          DO UPDATE SET
+             package_name = EXCLUDED.package_name,
+             source_entity_id = EXCLUDED.source_entity_id,
              covered_through_value = EXCLUDED.covered_through_value,
              covered_through_type = EXCLUDED.covered_through_type,
              watermark_dimension = EXCLUDED.watermark_dimension,
              merge_key_dimensions = EXCLUDED.merge_key_dimensions,
              derived_strategy = EXCLUDED.derived_strategy,
-             physical_table_name = EXCLUDED.physical_table_name,
-             connection_name = EXCLUDED.connection_name,
              advanced_by_materialization_id = EXCLUDED.advanced_by_materialization_id,
              advanced_at = EXCLUDED.advanced_at
           RETURNING *`, [
@@ -276185,9 +276187,10 @@ class IncrementalLedgerRepository {
     ]);
     return mapRow(rows[0]);
   }
-  async deleteEntry(environmentId, packageName, sourceEntityId) {
+  async deleteEntry(environmentId, connectionName, physicalTableName) {
     await this.db.run(`DELETE FROM incremental_ledger
-           WHERE environment_id = ? AND package_name = ? AND source_entity_id = ?`, [environmentId, packageName, sourceEntityId]);
+           WHERE environment_id = ? AND connection_name = ?
+             AND physical_table_name = ?`, [environmentId, connectionName, physicalTableName]);
   }
   async deleteByEnvironmentId(environmentId) {
     await this.db.run("DELETE FROM incremental_ledger WHERE environment_id = ?", [environmentId]);
@@ -276695,14 +276698,14 @@ class DuckDBRepository {
   async deleteMaterialization(id) {
     return this.materializationRepo.deleteById(id);
   }
-  async getIncrementalLedgerEntry(environmentId, packageName, sourceEntityId) {
-    return this.incrementalLedgerRepo.get(environmentId, packageName, sourceEntityId);
+  async getIncrementalLedgerEntry(environmentId, connectionName, physicalTableName) {
+    return this.incrementalLedgerRepo.get(environmentId, connectionName, physicalTableName);
   }
   async upsertIncrementalLedgerEntry(entry) {
     return this.incrementalLedgerRepo.upsert(entry);
   }
-  async deleteIncrementalLedgerEntry(environmentId, packageName, sourceEntityId) {
-    return this.incrementalLedgerRepo.deleteEntry(environmentId, packageName, sourceEntityId);
+  async deleteIncrementalLedgerEntry(environmentId, connectionName, physicalTableName) {
+    return this.incrementalLedgerRepo.deleteEntry(environmentId, connectionName, physicalTableName);
   }
 }
 
@@ -276717,6 +276720,7 @@ async function initializeSchema(db, force = false) {
     await dropLegacyProjectSchema(db);
     logger.info("Creating database schema for the first time...");
   }
+  await dropPackageKeyedIncrementalLedger(db);
   await db.run(`
     CREATE TABLE IF NOT EXISTS environments (
       id VARCHAR PRIMARY KEY,
@@ -276800,7 +276804,7 @@ async function initializeSchema(db, force = false) {
       advanced_by_materialization_id VARCHAR,
       advanced_at TIMESTAMP NOT NULL,
       created_at TIMESTAMP NOT NULL,
-      PRIMARY KEY (environment_id, package_name, source_entity_id)
+      PRIMARY KEY (environment_id, connection_name, physical_table_name)
     )
   `);
   await db.run(`
@@ -276819,6 +276823,19 @@ async function initializeSchema(db, force = false) {
   await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
   await db.run("CREATE INDEX IF NOT EXISTS idx_incremental_ledger_environment_package ON incremental_ledger(environment_id, package_name)");
 }
+async function dropPackageKeyedIncrementalLedger(db) {
+  const present = await db.all("SELECT name FROM sqlite_master WHERE type='table' AND name='incremental_ledger'");
+  if (!present || present.length === 0) {
+    return;
+  }
+  const columns = await db.all("PRAGMA table_info('incremental_ledger')");
+  const keyedOnPackage = columns.some((column) => column.name === "package_name" && Boolean(column.pk));
+  if (!keyedOnPackage) {
+    return;
+  }
+  logger.info("Re-keying the incremental ledger onto (environment, connection, table); " + "recorded covered_through boundaries are discarded, so each incremental " + "source rebuilds in full once and then resumes advancing by delta");
+  await db.run("DROP TABLE IF EXISTS incremental_ledger");
+}
 async function createEntityEmbeddingsTable(db) {
   await db.run(`
     CREATE TABLE IF NOT EXISTS entity_embeddings (
@@ -280634,6 +280651,9 @@ function ledgerLineageMismatch(entry, lineage) {
     };
   }
   const changed = (reason) => ({ reasonCode: "lineage_changed", reason });
+  if (entry.sourceEntityId !== lineage.sourceEntityId) {
+    return changed(`the source's content address changed, so the recorded boundary was ` + `measured over different SQL than this refresh would apply a delta to`);
+  }
   if (entry.connectionName !== lineage.connectionName) {
     return changed(`the recorded boundary was advanced on connection ` + `"${entry.connectionName}", not "${lineage.connectionName}"`);
   }
@@ -292740,6 +292760,7 @@ function incrementalLineage(params) {
   return {
     physicalTableName: params.physicalTableName,
     connectionName: params.connectionName,
+    sourceEntityId: params.sourceEntityId,
     watermarkName: watermark.name,
     watermarkType: watermark.malloyType,
     mergeKeys,
@@ -292750,7 +292771,7 @@ async function planSourceRefresh(params) {
   const { context, lineage } = params;
   let ledgerEntry = null;
   try {
-    ledgerEntry = await context.ledger.getIncrementalLedgerEntry(context.environmentId, context.packageName, params.sourceEntityId);
+    ledgerEntry = await context.ledger.getIncrementalLedgerEntry(context.environmentId, lineage.connectionName, lineage.physicalTableName);
   } catch (err) {
     return {
       mode: "seed",
@@ -292786,7 +292807,7 @@ async function advanceLedger(params) {
     await context.ledger.upsertIncrementalLedgerEntry({
       environmentId: context.environmentId,
       packageName: context.packageName,
-      sourceEntityId: params.sourceEntityId,
+      sourceEntityId: lineage.sourceEntityId,
       coveredThroughValue: params.coveredThrough.value,
       coveredThroughType: params.coveredThrough.malloyType,
       watermarkDimension: lineage.watermarkName,
@@ -292799,18 +292820,19 @@ async function advanceLedger(params) {
   } catch (err) {
     logger.warn("Failed to advance the covered_through boundary", {
       packageName: context.packageName,
-      sourceEntityId: params.sourceEntityId,
+      physicalTableName: lineage.physicalTableName,
+      sourceEntityId: lineage.sourceEntityId,
       error: errMessage(err)
     });
   }
 }
-async function resetLedger(context, sourceEntityId) {
+async function resetLedger(context, lineage) {
   try {
-    await context.ledger.deleteIncrementalLedgerEntry(context.environmentId, context.packageName, sourceEntityId);
+    await context.ledger.deleteIncrementalLedgerEntry(context.environmentId, lineage.connectionName, lineage.physicalTableName);
   } catch (err) {
     logger.warn("Failed to clear the covered_through boundary", {
       packageName: context.packageName,
-      sourceEntityId,
+      physicalTableName: lineage.physicalTableName,
       error: errMessage(err)
     });
   }
@@ -292821,7 +292843,7 @@ async function advanceLedgerAfterSeed(params) {
     if (boundary.error) {
       logger.warn("Could not read a covered_through boundary after a full rebuild; " + "the next refresh will rebuild again", {
         packageName: params.context.packageName,
-        sourceEntityId: params.sourceEntityId,
+        physicalTableName: params.lineage.physicalTableName,
         error: boundary.error
       });
     }
@@ -292830,7 +292852,6 @@ async function advanceLedgerAfterSeed(params) {
   await advanceLedger({
     context: params.context,
     lineage: params.lineage,
-    sourceEntityId: params.sourceEntityId,
     coveredThrough: boundary.bound
   });
   return boundary.bound;
@@ -293407,6 +293428,7 @@ class MaterializationService {
           dialect: persistSource.dialectName,
           physicalTableName: logicalName,
           connectionName: persistSource.connectionName,
+          sourceEntityId,
           isStorageBuild: destination !== undefined
         }) !== undefined;
         const prior = priorEntries[sourceEntityId];
@@ -293760,12 +293782,12 @@ class MaterializationService {
       dialect,
       physicalTableName,
       connectionName: persistSource.connectionName,
+      sourceEntityId: contentSourceEntityId,
       isStorageBuild
     }) : undefined;
     const incrementalRefresh = incremental && lineage && contentSourceEntityId ? {
       context: incremental,
-      lineage,
-      ledgerKey: contentSourceEntityId
+      lineage
     } : undefined;
     if (incrementalRefresh) {
       const applied = await this.refreshOneSourceIncrementally({
@@ -293787,7 +293809,7 @@ class MaterializationService {
     const quotedPhysical = quotedPhysicalPath;
     const quotedBareName = quoteIdentifier(bareName, dialect);
     if (incrementalRefresh) {
-      await resetLedger(incrementalRefresh.context, incrementalRefresh.ledgerKey);
+      await resetLedger(incrementalRefresh.context, incrementalRefresh.lineage);
     }
     const startTime = performance.now();
     await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`, runOptions);
@@ -293817,7 +293839,6 @@ class MaterializationService {
     const seededThrough = incrementalRefresh ? await advanceLedgerAfterSeed({
       context: incrementalRefresh.context,
       lineage: incrementalRefresh.lineage,
-      sourceEntityId: incrementalRefresh.ledgerKey,
       quotedTablePath: quotedPhysical,
       dialect,
       runner: (sql) => connection.runSQL(sql, runOptions)
@@ -293842,7 +293863,6 @@ class MaterializationService {
       context,
       lineage,
       persistSource,
-      sourceEntityId: params.ledgerKey,
       quotedTablePath: params.quotedTablePath,
       sourceSQL: params.buildSQL,
       columns: deriveColumns(persistSource).map((c) => String(c.name)),
@@ -293865,7 +293885,6 @@ class MaterializationService {
       await advanceLedger({
         context,
         lineage,
-        sourceEntityId: params.ledgerKey,
         coveredThrough: step.coveredThrough
       });
       const durationMs = Math.round(performance.now() - startTime);

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.239",
+   "version": "0.0.240",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"