@malloy-publisher/server 0.0.239 → 0.0.241
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +153 -46
- package/dist/app/assets/{EnvironmentPage-2oMC3DUO.js → EnvironmentPage-zu0NeDu-.js} +1 -1
- package/dist/app/assets/HomePage-kuBM3jiQ.js +1 -0
- package/dist/app/assets/{LightMode-BNaG4oAx.js → LightMode-DGBAdVPb.js} +1 -1
- package/dist/app/assets/{MainPage-Dje9yqIp.js → MainPage-CCVx9yPs.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-D4jCpScu.js → MaterializationsPage-M-bWIx8L.js} +1 -1
- package/dist/app/assets/ModelPage-D-L2pqIy.js +1 -0
- package/dist/app/assets/{PackagePage-DlO2UBEI.js → PackagePage-BZglxDlj.js} +1 -1
- package/dist/app/assets/RouteError-DGoCQGam.js +1 -0
- package/dist/app/assets/{ThemeEditorPage-DZzT5zt4.js → ThemeEditorPage-e8MLf7SV.js} +1 -1
- package/dist/app/assets/{WorkbookPage-CfDz8XAp.js → WorkbookPage-C8ufpN5i.js} +1 -1
- package/dist/app/assets/{core-BGHaIj_q.es-BeNX3GvK.js → core-KFsKleo0.es-D6cYhbct.js} +1 -1
- package/dist/app/assets/{index-ByceOJbS.js → index-B-jw9EkA.js} +52 -52
- package/dist/app/assets/{index-BfV8vaZu.js → index-CDNkCta4.js} +1 -1
- package/dist/app/assets/{index-BeZphO3v.js → index-CLwzlAx6.js} +1 -1
- package/dist/app/assets/{index-DXjc2oZx.js → index-Dnef41hg.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/runtime/publisher.js +1 -1
- package/dist/server.mjs +210 -53
- package/package.json +1 -1
- package/dist/app/assets/HomePage-DmgHdpH6.js +0 -1
- package/dist/app/assets/ModelPage-oI3O6TwY.js +0 -1
- package/dist/app/assets/RouteError-B5MgkwXx.js +0 -1
package/dist/server.mjs
CHANGED
|
@@ -276144,9 +276144,10 @@ class IncrementalLedgerRepository {
|
|
|
276144
276144
|
constructor(db) {
|
|
276145
276145
|
this.db = db;
|
|
276146
276146
|
}
|
|
276147
|
-
async get(environmentId,
|
|
276147
|
+
async get(environmentId, connectionName, physicalTableName) {
|
|
276148
276148
|
const row = await this.db.get(`SELECT * FROM incremental_ledger
|
|
276149
|
-
WHERE environment_id = ? AND
|
|
276149
|
+
WHERE environment_id = ? AND connection_name = ?
|
|
276150
|
+
AND physical_table_name = ?`, [environmentId, connectionName, physicalTableName]);
|
|
276150
276151
|
return row ? mapRow(row) : null;
|
|
276151
276152
|
}
|
|
276152
276153
|
async upsert(entry) {
|
|
@@ -276158,14 +276159,15 @@ class IncrementalLedgerRepository {
|
|
|
276158
276159
|
physical_table_name, connection_name,
|
|
276159
276160
|
advanced_by_materialization_id, advanced_at, created_at
|
|
276160
276161
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
276161
|
-
ON CONFLICT (environment_id,
|
|
276162
|
+
ON CONFLICT (environment_id, connection_name, physical_table_name)
|
|
276163
|
+
DO UPDATE SET
|
|
276164
|
+
package_name = EXCLUDED.package_name,
|
|
276165
|
+
source_entity_id = EXCLUDED.source_entity_id,
|
|
276162
276166
|
covered_through_value = EXCLUDED.covered_through_value,
|
|
276163
276167
|
covered_through_type = EXCLUDED.covered_through_type,
|
|
276164
276168
|
watermark_dimension = EXCLUDED.watermark_dimension,
|
|
276165
276169
|
merge_key_dimensions = EXCLUDED.merge_key_dimensions,
|
|
276166
276170
|
derived_strategy = EXCLUDED.derived_strategy,
|
|
276167
|
-
physical_table_name = EXCLUDED.physical_table_name,
|
|
276168
|
-
connection_name = EXCLUDED.connection_name,
|
|
276169
276171
|
advanced_by_materialization_id = EXCLUDED.advanced_by_materialization_id,
|
|
276170
276172
|
advanced_at = EXCLUDED.advanced_at
|
|
276171
276173
|
RETURNING *`, [
|
|
@@ -276185,9 +276187,10 @@ class IncrementalLedgerRepository {
|
|
|
276185
276187
|
]);
|
|
276186
276188
|
return mapRow(rows[0]);
|
|
276187
276189
|
}
|
|
276188
|
-
async deleteEntry(environmentId,
|
|
276190
|
+
async deleteEntry(environmentId, connectionName, physicalTableName) {
|
|
276189
276191
|
await this.db.run(`DELETE FROM incremental_ledger
|
|
276190
|
-
WHERE environment_id = ? AND
|
|
276192
|
+
WHERE environment_id = ? AND connection_name = ?
|
|
276193
|
+
AND physical_table_name = ?`, [environmentId, connectionName, physicalTableName]);
|
|
276191
276194
|
}
|
|
276192
276195
|
async deleteByEnvironmentId(environmentId) {
|
|
276193
276196
|
await this.db.run("DELETE FROM incremental_ledger WHERE environment_id = ?", [environmentId]);
|
|
@@ -276695,14 +276698,14 @@ class DuckDBRepository {
|
|
|
276695
276698
|
async deleteMaterialization(id) {
|
|
276696
276699
|
return this.materializationRepo.deleteById(id);
|
|
276697
276700
|
}
|
|
276698
|
-
async getIncrementalLedgerEntry(environmentId,
|
|
276699
|
-
return this.incrementalLedgerRepo.get(environmentId,
|
|
276701
|
+
async getIncrementalLedgerEntry(environmentId, connectionName, physicalTableName) {
|
|
276702
|
+
return this.incrementalLedgerRepo.get(environmentId, connectionName, physicalTableName);
|
|
276700
276703
|
}
|
|
276701
276704
|
async upsertIncrementalLedgerEntry(entry) {
|
|
276702
276705
|
return this.incrementalLedgerRepo.upsert(entry);
|
|
276703
276706
|
}
|
|
276704
|
-
async deleteIncrementalLedgerEntry(environmentId,
|
|
276705
|
-
return this.incrementalLedgerRepo.deleteEntry(environmentId,
|
|
276707
|
+
async deleteIncrementalLedgerEntry(environmentId, connectionName, physicalTableName) {
|
|
276708
|
+
return this.incrementalLedgerRepo.deleteEntry(environmentId, connectionName, physicalTableName);
|
|
276706
276709
|
}
|
|
276707
276710
|
}
|
|
276708
276711
|
|
|
@@ -276717,6 +276720,7 @@ async function initializeSchema(db, force = false) {
|
|
|
276717
276720
|
await dropLegacyProjectSchema(db);
|
|
276718
276721
|
logger.info("Creating database schema for the first time...");
|
|
276719
276722
|
}
|
|
276723
|
+
await dropPackageKeyedIncrementalLedger(db);
|
|
276720
276724
|
await db.run(`
|
|
276721
276725
|
CREATE TABLE IF NOT EXISTS environments (
|
|
276722
276726
|
id VARCHAR PRIMARY KEY,
|
|
@@ -276800,7 +276804,7 @@ async function initializeSchema(db, force = false) {
|
|
|
276800
276804
|
advanced_by_materialization_id VARCHAR,
|
|
276801
276805
|
advanced_at TIMESTAMP NOT NULL,
|
|
276802
276806
|
created_at TIMESTAMP NOT NULL,
|
|
276803
|
-
PRIMARY KEY (environment_id,
|
|
276807
|
+
PRIMARY KEY (environment_id, connection_name, physical_table_name)
|
|
276804
276808
|
)
|
|
276805
276809
|
`);
|
|
276806
276810
|
await db.run(`
|
|
@@ -276819,6 +276823,19 @@ async function initializeSchema(db, force = false) {
|
|
|
276819
276823
|
await db.run("CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)");
|
|
276820
276824
|
await db.run("CREATE INDEX IF NOT EXISTS idx_incremental_ledger_environment_package ON incremental_ledger(environment_id, package_name)");
|
|
276821
276825
|
}
|
|
276826
|
+
async function dropPackageKeyedIncrementalLedger(db) {
|
|
276827
|
+
const present = await db.all("SELECT name FROM sqlite_master WHERE type='table' AND name='incremental_ledger'");
|
|
276828
|
+
if (!present || present.length === 0) {
|
|
276829
|
+
return;
|
|
276830
|
+
}
|
|
276831
|
+
const columns = await db.all("PRAGMA table_info('incremental_ledger')");
|
|
276832
|
+
const keyedOnPackage = columns.some((column) => column.name === "package_name" && Boolean(column.pk));
|
|
276833
|
+
if (!keyedOnPackage) {
|
|
276834
|
+
return;
|
|
276835
|
+
}
|
|
276836
|
+
logger.info("Re-keying the incremental ledger onto (environment, connection, table); " + "recorded covered_through boundaries are discarded, so each incremental " + "source rebuilds in full once and then resumes advancing by delta");
|
|
276837
|
+
await db.run("DROP TABLE IF EXISTS incremental_ledger");
|
|
276838
|
+
}
|
|
276822
276839
|
async function createEntityEmbeddingsTable(db) {
|
|
276823
276840
|
await db.run(`
|
|
276824
276841
|
CREATE TABLE IF NOT EXISTS entity_embeddings (
|
|
@@ -280634,6 +280651,9 @@ function ledgerLineageMismatch(entry, lineage) {
|
|
|
280634
280651
|
};
|
|
280635
280652
|
}
|
|
280636
280653
|
const changed = (reason) => ({ reasonCode: "lineage_changed", reason });
|
|
280654
|
+
if (entry.sourceEntityId !== lineage.sourceEntityId) {
|
|
280655
|
+
return changed(`the source's content address is not the one the recorded boundary was ` + `measured under, so that boundary describes different SQL than this ` + `refresh would apply a delta to — either the model was edited, or ` + `this refresh is instructed through a different version of the ` + `package than the one that advanced the boundary`);
|
|
280656
|
+
}
|
|
280637
280657
|
if (entry.connectionName !== lineage.connectionName) {
|
|
280638
280658
|
return changed(`the recorded boundary was advanced on connection ` + `"${entry.connectionName}", not "${lineage.connectionName}"`);
|
|
280639
280659
|
}
|
|
@@ -284539,6 +284559,9 @@ class MaterializationController {
|
|
|
284539
284559
|
if (parsed.strictUpstreams !== undefined) {
|
|
284540
284560
|
result.strictUpstreams = parsed.strictUpstreams;
|
|
284541
284561
|
}
|
|
284562
|
+
if (parsed.ledger !== undefined) {
|
|
284563
|
+
result.ledger = parsed.ledger;
|
|
284564
|
+
}
|
|
284542
284565
|
}
|
|
284543
284566
|
if (body.forceRefresh !== undefined) {
|
|
284544
284567
|
if (typeof body.forceRefresh !== "boolean") {
|
|
@@ -284608,6 +284631,12 @@ class MaterializationController {
|
|
|
284608
284631
|
}
|
|
284609
284632
|
result.strictUpstreams = obj.strictUpstreams;
|
|
284610
284633
|
}
|
|
284634
|
+
if (obj.ledger !== undefined && obj.ledger !== null) {
|
|
284635
|
+
if (!Array.isArray(obj.ledger)) {
|
|
284636
|
+
throw new BadRequestError("buildInstructions.ledger must be an array");
|
|
284637
|
+
}
|
|
284638
|
+
result.ledger = obj.ledger.map((entry) => this.validateLedgerEntry(entry));
|
|
284639
|
+
}
|
|
284611
284640
|
return result;
|
|
284612
284641
|
}
|
|
284613
284642
|
validateManifestReference(raw) {
|
|
@@ -284658,6 +284687,40 @@ class MaterializationController {
|
|
|
284658
284687
|
...typeof instruction.reseed === "boolean" ? { reseed: instruction.reseed } : {}
|
|
284659
284688
|
};
|
|
284660
284689
|
}
|
|
284690
|
+
validateLedgerEntry(raw) {
|
|
284691
|
+
if (typeof raw !== "object" || raw === null) {
|
|
284692
|
+
throw new BadRequestError("Each buildInstructions.ledger entry must be an object");
|
|
284693
|
+
}
|
|
284694
|
+
const entry = raw;
|
|
284695
|
+
for (const field of [
|
|
284696
|
+
"connectionName",
|
|
284697
|
+
"physicalTableName",
|
|
284698
|
+
"coveredThrough",
|
|
284699
|
+
"coveredThroughType",
|
|
284700
|
+
"watermark",
|
|
284701
|
+
"sourceEntityId"
|
|
284702
|
+
]) {
|
|
284703
|
+
if (typeof entry[field] !== "string" || entry[field] === "") {
|
|
284704
|
+
throw new BadRequestError(`Ledger entry '${field}' must be a non-empty string`);
|
|
284705
|
+
}
|
|
284706
|
+
}
|
|
284707
|
+
if (entry.strategy !== "merge" && entry.strategy !== "range_replace") {
|
|
284708
|
+
throw new BadRequestError("Ledger entry 'strategy' must be 'merge' or 'range_replace'");
|
|
284709
|
+
}
|
|
284710
|
+
if (entry.mergeKeys !== undefined && (!Array.isArray(entry.mergeKeys) || entry.mergeKeys.some((k) => typeof k !== "string"))) {
|
|
284711
|
+
throw new BadRequestError("Ledger entry 'mergeKeys' must be an array of strings");
|
|
284712
|
+
}
|
|
284713
|
+
return {
|
|
284714
|
+
connectionName: entry.connectionName,
|
|
284715
|
+
physicalTableName: entry.physicalTableName,
|
|
284716
|
+
coveredThrough: entry.coveredThrough,
|
|
284717
|
+
coveredThroughType: entry.coveredThroughType,
|
|
284718
|
+
watermark: entry.watermark,
|
|
284719
|
+
strategy: entry.strategy,
|
|
284720
|
+
sourceEntityId: entry.sourceEntityId,
|
|
284721
|
+
...Array.isArray(entry.mergeKeys) ? { mergeKeys: entry.mergeKeys } : {}
|
|
284722
|
+
};
|
|
284723
|
+
}
|
|
284661
284724
|
async stopMaterialization(environmentName, packageName, materializationId) {
|
|
284662
284725
|
return this.materializationService.stopMaterialization(environmentName, packageName, materializationId);
|
|
284663
284726
|
}
|
|
@@ -292722,6 +292785,25 @@ import { Manifest } from "@malloydata/malloy";
|
|
|
292722
292785
|
|
|
292723
292786
|
// src/service/incremental_build.ts
|
|
292724
292787
|
init_logger();
|
|
292788
|
+
function ledgerTableKey(connectionName, physicalTableName) {
|
|
292789
|
+
return `${connectionName}\x00${physicalTableName}`;
|
|
292790
|
+
}
|
|
292791
|
+
function indexCallerLedger(entries) {
|
|
292792
|
+
const index = new Map;
|
|
292793
|
+
for (const entry of entries) {
|
|
292794
|
+
index.set(ledgerTableKey(entry.connectionName, entry.physicalTableName), {
|
|
292795
|
+
sourceEntityId: entry.sourceEntityId,
|
|
292796
|
+
coveredThroughValue: entry.coveredThrough,
|
|
292797
|
+
coveredThroughType: entry.coveredThroughType,
|
|
292798
|
+
watermarkDimension: entry.watermark,
|
|
292799
|
+
mergeKeyDimensions: entry.mergeKeys ?? [],
|
|
292800
|
+
derivedStrategy: entry.strategy,
|
|
292801
|
+
physicalTableName: entry.physicalTableName,
|
|
292802
|
+
connectionName: entry.connectionName
|
|
292803
|
+
});
|
|
292804
|
+
}
|
|
292805
|
+
return index;
|
|
292806
|
+
}
|
|
292725
292807
|
function incrementalLineage(params) {
|
|
292726
292808
|
const d = params.declaration;
|
|
292727
292809
|
if (!d?.incremental)
|
|
@@ -292740,6 +292822,7 @@ function incrementalLineage(params) {
|
|
|
292740
292822
|
return {
|
|
292741
292823
|
physicalTableName: params.physicalTableName,
|
|
292742
292824
|
connectionName: params.connectionName,
|
|
292825
|
+
sourceEntityId: params.sourceEntityId,
|
|
292743
292826
|
watermarkName: watermark.name,
|
|
292744
292827
|
watermarkType: watermark.malloyType,
|
|
292745
292828
|
mergeKeys,
|
|
@@ -292748,25 +292831,32 @@ function incrementalLineage(params) {
|
|
|
292748
292831
|
}
|
|
292749
292832
|
async function planSourceRefresh(params) {
|
|
292750
292833
|
const { context, lineage } = params;
|
|
292834
|
+
const forceRefresh = context.forceRefresh || params.reseed === true;
|
|
292835
|
+
const dialect = params.persistSource.dialectName;
|
|
292751
292836
|
let ledgerEntry = null;
|
|
292752
|
-
|
|
292753
|
-
ledgerEntry =
|
|
292754
|
-
}
|
|
292755
|
-
|
|
292756
|
-
|
|
292757
|
-
|
|
292758
|
-
|
|
292759
|
-
|
|
292837
|
+
if (context.callerLedger) {
|
|
292838
|
+
ledgerEntry = context.callerLedger.get(ledgerTableKey(lineage.connectionName, lineage.physicalTableName)) ?? null;
|
|
292839
|
+
} else {
|
|
292840
|
+
try {
|
|
292841
|
+
ledgerEntry = await context.ledger.getIncrementalLedgerEntry(context.environmentId, lineage.connectionName, lineage.physicalTableName);
|
|
292842
|
+
} catch (err) {
|
|
292843
|
+
return {
|
|
292844
|
+
mode: "seed",
|
|
292845
|
+
reasonCode: "ledger_unreadable",
|
|
292846
|
+
reason: `the covered_through ledger could not be read (${errMessage(err)})`
|
|
292847
|
+
};
|
|
292848
|
+
}
|
|
292760
292849
|
}
|
|
292761
|
-
const postgresVersionNum = lineage.strategy === "merge" &&
|
|
292850
|
+
const postgresVersionNum = lineage.strategy === "merge" && dialect === "postgres" && ledgerEntry !== null ? await probePostgresVersion(params.runner) : undefined;
|
|
292851
|
+
let step;
|
|
292762
292852
|
try {
|
|
292763
|
-
|
|
292853
|
+
step = await planIncrementalStep({
|
|
292764
292854
|
runner: params.runner,
|
|
292765
|
-
dialect
|
|
292855
|
+
dialect,
|
|
292766
292856
|
quotedTablePath: params.quotedTablePath,
|
|
292767
292857
|
lineage,
|
|
292768
292858
|
ledgerEntry,
|
|
292769
|
-
forceRefresh
|
|
292859
|
+
forceRefresh,
|
|
292770
292860
|
now: context.now,
|
|
292771
292861
|
sourceSQL: params.sourceSQL,
|
|
292772
292862
|
columns: params.columns,
|
|
@@ -292779,14 +292869,20 @@ async function planSourceRefresh(params) {
|
|
|
292779
292869
|
reason: `the delta could not be planned (${errMessage(err)})`
|
|
292780
292870
|
};
|
|
292781
292871
|
}
|
|
292872
|
+
if (context.callerLedger && step.mode === "seed" && (step.reasonCode === "lineage_changed" || step.reasonCode === "table_renamed")) {
|
|
292873
|
+
throw new Error(`The supplied ledger entry for table "${lineage.physicalTableName}" ` + `was measured under a different definition of this source: ` + `${step.reason}. Delete the entry to rebuild the source, or set ` + `reseed.`);
|
|
292874
|
+
}
|
|
292875
|
+
return step;
|
|
292782
292876
|
}
|
|
292783
292877
|
async function advanceLedger(params) {
|
|
292784
292878
|
const { context, lineage } = params;
|
|
292879
|
+
if (context.callerLedger)
|
|
292880
|
+
return;
|
|
292785
292881
|
try {
|
|
292786
292882
|
await context.ledger.upsertIncrementalLedgerEntry({
|
|
292787
292883
|
environmentId: context.environmentId,
|
|
292788
292884
|
packageName: context.packageName,
|
|
292789
|
-
sourceEntityId:
|
|
292885
|
+
sourceEntityId: lineage.sourceEntityId,
|
|
292790
292886
|
coveredThroughValue: params.coveredThrough.value,
|
|
292791
292887
|
coveredThroughType: params.coveredThrough.malloyType,
|
|
292792
292888
|
watermarkDimension: lineage.watermarkName,
|
|
@@ -292799,18 +292895,21 @@ async function advanceLedger(params) {
|
|
|
292799
292895
|
} catch (err) {
|
|
292800
292896
|
logger.warn("Failed to advance the covered_through boundary", {
|
|
292801
292897
|
packageName: context.packageName,
|
|
292802
|
-
|
|
292898
|
+
physicalTableName: lineage.physicalTableName,
|
|
292899
|
+
sourceEntityId: lineage.sourceEntityId,
|
|
292803
292900
|
error: errMessage(err)
|
|
292804
292901
|
});
|
|
292805
292902
|
}
|
|
292806
292903
|
}
|
|
292807
|
-
async function resetLedger(context,
|
|
292904
|
+
async function resetLedger(context, lineage) {
|
|
292905
|
+
if (context.callerLedger)
|
|
292906
|
+
return;
|
|
292808
292907
|
try {
|
|
292809
|
-
await context.ledger.deleteIncrementalLedgerEntry(context.environmentId,
|
|
292908
|
+
await context.ledger.deleteIncrementalLedgerEntry(context.environmentId, lineage.connectionName, lineage.physicalTableName);
|
|
292810
292909
|
} catch (err) {
|
|
292811
292910
|
logger.warn("Failed to clear the covered_through boundary", {
|
|
292812
292911
|
packageName: context.packageName,
|
|
292813
|
-
|
|
292912
|
+
physicalTableName: lineage.physicalTableName,
|
|
292814
292913
|
error: errMessage(err)
|
|
292815
292914
|
});
|
|
292816
292915
|
}
|
|
@@ -292821,7 +292920,7 @@ async function advanceLedgerAfterSeed(params) {
|
|
|
292821
292920
|
if (boundary.error) {
|
|
292822
292921
|
logger.warn("Could not read a covered_through boundary after a full rebuild; " + "the next refresh will rebuild again", {
|
|
292823
292922
|
packageName: params.context.packageName,
|
|
292824
|
-
|
|
292923
|
+
physicalTableName: params.lineage.physicalTableName,
|
|
292825
292924
|
error: boundary.error
|
|
292826
292925
|
});
|
|
292827
292926
|
}
|
|
@@ -292830,7 +292929,6 @@ async function advanceLedgerAfterSeed(params) {
|
|
|
292830
292929
|
await advanceLedger({
|
|
292831
292930
|
context: params.context,
|
|
292832
292931
|
lineage: params.lineage,
|
|
292833
|
-
sourceEntityId: params.sourceEntityId,
|
|
292834
292932
|
coveredThrough: boundary.bound
|
|
292835
292933
|
});
|
|
292836
292934
|
return boundary.bound;
|
|
@@ -293114,8 +293212,24 @@ async function resolveEnvironmentId(repository, environmentName) {
|
|
|
293114
293212
|
}
|
|
293115
293213
|
|
|
293116
293214
|
// src/service/materialization_service.ts
|
|
293117
|
-
function
|
|
293118
|
-
return
|
|
293215
|
+
function refreshFields(did) {
|
|
293216
|
+
return did ? { refresh: did } : {};
|
|
293217
|
+
}
|
|
293218
|
+
function ledgerFields(lineage, bound) {
|
|
293219
|
+
if (!lineage || !bound)
|
|
293220
|
+
return {};
|
|
293221
|
+
return {
|
|
293222
|
+
ledger: {
|
|
293223
|
+
connectionName: lineage.connectionName,
|
|
293224
|
+
physicalTableName: lineage.physicalTableName,
|
|
293225
|
+
coveredThrough: bound.value,
|
|
293226
|
+
coveredThroughType: bound.malloyType,
|
|
293227
|
+
watermark: lineage.watermarkName,
|
|
293228
|
+
...lineage.mergeKeys.length > 0 ? { mergeKeys: lineage.mergeKeys } : {},
|
|
293229
|
+
strategy: lineage.strategy,
|
|
293230
|
+
sourceEntityId: lineage.sourceEntityId
|
|
293231
|
+
}
|
|
293232
|
+
};
|
|
293119
293233
|
}
|
|
293120
293234
|
function connectionMetadataLayers(environment, connectionName) {
|
|
293121
293235
|
try {
|
|
@@ -293258,7 +293372,7 @@ class MaterializationService {
|
|
|
293258
293372
|
const buildInstructions = options.buildInstructions;
|
|
293259
293373
|
const orchestrated = buildInstructions !== undefined;
|
|
293260
293374
|
if (orchestrated) {
|
|
293261
|
-
this.validateInstructions(pkg.getBuildPlan(), buildInstructions);
|
|
293375
|
+
this.validateInstructions(pkg.getBuildPlan(), buildInstructions, options.ledger, options.reseed ?? false);
|
|
293262
293376
|
}
|
|
293263
293377
|
const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
|
|
293264
293378
|
if (active2) {
|
|
@@ -293289,6 +293403,7 @@ class MaterializationService {
|
|
|
293289
293403
|
forceRefresh,
|
|
293290
293404
|
reseed,
|
|
293291
293405
|
buildInstructions,
|
|
293406
|
+
ledger: orchestrated ? options.ledger : undefined,
|
|
293292
293407
|
referenceManifest: options.referenceManifest,
|
|
293293
293408
|
strictUpstreams: options.strictUpstreams,
|
|
293294
293409
|
trigger,
|
|
@@ -293325,7 +293440,8 @@ class MaterializationService {
|
|
|
293325
293440
|
packageName,
|
|
293326
293441
|
materializationId: id,
|
|
293327
293442
|
forceRefresh: opts.reseed ?? false,
|
|
293328
|
-
now: new Date(startedAt)
|
|
293443
|
+
now: new Date(startedAt),
|
|
293444
|
+
ledger: opts.ledger
|
|
293329
293445
|
});
|
|
293330
293446
|
let instructions;
|
|
293331
293447
|
let carried;
|
|
@@ -293407,6 +293523,7 @@ class MaterializationService {
|
|
|
293407
293523
|
dialect: persistSource.dialectName,
|
|
293408
293524
|
physicalTableName: logicalName,
|
|
293409
293525
|
connectionName: persistSource.connectionName,
|
|
293526
|
+
sourceEntityId,
|
|
293410
293527
|
isStorageBuild: destination !== undefined
|
|
293411
293528
|
}) !== undefined;
|
|
293412
293529
|
const prior = priorEntries[sourceEntityId];
|
|
@@ -293541,7 +293658,7 @@ class MaterializationService {
|
|
|
293541
293658
|
});
|
|
293542
293659
|
}
|
|
293543
293660
|
}
|
|
293544
|
-
validateInstructions(plan, instructions) {
|
|
293661
|
+
validateInstructions(plan, instructions, ledger, runReseed = false) {
|
|
293545
293662
|
if (!plan) {
|
|
293546
293663
|
throw new BadRequestError("Package has no persist sources; buildInstructions cannot be applied");
|
|
293547
293664
|
}
|
|
@@ -293557,6 +293674,41 @@ class MaterializationService {
|
|
|
293557
293674
|
throw new BadRequestError("realization=SNAPSHOT is not supported (COPY only)");
|
|
293558
293675
|
}
|
|
293559
293676
|
}
|
|
293677
|
+
if (ledger === undefined || ledger.length === 0)
|
|
293678
|
+
return;
|
|
293679
|
+
const instructed = new Map;
|
|
293680
|
+
const byAddress = new Map(Object.values(plan.sources).map((s) => [s.sourceEntityId, s]));
|
|
293681
|
+
for (const instruction of instructions) {
|
|
293682
|
+
const planSource = (instruction.sourceID ? plan.sources[instruction.sourceID] : undefined) ?? byAddress.get(instruction.sourceEntityId);
|
|
293683
|
+
if (!planSource)
|
|
293684
|
+
continue;
|
|
293685
|
+
instructed.set(`${planSource.connectionName}\x00${instruction.physicalTableName}`, {
|
|
293686
|
+
sourceEntityId: planSource.sourceEntityId,
|
|
293687
|
+
refresh: planSource.refresh ?? null,
|
|
293688
|
+
reseed: runReseed || instruction.reseed === true
|
|
293689
|
+
});
|
|
293690
|
+
}
|
|
293691
|
+
const seen = new Set;
|
|
293692
|
+
for (const entry of ledger) {
|
|
293693
|
+
const key = `${entry.connectionName}\x00${entry.physicalTableName}`;
|
|
293694
|
+
const table = `'${entry.physicalTableName}' on connection '${entry.connectionName}'`;
|
|
293695
|
+
if (seen.has(key)) {
|
|
293696
|
+
throw new BadRequestError(`Ledger entry for table ${table} appears more than once`);
|
|
293697
|
+
}
|
|
293698
|
+
seen.add(key);
|
|
293699
|
+
const target = instructed.get(key);
|
|
293700
|
+
if (!target) {
|
|
293701
|
+
throw new BadRequestError(`Ledger entry names table ${table}, which this run's ` + `instructions do not build`);
|
|
293702
|
+
}
|
|
293703
|
+
if (target.reseed)
|
|
293704
|
+
continue;
|
|
293705
|
+
if (target.refresh !== "incremental") {
|
|
293706
|
+
throw new BadRequestError(`Ledger entry names table ${table}, whose source does not ` + `declare refresh="incremental"`);
|
|
293707
|
+
}
|
|
293708
|
+
if (entry.sourceEntityId !== target.sourceEntityId) {
|
|
293709
|
+
throw new BadRequestError(`Ledger entry for table ${table} was measured under a ` + `different definition of its source (its sourceEntityId does ` + `not match the plan's) — expected after a publish or ` + `rollback that changed the source. Delete the entry to ` + `rebuild the source, or set reseed.`);
|
|
293710
|
+
}
|
|
293711
|
+
}
|
|
293560
293712
|
}
|
|
293561
293713
|
quoteSeedTablePath(sourceEntityId, physicalTableName, connectionName, connections) {
|
|
293562
293714
|
const connection = connectionName ? connections.get(connectionName) : undefined;
|
|
@@ -293577,7 +293729,13 @@ class MaterializationService {
|
|
|
293577
293729
|
}
|
|
293578
293730
|
if (Object.keys(declarations).length === 0)
|
|
293579
293731
|
return;
|
|
293580
|
-
|
|
293732
|
+
const { ledger, ...rest } = run;
|
|
293733
|
+
return {
|
|
293734
|
+
...rest,
|
|
293735
|
+
declarations,
|
|
293736
|
+
ledger: this.repository,
|
|
293737
|
+
...ledger !== undefined ? { callerLedger: indexCallerLedger(ledger) } : {}
|
|
293738
|
+
};
|
|
293581
293739
|
}
|
|
293582
293740
|
async executeInstructedBuild(compiled, environment, instructions, seedEntries, signal, strict = false, owner, buildMetadata, incremental) {
|
|
293583
293741
|
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
@@ -293760,12 +293918,12 @@ class MaterializationService {
|
|
|
293760
293918
|
dialect,
|
|
293761
293919
|
physicalTableName,
|
|
293762
293920
|
connectionName: persistSource.connectionName,
|
|
293921
|
+
sourceEntityId: contentSourceEntityId,
|
|
293763
293922
|
isStorageBuild
|
|
293764
293923
|
}) : undefined;
|
|
293765
293924
|
const incrementalRefresh = incremental && lineage && contentSourceEntityId ? {
|
|
293766
293925
|
context: incremental,
|
|
293767
|
-
lineage
|
|
293768
|
-
ledgerKey: contentSourceEntityId
|
|
293926
|
+
lineage
|
|
293769
293927
|
} : undefined;
|
|
293770
293928
|
if (incrementalRefresh) {
|
|
293771
293929
|
const applied = await this.refreshOneSourceIncrementally({
|
|
@@ -293787,7 +293945,7 @@ class MaterializationService {
|
|
|
293787
293945
|
const quotedPhysical = quotedPhysicalPath;
|
|
293788
293946
|
const quotedBareName = quoteIdentifier(bareName, dialect);
|
|
293789
293947
|
if (incrementalRefresh) {
|
|
293790
|
-
await resetLedger(incrementalRefresh.context, incrementalRefresh.
|
|
293948
|
+
await resetLedger(incrementalRefresh.context, incrementalRefresh.lineage);
|
|
293791
293949
|
}
|
|
293792
293950
|
const startTime = performance.now();
|
|
293793
293951
|
await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`, runOptions);
|
|
@@ -293817,7 +293975,6 @@ class MaterializationService {
|
|
|
293817
293975
|
const seededThrough = incrementalRefresh ? await advanceLedgerAfterSeed({
|
|
293818
293976
|
context: incrementalRefresh.context,
|
|
293819
293977
|
lineage: incrementalRefresh.lineage,
|
|
293820
|
-
sourceEntityId: incrementalRefresh.ledgerKey,
|
|
293821
293978
|
quotedTablePath: quotedPhysical,
|
|
293822
293979
|
dialect,
|
|
293823
293980
|
runner: (sql) => connection.runSQL(sql, runOptions)
|
|
@@ -293829,7 +293986,8 @@ class MaterializationService {
|
|
|
293829
293986
|
physicalTableName,
|
|
293830
293987
|
connectionName: persistSource.connectionName,
|
|
293831
293988
|
realization: instruction.realization,
|
|
293832
|
-
...
|
|
293989
|
+
...ledgerFields(incrementalRefresh?.lineage, seededThrough),
|
|
293990
|
+
...refreshFields(incrementalRefresh ? "full" : undefined),
|
|
293833
293991
|
rowCount: null
|
|
293834
293992
|
};
|
|
293835
293993
|
}
|
|
@@ -293842,7 +294000,6 @@ class MaterializationService {
|
|
|
293842
294000
|
context,
|
|
293843
294001
|
lineage,
|
|
293844
294002
|
persistSource,
|
|
293845
|
-
sourceEntityId: params.ledgerKey,
|
|
293846
294003
|
quotedTablePath: params.quotedTablePath,
|
|
293847
294004
|
sourceSQL: params.buildSQL,
|
|
293848
294005
|
columns: deriveColumns(persistSource).map((c) => String(c.name)),
|
|
@@ -293865,7 +294022,6 @@ class MaterializationService {
|
|
|
293865
294022
|
await advanceLedger({
|
|
293866
294023
|
context,
|
|
293867
294024
|
lineage,
|
|
293868
|
-
sourceEntityId: params.ledgerKey,
|
|
293869
294025
|
coveredThrough: step.coveredThrough
|
|
293870
294026
|
});
|
|
293871
294027
|
const durationMs = Math.round(performance.now() - startTime);
|
|
@@ -293890,7 +294046,8 @@ class MaterializationService {
|
|
|
293890
294046
|
connectionName: persistSource.connectionName,
|
|
293891
294047
|
realization: instruction.realization,
|
|
293892
294048
|
rowCount: null,
|
|
293893
|
-
...
|
|
294049
|
+
...ledgerFields(lineage, step.coveredThrough),
|
|
294050
|
+
...refreshFields(step.mode === "delta" ? "delta" : "none")
|
|
293894
294051
|
};
|
|
293895
294052
|
}
|
|
293896
294053
|
async buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, buildSQL, builtEntries, dependsOnStorageUpstream) {
|
|
@@ -294472,7 +294629,7 @@ var ASSET_EXTENSIONS = new Set([
|
|
|
294472
294629
|
"woff2",
|
|
294473
294630
|
"xlsx"
|
|
294474
294631
|
]);
|
|
294475
|
-
var SPA_OWNED_SEGMENTS = new Set(["pages", "workbook"]);
|
|
294632
|
+
var SPA_OWNED_SEGMENTS = new Set(["data-apps", "pages", "workbook"]);
|
|
294476
294633
|
function extensionOf(segment) {
|
|
294477
294634
|
const dot = segment.lastIndexOf(".");
|
|
294478
294635
|
if (dot <= 0 || dot === segment.length - 1)
|
|
@@ -294784,7 +294941,7 @@ app.get("/environments/:environmentName/packages/:packageName", (req, res, next)
|
|
|
294784
294941
|
res.redirect(308, withRequestQuery(req, canonical));
|
|
294785
294942
|
});
|
|
294786
294943
|
app.get("/environments/:environmentName/packages/:packageName/*", serveFromPackage);
|
|
294787
|
-
var
|
|
294944
|
+
var DATA_APPS_DEPTH_CAP = 3;
|
|
294788
294945
|
var NON_TAG_TEXT_PATTERN = /<!--[\s\S]*?-->|<!--[\s\S]*$|<(script|style|title|textarea)\b[\s\S]*?<\/\1\s*>/gi;
|
|
294789
294946
|
function stripNonTagText(input) {
|
|
294790
294947
|
let current = input;
|
|
@@ -294795,7 +294952,7 @@ function stripNonTagText(input) {
|
|
|
294795
294952
|
} while (current !== previous);
|
|
294796
294953
|
return current;
|
|
294797
294954
|
}
|
|
294798
|
-
async function
|
|
294955
|
+
async function listPackageDataApps(environmentName, packageName, publicRoot) {
|
|
294799
294956
|
const fs11 = await import("fs/promises");
|
|
294800
294957
|
const out = [];
|
|
294801
294958
|
let realPublicRoot;
|
|
@@ -294805,7 +294962,7 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
|
|
|
294805
294962
|
return out;
|
|
294806
294963
|
}
|
|
294807
294964
|
async function walk(dir, depth) {
|
|
294808
|
-
if (depth >
|
|
294965
|
+
if (depth > DATA_APPS_DEPTH_CAP)
|
|
294809
294966
|
return;
|
|
294810
294967
|
let entries;
|
|
294811
294968
|
try {
|
|
@@ -294902,14 +295059,14 @@ try {
|
|
|
294902
295059
|
logger.warn("Failed to register Prometheus metrics endpoint", { error });
|
|
294903
295060
|
}
|
|
294904
295061
|
app.use(drainingGuard);
|
|
294905
|
-
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/
|
|
295062
|
+
app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/data-apps`, async (req, res) => {
|
|
294906
295063
|
try {
|
|
294907
295064
|
const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
|
|
294908
295065
|
const pkg = await environment.getPackage(req.params.packageName, false);
|
|
294909
|
-
const
|
|
294910
|
-
res.json(
|
|
295066
|
+
const dataApps = await listPackageDataApps(req.params.environmentName, req.params.packageName, path14.join(pkg.getPackagePath(), "public"));
|
|
295067
|
+
res.json(dataApps);
|
|
294911
295068
|
} catch (error) {
|
|
294912
|
-
logger.error("Failed to list package
|
|
295069
|
+
logger.error("Failed to list package data apps", { error });
|
|
294913
295070
|
const { json, status } = internalErrorToHttpError(error);
|
|
294914
295071
|
res.status(status).json(json);
|
|
294915
295072
|
}
|
package/package.json
CHANGED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{$ as t,j as o,J as a}from"./index-ByceOJbS.js";function s(){const n=t();return o.jsx(a,{onClickEnvironment:n})}export{s as default};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{F as j,$ as y,N,j as e,a0 as c,a1 as f,G as s,a2 as k,a3 as u,a4 as v,T as o,a5 as t,a6 as l}from"./index-ByceOJbS.js";function b(){const a=j(),n=a["*"],d=y(),{server:p}=N();if(!a.environmentName)return e.jsx("div",{children:e.jsx("h2",{children:"Missing environment name"})});if(!a.packageName)return e.jsx("div",{children:e.jsx("h2",{children:"Missing package name"})});const i={p:3,maxWidth:1200,mx:"auto"};if(n?.startsWith("pages/")&&!n.endsWith(".malloy")&&!n.endsWith(".malloynb")){const h=n.slice(6),g=c({environmentName:a.environmentName,packageName:a.packageName,modelPath:h});return e.jsx(f,{resourceUri:g})}const r=c({environmentName:a.environmentName,packageName:a.packageName,modelPath:n});if(n?.endsWith(".malloy"))return e.jsx(s,{sx:i,children:e.jsx(k,{resourceUri:r,runOnDemand:!0,maxResultSize:512*1024})});if(n?.endsWith(".malloynb"))return e.jsx(s,{sx:i,children:e.jsx(u,{resourceUri:r,maxResultSize:1024*1024,onNavigate:d})});const x=/\.[^./]+$/.test(n??""),m=v({server:p,environmentName:a.environmentName,packageName:a.packageName,path:n??""});return e.jsxs(s,{sx:i,children:[e.jsx(o,{variant:"h6",sx:{fontWeight:600},children:"Nothing to open at this path"}),e.jsxs(o,{variant:"body2",color:"text.secondary",sx:{mt:1},children:[e.jsx(s,{component:"span",sx:{fontFamily:t},children:n})," ","does not name a"," ",e.jsx(s,{component:"span",sx:{fontFamily:t},children:".malloy"})," ","or"," ",e.jsx(s,{component:"span",sx:{fontFamily:t},children:".malloynb"})," ","file in package"," ",e.jsx(s,{component:"span",sx:{fontFamily:t},children:a.packageName}),". This address opens"," ",e.jsx(s,{component:"span",sx:{fontFamily:t},children:".malloy"})," ","and"," ",e.jsx(s,{component:"span",sx:{fontFamily:t},children:".malloynb"})," ","files."]}),x&&e.jsxs(o,{variant:"body2",color:"text.secondary",sx:{mt:1},children:["A file from the package's"," ",e.jsx(s,{component:"span",sx:{fontFamily:t},children:"public/"})," ","directory is served at ",e.jsx(l,{href:m,children:m}),"."]}),e.jsxs(o,{variant:"body2",color:"text.secondary",sx:{mt:1},children:[e.jsxs(l,{href:`/${encodeURIComponent(a.environmentName)}/${encodeURIComponent(a.packageName)}`,children:["Back to ",a.packageName]})," ","lists this package's models, notebooks, and pages."]})]})}export{b as default};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{aa as o,j as r,ab as s,ac as a,G as n,T as t}from"./index-ByceOJbS.js";function x(){const e=o();return console.error(e),r.jsx(s,{maxWidth:"lg",component:"main",sx:{display:"flex",flexDirection:"column",my:2,gap:0},children:r.jsxs(a,{sx:{m:"auto",flexDirection:"column"},children:[r.jsx(n,{sx:{height:"300px"}}),r.jsx("img",{src:"/error.png"}),r.jsx(t,{variant:"subtitle1",children:"An unexpected error occurred"})]})})}export{x as default};
|