@malloy-publisher/server 0.0.210 → 0.0.212
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +20 -3
- package/dist/app/assets/{EnvironmentPage-BRMCY9d8.js → EnvironmentPage-CuO6sty5.js} +1 -1
- package/dist/app/assets/HomePage-u3vxROIF.js +1 -0
- package/dist/app/assets/{MainPage-sZdUjUcu.js → MainPage-Bt2sYLbr.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-C_jQQUCJ.js → MaterializationsPage-B1rj61zs.js} +1 -1
- package/dist/app/assets/{ModelPage-Bh62OIEq.js → ModelPage-BpvONvSR.js} +1 -1
- package/dist/app/assets/{PackagePage-DJW4xjLp.js → PackagePage-DHNcwboW.js} +1 -1
- package/dist/app/assets/{RouteError-Vi5yGs_F.js → RouteError-D1vhLJvr.js} +1 -1
- package/dist/app/assets/{WorkbookPage-BvJMi21d.js → WorkbookPage-CZmud-yI.js} +1 -1
- package/dist/app/assets/{core-BbW0t3RQ.es-L-mZcOk3.js → core-XtBSnW2U.es-DE88sVsZ.js} +1 -1
- package/dist/app/assets/{index-CCcHdeew.js → index-BQdOB6m9.js} +1 -1
- package/dist/app/assets/{index-DuqTjxM_.js → index-BlnQtDZj.js} +137 -137
- package/dist/app/assets/{index-CNFX-CGL.js → index-CYf2akGH.js} +1 -1
- package/dist/app/assets/{index.umd-GgEb4WfT.js → index.umd-BdQ0R4hx.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +3 -0
- package/dist/server.mjs +160 -140
- package/package.json +1 -1
- package/src/materialization_metrics.spec.ts +84 -0
- package/src/materialization_metrics.ts +78 -119
- package/src/service/build_plan.spec.ts +152 -0
- package/src/service/build_plan.ts +81 -2
- package/src/service/materialization_service.spec.ts +186 -8
- package/src/service/materialization_service.ts +96 -83
- package/src/service/materialization_test_fixtures.ts +2 -0
- package/src/service/package.ts +14 -2
- package/src/service/persist_annotation_validation.spec.ts +77 -0
- package/src/service/persist_annotation_validation.ts +54 -0
- package/src/service/quoting.spec.ts +79 -0
- package/src/service/quoting.ts +40 -16
- package/src/utils.ts +5 -0
- package/dist/app/assets/HomePage-DQzgkiMI.js +0 -1
package/dist/server.mjs
CHANGED
|
@@ -237699,97 +237699,52 @@ import { pathToFileURL as pathToFileURL2 } from "url";
|
|
|
237699
237699
|
|
|
237700
237700
|
// src/materialization_metrics.ts
|
|
237701
237701
|
var import_api4 = __toESM(require_src(), 1);
|
|
237702
|
-
var
|
|
237703
|
-
|
|
237704
|
-
|
|
237705
|
-
|
|
237706
|
-
|
|
237707
|
-
|
|
237708
|
-
|
|
237709
|
-
|
|
237710
|
-
|
|
237711
|
-
|
|
237712
|
-
|
|
237713
|
-
|
|
237714
|
-
|
|
237715
|
-
|
|
237716
|
-
|
|
237717
|
-
|
|
237718
|
-
|
|
237719
|
-
|
|
237720
|
-
|
|
237721
|
-
|
|
237722
|
-
runDuration = meter2.createHistogram("publisher_materialization_run_duration_ms", {
|
|
237723
|
-
description: "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').",
|
|
237724
|
-
unit: "ms"
|
|
237725
|
-
});
|
|
237726
|
-
}
|
|
237727
|
-
return { counter: runCounter, duration: runDuration };
|
|
237728
|
-
}
|
|
237702
|
+
var resetHooks = [];
|
|
237703
|
+
function lazyCounter(name, description) {
|
|
237704
|
+
let instrument2 = null;
|
|
237705
|
+
resetHooks.push(() => instrument2 = null);
|
|
237706
|
+
return () => instrument2 ??= import_api4.metrics.getMeter("publisher").createCounter(name, { description });
|
|
237707
|
+
}
|
|
237708
|
+
function lazyHistogram(name, description, unit) {
|
|
237709
|
+
let instrument2 = null;
|
|
237710
|
+
resetHooks.push(() => instrument2 = null);
|
|
237711
|
+
return () => instrument2 ??= import_api4.metrics.getMeter("publisher").createHistogram(name, { description, unit });
|
|
237712
|
+
}
|
|
237713
|
+
var runCounter = lazyCounter("publisher_materialization_runs_total", "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').");
|
|
237714
|
+
var runDuration = lazyHistogram("publisher_materialization_run_duration_ms", "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').", "ms");
|
|
237715
|
+
var sourcesCounter = lazyCounter("publisher_materialization_sources_total", "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').");
|
|
237716
|
+
var buildPlanComputeDuration = lazyHistogram("publisher_materialization_build_plan_compute_duration_ms", "Wall-clock duration of compiling a package's build plan (Package.buildPlan).", "ms");
|
|
237717
|
+
var autoLoadCounter = lazyCounter("publisher_materialization_auto_load_total", "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').");
|
|
237718
|
+
var connectionDigestSkipCounter = lazyCounter("publisher_materialization_connection_digest_skipped_total", "Connection digests skipped during build-plan compile because the connection did not resolve.");
|
|
237719
|
+
var manifestBindCounter = lazyCounter("publisher_materialization_manifest_bind_total", "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').");
|
|
237720
|
+
var sourceBuildDuration = lazyHistogram("publisher_materialization_source_build_duration_ms", "Wall-clock duration of building a single persist source.", "ms");
|
|
237721
|
+
var dropTablesCounter = lazyCounter("publisher_materialization_drop_tables_total", "Physical tables dropped on delete. Label: outcome ('success'|'failure').");
|
|
237729
237722
|
function recordMaterializationRun(mode, outcome, durationMs) {
|
|
237730
|
-
|
|
237731
|
-
|
|
237732
|
-
duration.record(durationMs, { mode });
|
|
237723
|
+
runCounter().add(1, { mode, outcome });
|
|
237724
|
+
runDuration().record(durationMs, { mode });
|
|
237733
237725
|
}
|
|
237734
237726
|
function recordSourcesOutcome(outcome, count) {
|
|
237735
237727
|
if (count <= 0)
|
|
237736
237728
|
return;
|
|
237737
|
-
|
|
237738
|
-
sourcesCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_sources_total", {
|
|
237739
|
-
description: "Persist sources processed by a materialization run. Label: outcome ('built'|'reused')."
|
|
237740
|
-
});
|
|
237741
|
-
}
|
|
237742
|
-
sourcesCounter.add(count, { outcome });
|
|
237729
|
+
sourcesCounter().add(count, { outcome });
|
|
237743
237730
|
}
|
|
237744
237731
|
function recordBuildPlanComputeDuration(durationMs) {
|
|
237745
|
-
|
|
237746
|
-
buildPlanComputeDuration = import_api4.metrics.getMeter("publisher").createHistogram("publisher_materialization_build_plan_compute_duration_ms", {
|
|
237747
|
-
description: "Wall-clock duration of compiling a package's build plan (Package.buildPlan).",
|
|
237748
|
-
unit: "ms"
|
|
237749
|
-
});
|
|
237750
|
-
}
|
|
237751
|
-
buildPlanComputeDuration.record(durationMs);
|
|
237732
|
+
buildPlanComputeDuration().record(durationMs);
|
|
237752
237733
|
}
|
|
237753
237734
|
function recordAutoLoadOutcome(outcome) {
|
|
237754
|
-
|
|
237755
|
-
autoLoadCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_auto_load_total", {
|
|
237756
|
-
description: "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure')."
|
|
237757
|
-
});
|
|
237758
|
-
}
|
|
237759
|
-
autoLoadCounter.add(1, { outcome });
|
|
237735
|
+
autoLoadCounter().add(1, { outcome });
|
|
237760
237736
|
}
|
|
237761
237737
|
function recordConnectionDigestSkipped() {
|
|
237762
|
-
|
|
237763
|
-
connectionDigestSkipCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_connection_digest_skipped_total", {
|
|
237764
|
-
description: "Connection digests skipped during build-plan compile because the connection did not resolve."
|
|
237765
|
-
});
|
|
237766
|
-
}
|
|
237767
|
-
connectionDigestSkipCounter.add(1);
|
|
237738
|
+
connectionDigestSkipCounter().add(1);
|
|
237768
237739
|
}
|
|
237769
237740
|
function recordManifestBind(outcome) {
|
|
237770
|
-
|
|
237771
|
-
manifestBindCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_manifest_bind_total", {
|
|
237772
|
-
description: "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout')."
|
|
237773
|
-
});
|
|
237774
|
-
}
|
|
237775
|
-
manifestBindCounter.add(1, { outcome });
|
|
237741
|
+
manifestBindCounter().add(1, { outcome });
|
|
237776
237742
|
}
|
|
237777
237743
|
function recordSourceBuildDuration(durationMs) {
|
|
237778
|
-
|
|
237779
|
-
sourceBuildDuration = import_api4.metrics.getMeter("publisher").createHistogram("publisher_materialization_source_build_duration_ms", {
|
|
237780
|
-
description: "Wall-clock duration of building a single persist source.",
|
|
237781
|
-
unit: "ms"
|
|
237782
|
-
});
|
|
237783
|
-
}
|
|
237784
|
-
sourceBuildDuration.record(durationMs);
|
|
237744
|
+
sourceBuildDuration().record(durationMs);
|
|
237785
237745
|
}
|
|
237786
237746
|
function recordDropTables(outcome) {
|
|
237787
|
-
|
|
237788
|
-
dropTablesCounter = import_api4.metrics.getMeter("publisher").createCounter("publisher_materialization_drop_tables_total", {
|
|
237789
|
-
description: "Physical tables dropped on delete. Label: outcome ('success'|'failure')."
|
|
237790
|
-
});
|
|
237791
|
-
}
|
|
237792
|
-
dropTablesCounter.add(1, { outcome });
|
|
237747
|
+
dropTablesCounter().add(1, { outcome });
|
|
237793
237748
|
}
|
|
237794
237749
|
|
|
237795
237750
|
// src/utils.ts
|
|
@@ -237808,6 +237763,9 @@ var URL_READER = {
|
|
|
237808
237763
|
function ignoreDotfiles(file) {
|
|
237809
237764
|
return path5.basename(file).startsWith(".");
|
|
237810
237765
|
}
|
|
237766
|
+
function errMessage(err) {
|
|
237767
|
+
return err instanceof Error ? err.message : String(err);
|
|
237768
|
+
}
|
|
237811
237769
|
|
|
237812
237770
|
// src/service/manifest_loader.ts
|
|
237813
237771
|
init_logger();
|
|
@@ -237889,6 +237847,7 @@ import {
|
|
|
237889
237847
|
} from "@malloydata/malloy";
|
|
237890
237848
|
|
|
237891
237849
|
// src/service/build_plan.ts
|
|
237850
|
+
init_constants();
|
|
237892
237851
|
init_logger();
|
|
237893
237852
|
|
|
237894
237853
|
// src/service/model.ts
|
|
@@ -239307,14 +239266,45 @@ function deriveColumns(persistSource) {
|
|
|
239307
239266
|
} catch (err) {
|
|
239308
239267
|
logger.warn("Failed to derive columns for persist source", {
|
|
239309
239268
|
sourceID: persistSource.sourceID,
|
|
239310
|
-
error:
|
|
239269
|
+
error: errMessage(err)
|
|
239311
239270
|
});
|
|
239312
239271
|
return [];
|
|
239313
239272
|
}
|
|
239314
239273
|
}
|
|
239274
|
+
function deriveAnnotationFields(persistSource) {
|
|
239275
|
+
const out = {};
|
|
239276
|
+
try {
|
|
239277
|
+
const tag = persistSource.annotations.parseAsTag("@").tag;
|
|
239278
|
+
for (const [key, value] of tag.entries()) {
|
|
239279
|
+
const text = value.text();
|
|
239280
|
+
if (text !== undefined)
|
|
239281
|
+
out[key] = text;
|
|
239282
|
+
}
|
|
239283
|
+
} catch {}
|
|
239284
|
+
return out;
|
|
239285
|
+
}
|
|
239315
239286
|
function flattenDependsOn(node) {
|
|
239316
239287
|
return node.dependsOn.map((d) => d.sourceID);
|
|
239317
239288
|
}
|
|
239289
|
+
function* iterGraphSources(graph, sources) {
|
|
239290
|
+
const seen = new Set;
|
|
239291
|
+
function* visit(node) {
|
|
239292
|
+
if (seen.has(node.sourceID))
|
|
239293
|
+
return;
|
|
239294
|
+
for (const dep of node.dependsOn) {
|
|
239295
|
+
yield* visit(dep);
|
|
239296
|
+
}
|
|
239297
|
+
seen.add(node.sourceID);
|
|
239298
|
+
const source = sources[node.sourceID];
|
|
239299
|
+
if (source)
|
|
239300
|
+
yield source;
|
|
239301
|
+
}
|
|
239302
|
+
for (const level of graph.nodes) {
|
|
239303
|
+
for (const node of level) {
|
|
239304
|
+
yield* visit(node);
|
|
239305
|
+
}
|
|
239306
|
+
}
|
|
239307
|
+
}
|
|
239318
239308
|
function computeBuildId(source, connectionDigests) {
|
|
239319
239309
|
return source.makeBuildId(connectionDigests[source.connectionName], source.getSQL());
|
|
239320
239310
|
}
|
|
@@ -239329,7 +239319,7 @@ async function resolvePackageConnections(pkg, names) {
|
|
|
239329
239319
|
map.set(name, await pkg.getMalloyConnection(name));
|
|
239330
239320
|
} catch (err) {
|
|
239331
239321
|
logger.warn(`Failed to resolve connection ${name}`, {
|
|
239332
|
-
error:
|
|
239322
|
+
error: errMessage(err)
|
|
239333
239323
|
});
|
|
239334
239324
|
}
|
|
239335
239325
|
}
|
|
@@ -239339,6 +239329,8 @@ async function compilePackageBuildPlan(pkg, signal) {
|
|
|
239339
239329
|
const allGraphs = [];
|
|
239340
239330
|
const allSources = {};
|
|
239341
239331
|
for (const modelPath of pkg.getModelPaths()) {
|
|
239332
|
+
if (!modelPath.endsWith(MODEL_FILE_SUFFIX))
|
|
239333
|
+
continue;
|
|
239342
239334
|
if (signal?.aborted)
|
|
239343
239335
|
throw new Error("Build cancelled");
|
|
239344
239336
|
const { runtime, modelURL, importBaseURL } = await Model.getModelRuntime(pkg.getPackagePath(), modelPath, pkg.getMalloyConfig());
|
|
@@ -239400,7 +239392,8 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames) {
|
|
|
239400
239392
|
dialect: source.dialectName,
|
|
239401
239393
|
buildId: computeBuildId(source, connectionDigests),
|
|
239402
239394
|
sql: source.getSQL(),
|
|
239403
|
-
columns: deriveColumns(source)
|
|
239395
|
+
columns: deriveColumns(source),
|
|
239396
|
+
annotationFields: deriveAnnotationFields(source)
|
|
239404
239397
|
};
|
|
239405
239398
|
}
|
|
239406
239399
|
return { graphs: wireGraphs, sources: wireSources };
|
|
@@ -239413,6 +239406,27 @@ async function computePackageBuildPlan(pkg, signal) {
|
|
|
239413
239406
|
return deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests);
|
|
239414
239407
|
}
|
|
239415
239408
|
|
|
239409
|
+
// src/service/persist_annotation_validation.ts
|
|
239410
|
+
init_errors();
|
|
239411
|
+
var PERSIST_LINE_PATTERN = /^\s*#@\s+persist\b/;
|
|
239412
|
+
var UNQUOTED_NAME_PATTERN = /\bname\s*=\s*(?!["'])/;
|
|
239413
|
+
function assertPersistNamesQuoted(modelSource, modelPath) {
|
|
239414
|
+
const offenders = [];
|
|
239415
|
+
for (const rawLine of modelSource.split(`
|
|
239416
|
+
`)) {
|
|
239417
|
+
if (!PERSIST_LINE_PATTERN.test(rawLine))
|
|
239418
|
+
continue;
|
|
239419
|
+
if (UNQUOTED_NAME_PATTERN.test(rawLine)) {
|
|
239420
|
+
offenders.push(rawLine.trim());
|
|
239421
|
+
}
|
|
239422
|
+
}
|
|
239423
|
+
if (offenders.length > 0) {
|
|
239424
|
+
throw new ModelCompilationError({
|
|
239425
|
+
message: `${modelPath}: persist annotation name must be quoted. Write a quoted ` + `value like name="engaged_events" (or a dialect table path such as ` + `name="my_dataset.engaged_events"), not a bare value — an unquoted ` + `persist name is dropped from the build plan, so the source would ` + `publish but never materialize. Offending annotation(s): ` + `${offenders.join("; ")}.`
|
|
239426
|
+
});
|
|
239427
|
+
}
|
|
239428
|
+
}
|
|
239429
|
+
|
|
239416
239430
|
// src/service/package.ts
|
|
239417
239431
|
class Package {
|
|
239418
239432
|
environmentName;
|
|
@@ -239556,6 +239570,10 @@ class Package {
|
|
|
239556
239570
|
}
|
|
239557
239571
|
const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
|
|
239558
239572
|
await model.validateRenderTags();
|
|
239573
|
+
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
239574
|
+
const modelSource = await fs6.readFile(path7.join(packagePath, sm.modelPath), "utf-8");
|
|
239575
|
+
assertPersistNamesQuoted(modelSource, sm.modelPath);
|
|
239576
|
+
}
|
|
239559
239577
|
models.set(sm.modelPath, model);
|
|
239560
239578
|
}
|
|
239561
239579
|
const endTime = performance.now();
|
|
@@ -239576,7 +239594,7 @@ class Package {
|
|
|
239576
239594
|
} catch (err) {
|
|
239577
239595
|
logger.warn(`Failed to compute build plan for package ${packageName}`, {
|
|
239578
239596
|
packageName,
|
|
239579
|
-
error:
|
|
239597
|
+
error: errMessage(err)
|
|
239580
239598
|
});
|
|
239581
239599
|
}
|
|
239582
239600
|
const invalidMsg = pkg.formatInvalidExplores();
|
|
@@ -248569,15 +248587,19 @@ init_logger();
|
|
|
248569
248587
|
import { Manifest } from "@malloydata/malloy";
|
|
248570
248588
|
|
|
248571
248589
|
// src/service/quoting.ts
|
|
248572
|
-
function
|
|
248590
|
+
function bareTableName(tableName) {
|
|
248573
248591
|
const lastDot = tableName.lastIndexOf(".");
|
|
248574
|
-
|
|
248575
|
-
|
|
248576
|
-
|
|
248577
|
-
|
|
248578
|
-
|
|
248592
|
+
return lastDot >= 0 ? tableName.substring(lastDot + 1) : tableName;
|
|
248593
|
+
}
|
|
248594
|
+
var BACKTICK_DIALECTS = new Set(["standardsql", "mysql", "databricks"]);
|
|
248595
|
+
function quoteIdentifier(identifier, dialect) {
|
|
248596
|
+
if (BACKTICK_DIALECTS.has(dialect)) {
|
|
248597
|
+
return "`" + identifier.replace(/`/g, "``") + "`";
|
|
248579
248598
|
}
|
|
248580
|
-
return
|
|
248599
|
+
return '"' + identifier.replace(/"/g, '""') + '"';
|
|
248600
|
+
}
|
|
248601
|
+
function quoteTablePath(tableName, dialect) {
|
|
248602
|
+
return tableName.split(".").map((segment) => quoteIdentifier(segment, dialect)).join(".");
|
|
248581
248603
|
}
|
|
248582
248604
|
|
|
248583
248605
|
// src/service/resolve_environment.ts
|
|
@@ -248596,11 +248618,7 @@ function stagingSuffix(buildId) {
|
|
|
248596
248618
|
return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
|
|
248597
248619
|
}
|
|
248598
248620
|
function selfAssignTableName(persistSource) {
|
|
248599
|
-
|
|
248600
|
-
return persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
|
|
248601
|
-
} catch {
|
|
248602
|
-
return persistSource.name;
|
|
248603
|
-
}
|
|
248621
|
+
return deriveAnnotationFields(persistSource).name || persistSource.name;
|
|
248604
248622
|
}
|
|
248605
248623
|
function outcomeFor(_err, signal) {
|
|
248606
248624
|
return signal?.aborted ? "cancelled" : "failed";
|
|
@@ -248668,7 +248686,7 @@ class MaterializationService {
|
|
|
248668
248686
|
}
|
|
248669
248687
|
const active2 = await this.repository.getActiveMaterialization(environmentId, packageName);
|
|
248670
248688
|
if (active2) {
|
|
248671
|
-
throw
|
|
248689
|
+
throw this.activeConflict(packageName, active2.id);
|
|
248672
248690
|
}
|
|
248673
248691
|
const forceRefresh = options.forceRefresh ?? false;
|
|
248674
248692
|
const metadata = {
|
|
@@ -248682,7 +248700,7 @@ class MaterializationService {
|
|
|
248682
248700
|
} catch (err) {
|
|
248683
248701
|
if (err instanceof DuplicateActiveMaterializationError) {
|
|
248684
248702
|
const winner = await this.repository.getActiveMaterialization(environmentId, packageName);
|
|
248685
|
-
throw
|
|
248703
|
+
throw this.activeConflict(packageName, winner?.id);
|
|
248686
248704
|
}
|
|
248687
248705
|
throw err;
|
|
248688
248706
|
}
|
|
@@ -248703,6 +248721,9 @@ class MaterializationService {
|
|
|
248703
248721
|
});
|
|
248704
248722
|
const startedAt = Date.now();
|
|
248705
248723
|
try {
|
|
248724
|
+
await this.repository.updateMaterialization(id, {
|
|
248725
|
+
startedAt: new Date(startedAt)
|
|
248726
|
+
});
|
|
248706
248727
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
248707
248728
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
248708
248729
|
const pkg = await environment.getPackage(packageName, false);
|
|
@@ -248753,29 +248774,24 @@ class MaterializationService {
|
|
|
248753
248774
|
const carried = {};
|
|
248754
248775
|
const seen = new Set;
|
|
248755
248776
|
for (const graph of compiled.graphs) {
|
|
248756
|
-
for (const
|
|
248757
|
-
|
|
248758
|
-
|
|
248759
|
-
|
|
248760
|
-
|
|
248761
|
-
|
|
248762
|
-
|
|
248763
|
-
|
|
248764
|
-
|
|
248765
|
-
|
|
248766
|
-
|
|
248767
|
-
const prior = priorEntries[buildId];
|
|
248768
|
-
if (prior && prior.physicalTableName) {
|
|
248769
|
-
carried[buildId] = prior;
|
|
248770
|
-
continue;
|
|
248771
|
-
}
|
|
248772
|
-
instructions.push({
|
|
248773
|
-
buildId,
|
|
248774
|
-
materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
|
|
248775
|
-
physicalTableName: selfAssignTableName(persistSource),
|
|
248776
|
-
realization: "COPY"
|
|
248777
|
-
});
|
|
248777
|
+
for (const persistSource of iterGraphSources(graph, compiled.sources)) {
|
|
248778
|
+
if (include && !include.has(persistSource.name))
|
|
248779
|
+
continue;
|
|
248780
|
+
const buildId = computeBuildId(persistSource, compiled.connectionDigests);
|
|
248781
|
+
if (seen.has(buildId))
|
|
248782
|
+
continue;
|
|
248783
|
+
seen.add(buildId);
|
|
248784
|
+
const prior = priorEntries[buildId];
|
|
248785
|
+
if (prior && prior.physicalTableName) {
|
|
248786
|
+
carried[buildId] = prior;
|
|
248787
|
+
continue;
|
|
248778
248788
|
}
|
|
248789
|
+
instructions.push({
|
|
248790
|
+
buildId,
|
|
248791
|
+
materializedTableId: `local-${buildId.substring(0, STAGING_BUILD_ID_LEN)}`,
|
|
248792
|
+
physicalTableName: selfAssignTableName(persistSource),
|
|
248793
|
+
realization: "COPY"
|
|
248794
|
+
});
|
|
248779
248795
|
}
|
|
248780
248796
|
}
|
|
248781
248797
|
return { instructions, carried };
|
|
@@ -248849,20 +248865,15 @@ class MaterializationService {
|
|
|
248849
248865
|
if (!connection) {
|
|
248850
248866
|
throw new BadRequestError(`Connection '${graph.connectionName}' not found`);
|
|
248851
248867
|
}
|
|
248852
|
-
for (const
|
|
248853
|
-
|
|
248854
|
-
|
|
248855
|
-
|
|
248856
|
-
|
|
248857
|
-
|
|
248858
|
-
|
|
248859
|
-
|
|
248860
|
-
|
|
248861
|
-
if (!instruction)
|
|
248862
|
-
continue;
|
|
248863
|
-
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
|
|
248864
|
-
entries[buildId] = entry;
|
|
248865
|
-
}
|
|
248868
|
+
for (const persistSource of iterGraphSources(graph, sources)) {
|
|
248869
|
+
if (signal.aborted)
|
|
248870
|
+
throw new Error("Build cancelled");
|
|
248871
|
+
const buildId = computeBuildId(persistSource, connectionDigests);
|
|
248872
|
+
const instruction = byBuildId.get(buildId);
|
|
248873
|
+
if (!instruction)
|
|
248874
|
+
continue;
|
|
248875
|
+
const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest);
|
|
248876
|
+
entries[buildId] = entry;
|
|
248866
248877
|
}
|
|
248867
248878
|
}
|
|
248868
248879
|
return entries;
|
|
@@ -248874,22 +248885,26 @@ class MaterializationService {
|
|
|
248874
248885
|
buildManifest: manifest.buildManifest,
|
|
248875
248886
|
connectionDigests
|
|
248876
248887
|
});
|
|
248877
|
-
const
|
|
248888
|
+
const bareName = bareTableName(physicalTableName);
|
|
248878
248889
|
const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
|
|
248890
|
+
const dialect = persistSource.dialectName;
|
|
248891
|
+
const quotedStaging = quoteTablePath(stagingTableName, dialect);
|
|
248892
|
+
const quotedPhysical = quoteTablePath(physicalTableName, dialect);
|
|
248893
|
+
const quotedBareName = quoteIdentifier(bareName, dialect);
|
|
248879
248894
|
const startTime = performance.now();
|
|
248880
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
248895
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
|
|
248881
248896
|
try {
|
|
248882
|
-
await connection.runSQL(`CREATE TABLE ${
|
|
248883
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
248884
|
-
await connection.runSQL(`ALTER TABLE ${
|
|
248897
|
+
await connection.runSQL(`CREATE TABLE ${quotedStaging} AS (${buildSQL})`);
|
|
248898
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${quotedPhysical}`);
|
|
248899
|
+
await connection.runSQL(`ALTER TABLE ${quotedStaging} RENAME TO ${quotedBareName}`);
|
|
248885
248900
|
} catch (err) {
|
|
248886
248901
|
try {
|
|
248887
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${
|
|
248902
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
|
|
248888
248903
|
} catch (cleanupErr) {
|
|
248889
248904
|
logger.warn("Failed to clean up staging table after a failed build; physical leak", {
|
|
248890
248905
|
stagingTableName,
|
|
248891
248906
|
connectionName: persistSource.connectionName,
|
|
248892
|
-
cleanupError:
|
|
248907
|
+
cleanupError: errMessage(cleanupErr)
|
|
248893
248908
|
});
|
|
248894
248909
|
}
|
|
248895
248910
|
throw err;
|
|
@@ -248969,8 +248984,9 @@ class MaterializationService {
|
|
|
248969
248984
|
connection = await pkg.getMalloyConnection(connectionName);
|
|
248970
248985
|
connectionCache.set(connectionName, connection);
|
|
248971
248986
|
}
|
|
248972
|
-
|
|
248973
|
-
await connection.runSQL(`DROP TABLE IF EXISTS ${physicalTableName
|
|
248987
|
+
const dialect = connection.dialectName;
|
|
248988
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${quoteTablePath(physicalTableName, dialect)}`);
|
|
248989
|
+
await connection.runSQL(`DROP TABLE IF EXISTS ${quoteTablePath(`${physicalTableName}${stagingSuffix(entry.buildId)}`, dialect)}`);
|
|
248974
248990
|
recordDropTables("success");
|
|
248975
248991
|
logger.info("Dropped materialized table on delete", {
|
|
248976
248992
|
materializationId: m.id,
|
|
@@ -248983,7 +248999,7 @@ class MaterializationService {
|
|
|
248983
248999
|
materializationId: m.id,
|
|
248984
249000
|
physicalTableName,
|
|
248985
249001
|
connectionName,
|
|
248986
|
-
error:
|
|
249002
|
+
error: errMessage(err)
|
|
248987
249003
|
});
|
|
248988
249004
|
}
|
|
248989
249005
|
}
|
|
@@ -249001,6 +249017,10 @@ class MaterializationService {
|
|
|
249001
249017
|
metadata
|
|
249002
249018
|
});
|
|
249003
249019
|
}
|
|
249020
|
+
activeConflict(packageName, activeId) {
|
|
249021
|
+
const suffix = activeId ? ` (${activeId})` : "";
|
|
249022
|
+
return new MaterializationConflictError(`Package ${packageName} already has an active materialization${suffix}`);
|
|
249023
|
+
}
|
|
249004
249024
|
recordRun(mode, outcome, startedAtMs) {
|
|
249005
249025
|
recordMaterializationRun(mode, outcome, Date.now() - startedAtMs);
|
|
249006
249026
|
}
|
|
@@ -249008,7 +249028,7 @@ class MaterializationService {
|
|
|
249008
249028
|
const abortController = new AbortController;
|
|
249009
249029
|
this.runningAbortControllers.set(id, abortController);
|
|
249010
249030
|
run(abortController.signal).catch(async (err) => {
|
|
249011
|
-
const message =
|
|
249031
|
+
const message = errMessage(err);
|
|
249012
249032
|
const next = abortController.signal.aborted ? "CANCELLED" : "FAILED";
|
|
249013
249033
|
try {
|
|
249014
249034
|
await this.repository.updateMaterialization(id, {
|
|
@@ -249020,7 +249040,7 @@ class MaterializationService {
|
|
|
249020
249040
|
logger.error("Failed to record materialization failure", {
|
|
249021
249041
|
materializationId: id,
|
|
249022
249042
|
originalError: message,
|
|
249023
|
-
transitionError:
|
|
249043
|
+
transitionError: errMessage(transitionErr)
|
|
249024
249044
|
});
|
|
249025
249045
|
}
|
|
249026
249046
|
}).finally(() => {
|
package/package.json
CHANGED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
recordConnectionDigestSkipped,
|
|
5
|
+
recordDropTables,
|
|
6
|
+
recordMaterializationRun,
|
|
7
|
+
recordSourcesOutcome,
|
|
8
|
+
resetMaterializationTelemetryForTesting,
|
|
9
|
+
} from "./materialization_metrics";
|
|
10
|
+
import {
|
|
11
|
+
startMetricsHarness,
|
|
12
|
+
type MetricsHarness,
|
|
13
|
+
} from "./test_helpers/metrics_harness";
|
|
14
|
+
|
|
15
|
+
describe("materialization_metrics", () => {
|
|
16
|
+
let harness: MetricsHarness;
|
|
17
|
+
|
|
18
|
+
beforeEach(async () => {
|
|
19
|
+
harness = await startMetricsHarness();
|
|
20
|
+
// Drop cached instruments so they re-bind to this test's provider.
|
|
21
|
+
resetMaterializationTelemetryForTesting();
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
afterEach(async () => {
|
|
25
|
+
resetMaterializationTelemetryForTesting();
|
|
26
|
+
await harness.shutdown();
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it("counts runs labeled by mode and outcome", async () => {
|
|
30
|
+
recordMaterializationRun("auto", "success", 10);
|
|
31
|
+
recordMaterializationRun("auto", "success", 20);
|
|
32
|
+
recordMaterializationRun("orchestrated", "failed", 5);
|
|
33
|
+
|
|
34
|
+
expect(
|
|
35
|
+
await harness.collectCounter("publisher_materialization_runs_total", {
|
|
36
|
+
mode: "auto",
|
|
37
|
+
outcome: "success",
|
|
38
|
+
}),
|
|
39
|
+
).toBe(2);
|
|
40
|
+
expect(
|
|
41
|
+
await harness.collectCounter("publisher_materialization_runs_total", {
|
|
42
|
+
mode: "orchestrated",
|
|
43
|
+
outcome: "failed",
|
|
44
|
+
}),
|
|
45
|
+
).toBe(1);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it("adds the source count labeled by outcome, ignoring non-positive counts", async () => {
|
|
49
|
+
recordSourcesOutcome("built", 3);
|
|
50
|
+
recordSourcesOutcome("reused", 2);
|
|
51
|
+
recordSourcesOutcome("built", 0); // guarded: no emission
|
|
52
|
+
|
|
53
|
+
expect(
|
|
54
|
+
await harness.collectCounter(
|
|
55
|
+
"publisher_materialization_sources_total",
|
|
56
|
+
{ outcome: "built" },
|
|
57
|
+
),
|
|
58
|
+
).toBe(3);
|
|
59
|
+
expect(
|
|
60
|
+
await harness.collectCounter(
|
|
61
|
+
"publisher_materialization_sources_total",
|
|
62
|
+
{ outcome: "reused" },
|
|
63
|
+
),
|
|
64
|
+
).toBe(2);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
it("counts drop-table outcomes and connection-digest skips", async () => {
|
|
68
|
+
recordDropTables("success");
|
|
69
|
+
recordDropTables("failure");
|
|
70
|
+
recordConnectionDigestSkipped();
|
|
71
|
+
|
|
72
|
+
expect(
|
|
73
|
+
await harness.collectCounter(
|
|
74
|
+
"publisher_materialization_drop_tables_total",
|
|
75
|
+
{ outcome: "failure" },
|
|
76
|
+
),
|
|
77
|
+
).toBe(1);
|
|
78
|
+
expect(
|
|
79
|
+
await harness.collectCounter(
|
|
80
|
+
"publisher_materialization_connection_digest_skipped_total",
|
|
81
|
+
),
|
|
82
|
+
).toBe(1);
|
|
83
|
+
});
|
|
84
|
+
});
|