@malloy-publisher/server 0.0.228 → 0.0.229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +84 -1
- package/dist/app/assets/EnvironmentPage-QOoHiVeJ.js +1 -0
- package/dist/app/assets/{HomePage-Bkwc9Woc.js → HomePage-C71GOfVW.js} +1 -1
- package/dist/app/assets/{LightMode-Bum_KBpN.js → LightMode-CrgCAwLe.js} +1 -1
- package/dist/app/assets/MainPage-BG5__FN3.js +2 -0
- package/dist/app/assets/MaterializationsPage-DE6PnrDR.js +1 -0
- package/dist/app/assets/ModelPage-CcBjcbLm.js +1 -0
- package/dist/app/assets/PackagePage-JTy3ztkB.js +1 -0
- package/dist/app/assets/{RouteError-DmJLpLXm.js → RouteError-Cymbp47a.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-C_nMnHr8.js +1 -0
- package/dist/app/assets/WorkbookPage-CPQu-DQx.js +1 -0
- package/dist/app/assets/{core-CEDZMHV1.es-_yGzNgNH.js → core-Coi3caGs.es-CSOmajHS.js} +10 -10
- package/dist/app/assets/{index-DDMrjIT3.js → index-CM2qhQCI.js} +1 -1
- package/dist/app/assets/{index-CE9xhdra.js → index-CcuuST2X.js} +1 -1
- package/dist/app/assets/{index-OEjKNSYb.js → index-CkmABCAw.js} +388 -241
- package/dist/app/assets/{index-CdmFub34.js → index-DlWCXghy.js} +1 -1
- package/dist/app/assets/index-DxArlgRD.js +529 -0
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +1 -1
- package/dist/package_load_worker.mjs +1 -1
- package/dist/server.mjs +6934 -36
- package/package.json +4 -2
- package/src/config.spec.ts +87 -1
- package/src/config.ts +102 -1
- package/src/controller/materialization.controller.spec.ts +9 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/materialization_metrics.ts +33 -0
- package/src/mcp/skills/build_skills_bundle.ts +29 -9
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +31 -2
- package/src/query_param_utils.ts +11 -0
- package/src/server.ts +67 -9
- package/src/service/connection.spec.ts +8 -7
- package/src/service/cron_evaluator.spec.ts +79 -0
- package/src/service/cron_evaluator.ts +105 -0
- package/src/service/environment.ts +118 -13
- package/src/service/environment_store.spec.ts +372 -2
- package/src/service/environment_store.ts +149 -17
- package/src/service/environment_store_anchoring.spec.ts +107 -0
- package/src/service/manifest_loader.spec.ts +3 -1
- package/src/service/manifest_loader.ts +9 -5
- package/src/service/materialization_scheduler.spec.ts +260 -0
- package/src/service/materialization_scheduler.ts +304 -0
- package/src/service/materialization_service.spec.ts +167 -4
- package/src/service/materialization_service.ts +121 -8
- package/src/service/materialization_test_fixtures.ts +10 -1
- package/src/service/package.ts +90 -1
- package/src/service/package_quote_bound_tables.spec.ts +171 -0
- package/src/service/persistence_policy.spec.ts +31 -0
- package/src/service/quoting.spec.ts +57 -1
- package/src/service/quoting.ts +33 -0
- package/src/storage/DatabaseInterface.ts +15 -0
- package/src/storage/duckdb/DuckDBRepository.ts +17 -0
- package/src/storage/duckdb/MaterializationRepository.spec.ts +154 -0
- package/src/storage/duckdb/MaterializationRepository.ts +56 -0
- package/tests/fixtures/persist-schedule-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-test/persist_schedule_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-test/publisher.json +7 -0
- package/tests/fixtures/persist-schedule-warn-test/data/orders.csv +5 -0
- package/tests/fixtures/persist-schedule-warn-test/persist_warn_test.malloy +11 -0
- package/tests/fixtures/persist-schedule-warn-test/publisher.json +6 -0
- package/tests/integration/materialization/environment_list.integration.spec.ts +133 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +7 -2
- package/tests/integration/materialization/orchestrated_rebind.integration.spec.ts +150 -0
- package/tests/integration/materialization/schedule_edit.integration.spec.ts +126 -0
- package/tests/integration/materialization/scheduler.integration.spec.ts +200 -0
- package/tests/integration/materialization/scheduler_recovery.integration.spec.ts +172 -0
- package/tests/integration/materialization/scheduler_wiring.integration.spec.ts +143 -0
- package/dist/app/assets/EnvironmentPage-EW2lbGvb.js +0 -1
- package/dist/app/assets/MainPage-oiEy7TNM.js +0 -2
- package/dist/app/assets/MaterializationsPage-C_VJsTgU.js +0 -1
- package/dist/app/assets/ModelPage-z8REqAmk.js +0 -1
- package/dist/app/assets/PackagePage-C2Vtt1Ln.js +0 -1
- package/dist/app/assets/ThemeEditorPage-BywFjC7A.js +0 -1
- package/dist/app/assets/WorkbookPage-DCMizDMR.js +0 -1
- package/dist/app/assets/index-EqslXZ44.js +0 -527
|
@@ -14,13 +14,13 @@ import {
|
|
|
14
14
|
MaterializationMode,
|
|
15
15
|
recordAutoLoadOutcome,
|
|
16
16
|
recordDropTables,
|
|
17
|
+
recordManifestBindDegraded,
|
|
17
18
|
recordMaterializationRun,
|
|
18
19
|
recordSourceBuildDuration,
|
|
19
20
|
recordSourcesOutcome,
|
|
20
21
|
} from "../materialization_metrics";
|
|
21
22
|
import {
|
|
22
23
|
BuildInstruction,
|
|
23
|
-
BuildManifest,
|
|
24
24
|
BuildManifestResult,
|
|
25
25
|
BuildPlan,
|
|
26
26
|
FreshnessManifest,
|
|
@@ -41,7 +41,12 @@ import {
|
|
|
41
41
|
iterGraphSources,
|
|
42
42
|
} from "./build_plan";
|
|
43
43
|
import { EnvironmentStore } from "./environment_store";
|
|
44
|
-
import {
|
|
44
|
+
import {
|
|
45
|
+
bareTableName,
|
|
46
|
+
quoteIdentifier,
|
|
47
|
+
quoteManifestTablePath,
|
|
48
|
+
quoteTablePath,
|
|
49
|
+
} from "./quoting";
|
|
45
50
|
import { resolveEnvironmentId } from "./resolve_environment";
|
|
46
51
|
|
|
47
52
|
/**
|
|
@@ -177,6 +182,38 @@ export class MaterializationService {
|
|
|
177
182
|
);
|
|
178
183
|
}
|
|
179
184
|
|
|
185
|
+
/**
|
|
186
|
+
* Every materialization across all packages in an environment, newest first.
|
|
187
|
+
* Each record carries its `packageName`, so an env-scoped view can group or
|
|
188
|
+
* label by package without a per-package fan-out.
|
|
189
|
+
*/
|
|
190
|
+
async listEnvironmentMaterializations(
|
|
191
|
+
environmentName: string,
|
|
192
|
+
options?: { limit?: number; offset?: number },
|
|
193
|
+
): Promise<Materialization[]> {
|
|
194
|
+
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
195
|
+
return this.repository.listMaterializationsByEnvironment(
|
|
196
|
+
environmentId,
|
|
197
|
+
options,
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* `created_at` of the newest scheduler-fired materialization for a package,
|
|
203
|
+
* or null if none. The standalone scheduler uses this on its first arm to
|
|
204
|
+
* recover a fire missed during downtime (see MaterializationScheduler.arm).
|
|
205
|
+
*/
|
|
206
|
+
async getLatestScheduledFireAt(
|
|
207
|
+
environmentName: string,
|
|
208
|
+
packageName: string,
|
|
209
|
+
): Promise<Date | null> {
|
|
210
|
+
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
211
|
+
return this.repository.getLatestScheduledFireAt(
|
|
212
|
+
environmentId,
|
|
213
|
+
packageName,
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
180
217
|
async getMaterialization(
|
|
181
218
|
environmentName: string,
|
|
182
219
|
packageName: string,
|
|
@@ -219,6 +256,13 @@ export class MaterializationService {
|
|
|
219
256
|
buildInstructions?: BuildInstruction[];
|
|
220
257
|
referenceManifest?: ManifestReference[];
|
|
221
258
|
strictUpstreams?: boolean;
|
|
259
|
+
/**
|
|
260
|
+
* What initiated this run. `ON_DEMAND` (default) = a manual/API create;
|
|
261
|
+
* `SCHEDULER` = the standalone materialization scheduler firing a
|
|
262
|
+
* package's `materialization.schedule` cron. Recorded on the run
|
|
263
|
+
* metadata so a scheduled rebuild is distinguishable from a manual one.
|
|
264
|
+
*/
|
|
265
|
+
trigger?: "ON_DEMAND" | "SCHEDULER";
|
|
222
266
|
} = {},
|
|
223
267
|
): Promise<Materialization> {
|
|
224
268
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
@@ -244,10 +288,12 @@ export class MaterializationService {
|
|
|
244
288
|
}
|
|
245
289
|
|
|
246
290
|
const forceRefresh = options.forceRefresh ?? false;
|
|
291
|
+
const trigger = options.trigger ?? "ON_DEMAND";
|
|
247
292
|
const metadata = {
|
|
248
293
|
forceRefresh,
|
|
249
294
|
sourceNames: options.sourceNames ?? null,
|
|
250
295
|
mode: orchestrated ? "orchestrated" : "auto",
|
|
296
|
+
trigger,
|
|
251
297
|
};
|
|
252
298
|
|
|
253
299
|
let created: Materialization;
|
|
@@ -280,6 +326,7 @@ export class MaterializationService {
|
|
|
280
326
|
buildInstructions,
|
|
281
327
|
referenceManifest: options.referenceManifest,
|
|
282
328
|
strictUpstreams: options.strictUpstreams,
|
|
329
|
+
trigger,
|
|
283
330
|
},
|
|
284
331
|
signal,
|
|
285
332
|
),
|
|
@@ -306,6 +353,7 @@ export class MaterializationService {
|
|
|
306
353
|
buildInstructions: BuildInstruction[] | undefined;
|
|
307
354
|
referenceManifest: ManifestReference[] | undefined;
|
|
308
355
|
strictUpstreams: boolean | undefined;
|
|
356
|
+
trigger: "ON_DEMAND" | "SCHEDULER";
|
|
309
357
|
},
|
|
310
358
|
signal: AbortSignal,
|
|
311
359
|
): Promise<void> {
|
|
@@ -379,6 +427,7 @@ export class MaterializationService {
|
|
|
379
427
|
forceRefresh: opts.forceRefresh,
|
|
380
428
|
sourceNames: opts.sourceNames ?? null,
|
|
381
429
|
mode,
|
|
430
|
+
trigger: opts.trigger,
|
|
382
431
|
sourcesBuilt,
|
|
383
432
|
sourcesReused,
|
|
384
433
|
durationMs,
|
|
@@ -466,9 +515,13 @@ export class MaterializationService {
|
|
|
466
515
|
/**
|
|
467
516
|
* Project the caller-supplied upstream reference manifest into the seed
|
|
468
517
|
* entry map `executeInstructedBuild` consumes. Each reference is keyed by the
|
|
469
|
-
* compiler's manifest-lookup sourceEntityId and carries
|
|
470
|
-
*
|
|
471
|
-
* reference to the existing table
|
|
518
|
+
* compiler's manifest-lookup sourceEntityId and carries the physical table
|
|
519
|
+
* name plus the connection it lives on — enough for the build Manifest to
|
|
520
|
+
* resolve a downstream persist reference to the existing table, and to quote
|
|
521
|
+
* that reference for the connection's dialect so it resolves on a
|
|
522
|
+
* case-folding engine (see {@link quoteSeedTablePath}). `connectionName` is
|
|
523
|
+
* optional on the wire: an older control plane that omits it seeds unquoted,
|
|
524
|
+
* exactly as before.
|
|
472
525
|
*/
|
|
473
526
|
private referenceManifestToEntries(
|
|
474
527
|
referenceManifest: ManifestReference[] | undefined,
|
|
@@ -478,6 +531,7 @@ export class MaterializationService {
|
|
|
478
531
|
entries[ref.sourceEntityId] = {
|
|
479
532
|
sourceEntityId: ref.sourceEntityId,
|
|
480
533
|
physicalTableName: ref.physicalTableName,
|
|
534
|
+
connectionName: ref.connectionName,
|
|
481
535
|
};
|
|
482
536
|
}
|
|
483
537
|
return entries;
|
|
@@ -524,11 +578,17 @@ export class MaterializationService {
|
|
|
524
578
|
// stamps freshness (dataAsOf/window/fallback) on the wire manifest it
|
|
525
579
|
// distributes, not on this in-memory post-build load, so these sources are
|
|
526
580
|
// bound un-gated (always serve the freshly-built table).
|
|
527
|
-
const manifestEntries:
|
|
581
|
+
const manifestEntries: FreshnessManifest = {};
|
|
528
582
|
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
529
583
|
if (entry.physicalTableName) {
|
|
530
584
|
manifestEntries[sourceEntityId] = {
|
|
531
585
|
tableName: entry.physicalTableName,
|
|
586
|
+
// Carried so the bind step can quote the physical path for the
|
|
587
|
+
// connection's dialect (Package.quoteBoundTableNames) — the
|
|
588
|
+
// build CREATEd it quoted, so an unquoted read would miss on a
|
|
589
|
+
// case-folding engine in the window before the control plane's
|
|
590
|
+
// wire-manifest rebind.
|
|
591
|
+
connectionName: entry.connectionName,
|
|
532
592
|
};
|
|
533
593
|
}
|
|
534
594
|
}
|
|
@@ -586,6 +646,43 @@ export class MaterializationService {
|
|
|
586
646
|
}
|
|
587
647
|
}
|
|
588
648
|
|
|
649
|
+
/**
|
|
650
|
+
* Quote a seeded upstream's physical path for the in-memory build Manifest,
|
|
651
|
+
* mirroring the CREATE side ({@link quoteManifestTablePath}) so a downstream
|
|
652
|
+
* persist source resolves the reference on a case-folding engine. The seed's
|
|
653
|
+
* own connection (carried on the entry) supplies the dialect. If it can't be
|
|
654
|
+
* resolved — an older control plane omitted `connectionName`, or the named
|
|
655
|
+
* connection isn't part of this build — the path binds unquoted: the build
|
|
656
|
+
* then fails loudly at the downstream CREATE on a case-folding engine (a
|
|
657
|
+
* self-signaling miss, unlike the serve path), and is unaffected elsewhere.
|
|
658
|
+
*/
|
|
659
|
+
private quoteSeedTablePath(
|
|
660
|
+
sourceEntityId: string,
|
|
661
|
+
physicalTableName: string,
|
|
662
|
+
connectionName: string | undefined,
|
|
663
|
+
connections: Map<string, MalloyConnection>,
|
|
664
|
+
): string {
|
|
665
|
+
const connection = connectionName
|
|
666
|
+
? connections.get(connectionName)
|
|
667
|
+
: undefined;
|
|
668
|
+
if (!connection) {
|
|
669
|
+
// A named-but-absent connection is a real gap (the seed can't be
|
|
670
|
+
// quoted); a missing name is the benign older-CP default. Only the
|
|
671
|
+
// former is worth a signal.
|
|
672
|
+
if (connectionName) {
|
|
673
|
+
recordManifestBindDegraded();
|
|
674
|
+
logger.warn(
|
|
675
|
+
"Seeded upstream names a connection not present in this build; " +
|
|
676
|
+
"leaving its manifest path unquoted (a downstream build will " +
|
|
677
|
+
"fail on a case-folding engine)",
|
|
678
|
+
{ sourceEntityId, connectionName },
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
return physicalTableName;
|
|
682
|
+
}
|
|
683
|
+
return quoteManifestTablePath(physicalTableName, connection.dialectName);
|
|
684
|
+
}
|
|
685
|
+
|
|
589
686
|
/**
|
|
590
687
|
* Shared build loop for both auto-run and orchestrated builds. Seeds the
|
|
591
688
|
* manifest with carried-forward (reused) upstream entries so downstream
|
|
@@ -630,8 +727,19 @@ export class MaterializationService {
|
|
|
630
727
|
const entries: Record<string, ManifestEntry> = {};
|
|
631
728
|
for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
|
|
632
729
|
if (entry.physicalTableName) {
|
|
730
|
+
// The build Manifest feeds a downstream persist's `FROM` verbatim,
|
|
731
|
+
// so a seeded upstream must carry the SAME quoting the builder
|
|
732
|
+
// CREATEd it with — else the downstream CREATE misses the
|
|
733
|
+
// case-preserved table on a case-folding engine. The seed keeps its
|
|
734
|
+
// logical (unquoted) name in `entries` (the committed manifest, in
|
|
735
|
+
// logical-name space); only the in-memory build Manifest is quoted.
|
|
633
736
|
manifest.update(sourceEntityId, {
|
|
634
|
-
tableName:
|
|
737
|
+
tableName: this.quoteSeedTablePath(
|
|
738
|
+
sourceEntityId,
|
|
739
|
+
entry.physicalTableName,
|
|
740
|
+
entry.connectionName,
|
|
741
|
+
connections,
|
|
742
|
+
),
|
|
635
743
|
});
|
|
636
744
|
}
|
|
637
745
|
entries[sourceEntityId] = entry;
|
|
@@ -733,7 +841,12 @@ export class MaterializationService {
|
|
|
733
841
|
}
|
|
734
842
|
|
|
735
843
|
// Make this table visible to downstream sources built later in this run.
|
|
736
|
-
|
|
844
|
+
// Record the SAME quoted path the CREATE used (not the logical name): the
|
|
845
|
+
// build Manifest is pasted into a downstream `FROM` verbatim, so on a
|
|
846
|
+
// case-folding engine an unquoted name would miss the case-preserved table
|
|
847
|
+
// just written. The returned entry (below) keeps the logical name for the
|
|
848
|
+
// committed manifest; only this in-memory build Manifest is quoted.
|
|
849
|
+
manifest.update(sourceEntityId, { tableName: quotedPhysical });
|
|
737
850
|
|
|
738
851
|
const durationMs = Math.round(performance.now() - startTime);
|
|
739
852
|
recordSourceBuildDuration(durationMs);
|
|
@@ -120,6 +120,12 @@ export function fakeSource(opts: {
|
|
|
120
120
|
freshnessSchedule?: FakeFreshnessSchedule;
|
|
121
121
|
/** Model-file-level (`##`) freshness default. */
|
|
122
122
|
modelFreshnessSchedule?: FakeFreshnessSchedule;
|
|
123
|
+
/**
|
|
124
|
+
* Spy on the args Malloy's SQL generation is handed — chiefly the
|
|
125
|
+
* `buildManifest` that resolves upstream persist references — so a test can
|
|
126
|
+
* assert what physical name a downstream build sees for its upstream.
|
|
127
|
+
*/
|
|
128
|
+
onGetSQL?: (sqlOpts: unknown) => void;
|
|
123
129
|
}): PersistSource {
|
|
124
130
|
const fields = opts.annotationFields;
|
|
125
131
|
return {
|
|
@@ -128,7 +134,10 @@ export function fakeSource(opts: {
|
|
|
128
134
|
connectionName: opts.connectionName ?? "duckdb",
|
|
129
135
|
dialectName: opts.dialectName ?? "duckdb",
|
|
130
136
|
makeBuildId: () => opts.sourceEntityId,
|
|
131
|
-
getSQL: () =>
|
|
137
|
+
getSQL: (sqlOpts?: unknown) => {
|
|
138
|
+
opts.onGetSQL?.(sqlOpts);
|
|
139
|
+
return opts.sql ?? "SELECT 1";
|
|
140
|
+
},
|
|
132
141
|
annotations: {
|
|
133
142
|
parseAsTag: () => ({
|
|
134
143
|
tag: fakeTag(fields, opts.freshnessSchedule),
|
package/src/service/package.ts
CHANGED
|
@@ -28,7 +28,10 @@ import {
|
|
|
28
28
|
ServiceUnavailableError,
|
|
29
29
|
} from "../errors";
|
|
30
30
|
import { formatDuration, logger } from "../logger";
|
|
31
|
-
import {
|
|
31
|
+
import {
|
|
32
|
+
recordBuildPlanComputeDuration,
|
|
33
|
+
recordManifestBindDegraded,
|
|
34
|
+
} from "../materialization_metrics";
|
|
32
35
|
import {
|
|
33
36
|
LOAD_DURATION_BUCKETS_MS,
|
|
34
37
|
recordPackageLoadPhases,
|
|
@@ -42,7 +45,9 @@ import {
|
|
|
42
45
|
} from "../storage/DatabaseInterface";
|
|
43
46
|
import { errMessage, ignoreDotfiles } from "../utils";
|
|
44
47
|
import { computePackageBuildPlan } from "./build_plan";
|
|
48
|
+
import { CronEvaluator } from "./cron_evaluator";
|
|
45
49
|
import { filterFreshManifest } from "./freshness";
|
|
50
|
+
import { isQuotedIdentifierPath, quoteManifestTablePath } from "./quoting";
|
|
46
51
|
import { Model } from "./model";
|
|
47
52
|
import { assertPersistNamesQuoted } from "./persist_annotation_validation";
|
|
48
53
|
|
|
@@ -865,6 +870,20 @@ export class Package {
|
|
|
865
870
|
}
|
|
866
871
|
}
|
|
867
872
|
|
|
873
|
+
// Rule 4: the cron must be a valid 5-field UNIX expression (no L/W/#/?
|
|
874
|
+
// extensions — see CronEvaluator). Enforced here so publish (strict),
|
|
875
|
+
// PATCH (strict), package load (warn), and the standalone scheduler all
|
|
876
|
+
// apply the identical rule — a garbage cron can no longer pass publish
|
|
877
|
+
// and then silently never arm.
|
|
878
|
+
if (packageSchedule && !new CronEvaluator().isValid(packageSchedule)) {
|
|
879
|
+
warnings.push(
|
|
880
|
+
`materialization.schedule in ${PACKAGE_MANIFEST_NAME} is not a valid ` +
|
|
881
|
+
`5-field UNIX cron: ${JSON.stringify(packageSchedule)}. Use ` +
|
|
882
|
+
`"minute hour day-of-month month day-of-week" (no L/W/#/? ` +
|
|
883
|
+
`extensions).`,
|
|
884
|
+
);
|
|
885
|
+
}
|
|
886
|
+
|
|
868
887
|
return warnings;
|
|
869
888
|
}
|
|
870
889
|
|
|
@@ -928,6 +947,71 @@ export class Package {
|
|
|
928
947
|
return this.malloyConfig.connections.lookupConnection(connectionName);
|
|
929
948
|
}
|
|
930
949
|
|
|
950
|
+
/**
|
|
951
|
+
* Quote each manifest entry's physical table path for its connection's
|
|
952
|
+
* dialect, mirroring the build side: the builder CREATEs the table with
|
|
953
|
+
* {@link quoteTablePath} (per-segment, case-preserved), so on a case-folding
|
|
954
|
+
* engine (Snowflake uppercases unquoted identifiers) the stored name is only
|
|
955
|
+
* reachable through the same quoting. Malloy pastes a manifest `tableName`
|
|
956
|
+
* into `FROM` verbatim by contract (a bare name means "let the engine
|
|
957
|
+
* fold"), so the case-preserving producer — us — must hand it the quoted
|
|
958
|
+
* form. Same module quotes CREATE and read: the two sides cannot drift.
|
|
959
|
+
*
|
|
960
|
+
* A name already carrying a quote character is passed through verbatim (it
|
|
961
|
+
* is already canonical SQL; control-plane-assigned names are sanitized to
|
|
962
|
+
* `[A-Za-z0-9_\-.]` and can never contain one).
|
|
963
|
+
*
|
|
964
|
+
* Two cases bind verbatim, and they are NOT the same signal:
|
|
965
|
+
* - No `connectionName`: a bare/engine-folding producer that never recorded
|
|
966
|
+
* a connection (simple_builder / malloy-cli). Expected and benign — the
|
|
967
|
+
* pre-change behavior, bound silently, nothing regresses.
|
|
968
|
+
* - `connectionName` present but unresolvable: a genuine misconfiguration
|
|
969
|
+
* (the connection was renamed/removed, or the manifest is out of sync with
|
|
970
|
+
* this package's config). This one entry is degraded — but the source
|
|
971
|
+
* would not serve regardless of quoting, since Malloy needs the connection
|
|
972
|
+
* to run any query against it, so we degrade just this entry rather than
|
|
973
|
+
* fail the whole package bind, and log at ERROR with a fix.
|
|
974
|
+
*/
|
|
975
|
+
private async quoteBoundTableNames(
|
|
976
|
+
entries: FreshnessManifest,
|
|
977
|
+
): Promise<FreshnessManifest> {
|
|
978
|
+
const out: FreshnessManifest = {};
|
|
979
|
+
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
980
|
+
let tableName = entry.tableName;
|
|
981
|
+
if (entry.connectionName && !isQuotedIdentifierPath(tableName)) {
|
|
982
|
+
try {
|
|
983
|
+
const connection = await this.getMalloyConnection(
|
|
984
|
+
entry.connectionName,
|
|
985
|
+
);
|
|
986
|
+
tableName = quoteManifestTablePath(
|
|
987
|
+
tableName,
|
|
988
|
+
connection.dialectName,
|
|
989
|
+
);
|
|
990
|
+
} catch (err) {
|
|
991
|
+
recordManifestBindDegraded();
|
|
992
|
+
logger.error(
|
|
993
|
+
`Manifest entry '${sourceEntityId}' names connection ` +
|
|
994
|
+
`'${entry.connectionName}', which this package cannot ` +
|
|
995
|
+
`resolve: binding its table path unquoted, so this source ` +
|
|
996
|
+
`will not serve on a case-folding engine (and queries ` +
|
|
997
|
+
`against it fail regardless, since Malloy needs the ` +
|
|
998
|
+
`connection to run them). Fix: ensure a connection named ` +
|
|
999
|
+
`'${entry.connectionName}' exists in this package's ` +
|
|
1000
|
+
`config, or rebuild the manifest against the current config.`,
|
|
1001
|
+
{
|
|
1002
|
+
packageName: this.packageName,
|
|
1003
|
+
sourceEntityId,
|
|
1004
|
+
connectionName: entry.connectionName,
|
|
1005
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1006
|
+
},
|
|
1007
|
+
);
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
out[sourceEntityId] = { ...entry, tableName };
|
|
1011
|
+
}
|
|
1012
|
+
return out;
|
|
1013
|
+
}
|
|
1014
|
+
|
|
931
1015
|
public getMalloyConfig(): MalloyConfig {
|
|
932
1016
|
return this.malloyConfig;
|
|
933
1017
|
}
|
|
@@ -956,6 +1040,11 @@ export class Package {
|
|
|
956
1040
|
* — the caller (manifest service) decides how to retry.
|
|
957
1041
|
*/
|
|
958
1042
|
public async reloadAllModels(entries: FreshnessManifest): Promise<void> {
|
|
1043
|
+
// Quote each bound physical name for its connection's dialect BEFORE any
|
|
1044
|
+
// projection: everything downstream (model hydration, the per-query
|
|
1045
|
+
// freshness gate, /compile, /status) reads the entries recorded here, so
|
|
1046
|
+
// this is the one place the write side's quoting is mirrored onto reads.
|
|
1047
|
+
entries = await this.quoteBoundTableNames(entries);
|
|
959
1048
|
// Models are hydrated against the tableName-only projection; the freshness
|
|
960
1049
|
// fields gate the serve path per query (via getFreshBuildManifest), not
|
|
961
1050
|
// model hydration.
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { afterEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import sinon from "sinon";
|
|
3
|
+
import type { Connection } from "@malloydata/malloy";
|
|
4
|
+
import { logger } from "../logger";
|
|
5
|
+
import type { FreshnessManifest } from "../storage/DatabaseInterface";
|
|
6
|
+
import { Model } from "./model";
|
|
7
|
+
import { Package } from "./package";
|
|
8
|
+
|
|
9
|
+
// Unit coverage for Package.quoteBoundTableNames — the bind-time step that
|
|
10
|
+
// mirrors the CREATE side's quoting onto manifest table paths (PR #904). The
|
|
11
|
+
// integration test proves quote chars appear in real DuckDB SQL, but DuckDB is
|
|
12
|
+
// not case-folding and never exhibited the bug; these tests pin the actual
|
|
13
|
+
// quoted contract per dialect and exercise every branch (dialect selection,
|
|
14
|
+
// no-connection passthrough, already-quoted passthrough, and the
|
|
15
|
+
// unresolvable-connection fallback) without needing a live warehouse. The
|
|
16
|
+
// quoting itself runs through the real `quoteTablePath`, so this is not a
|
|
17
|
+
// re-implementation of the oracle — it drives the shipping code.
|
|
18
|
+
describe("service/package quoteBoundTableNames", () => {
|
|
19
|
+
afterEach(() => sinon.restore());
|
|
20
|
+
|
|
21
|
+
function makePackage(): Package {
|
|
22
|
+
return new Package(
|
|
23
|
+
"testEnv",
|
|
24
|
+
"testPackage",
|
|
25
|
+
"/tmp/does-not-matter",
|
|
26
|
+
{ name: "testPackage", description: "Test package" },
|
|
27
|
+
[],
|
|
28
|
+
new Map<string, Model>(),
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Reach the private method the same way package.spec.ts reaches private
|
|
33
|
+
// statics — this is the smallest seam that drives the real branch logic.
|
|
34
|
+
function quote(
|
|
35
|
+
pkg: Package,
|
|
36
|
+
entries: FreshnessManifest,
|
|
37
|
+
): Promise<FreshnessManifest> {
|
|
38
|
+
// @ts-expect-error Accessing private method for testing
|
|
39
|
+
return pkg.quoteBoundTableNames(entries);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function stubDialect(pkg: Package, dialectName: string): void {
|
|
43
|
+
sinon
|
|
44
|
+
.stub(pkg, "getMalloyConnection")
|
|
45
|
+
.resolves({ dialectName } as Connection);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
it("double-quotes each path segment for a case-folding dialect (Snowflake)", async () => {
|
|
49
|
+
const pkg = makePackage();
|
|
50
|
+
stubDialect(pkg, "snowflake");
|
|
51
|
+
|
|
52
|
+
const out = await quote(pkg, {
|
|
53
|
+
s1: { tableName: "schema.order_summary__g0", connectionName: "sf" },
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
expect(out.s1.tableName).toBe('"schema"."order_summary__g0"');
|
|
57
|
+
// connectionName and any other fields survive the rewrite.
|
|
58
|
+
expect(out.s1.connectionName).toBe("sf");
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("double-quotes for other double-quote dialects (Postgres)", async () => {
|
|
62
|
+
const pkg = makePackage();
|
|
63
|
+
stubDialect(pkg, "postgres");
|
|
64
|
+
|
|
65
|
+
const out = await quote(pkg, {
|
|
66
|
+
s1: { tableName: "public.daily", connectionName: "pg" },
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
expect(out.s1.tableName).toBe('"public"."daily"');
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it("backticks each segment for a backtick dialect (BigQuery)", async () => {
|
|
73
|
+
const pkg = makePackage();
|
|
74
|
+
stubDialect(pkg, "standardsql");
|
|
75
|
+
|
|
76
|
+
const out = await quote(pkg, {
|
|
77
|
+
s1: { tableName: "my-proj.ds.events", connectionName: "bq" },
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
expect(out.s1.tableName).toBe("`my-proj`.`ds`.`events`");
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it("binds verbatim when the producer recorded no connectionName", async () => {
|
|
84
|
+
const pkg = makePackage();
|
|
85
|
+
const spy = sinon.stub(pkg, "getMalloyConnection");
|
|
86
|
+
|
|
87
|
+
const out = await quote(pkg, {
|
|
88
|
+
s1: { tableName: "schema.bare_builder_table" },
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
expect(out.s1.tableName).toBe("schema.bare_builder_table");
|
|
92
|
+
// No connection lookup attempted — this is the benign default path.
|
|
93
|
+
expect(spy.called).toBe(false);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it("passes an already-quoted name through without double-quoting", async () => {
|
|
97
|
+
const pkg = makePackage();
|
|
98
|
+
const spy = sinon.stub(pkg, "getMalloyConnection");
|
|
99
|
+
|
|
100
|
+
const out = await quote(pkg, {
|
|
101
|
+
s1: {
|
|
102
|
+
tableName: '"schema"."already_canonical"',
|
|
103
|
+
connectionName: "sf",
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
expect(out.s1.tableName).toBe('"schema"."already_canonical"');
|
|
108
|
+
expect(spy.called).toBe(false);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it("passes an already-backticked name through without re-quoting", async () => {
|
|
112
|
+
const pkg = makePackage();
|
|
113
|
+
const spy = sinon.stub(pkg, "getMalloyConnection");
|
|
114
|
+
|
|
115
|
+
const out = await quote(pkg, {
|
|
116
|
+
s1: { tableName: "`ds`.`events`", connectionName: "bq" },
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
expect(out.s1.tableName).toBe("`ds`.`events`");
|
|
120
|
+
expect(spy.called).toBe(false);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it("degrades one entry (unquoted) and logs an actionable error when its connection is unresolvable", async () => {
|
|
124
|
+
const pkg = makePackage();
|
|
125
|
+
sinon
|
|
126
|
+
.stub(pkg, "getMalloyConnection")
|
|
127
|
+
.rejects(new Error("no connection named 'gone'"));
|
|
128
|
+
const errSpy = sinon.stub(logger, "error");
|
|
129
|
+
|
|
130
|
+
const out = await quote(pkg, {
|
|
131
|
+
s1: { tableName: "schema.orphaned", connectionName: "gone" },
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
// The one entry degrades to the unquoted name rather than throwing out
|
|
135
|
+
// of the loop and failing the whole package bind.
|
|
136
|
+
expect(out.s1.tableName).toBe("schema.orphaned");
|
|
137
|
+
// ...but the misconfiguration is a distinct, loud, actionable signal —
|
|
138
|
+
// not folded silently into the benign no-connection default.
|
|
139
|
+
expect(errSpy.calledOnce).toBe(true);
|
|
140
|
+
// winston's error() is overloaded, so widen the recorded args to assert
|
|
141
|
+
// the (message, context) contract this branch emits.
|
|
142
|
+
const [message, ctx] = errSpy.firstCall.args as unknown as [
|
|
143
|
+
string,
|
|
144
|
+
Record<string, unknown>,
|
|
145
|
+
];
|
|
146
|
+
expect(message).toContain("gone");
|
|
147
|
+
expect(message).toContain("Fix:");
|
|
148
|
+
expect(ctx).toMatchObject({
|
|
149
|
+
sourceEntityId: "s1",
|
|
150
|
+
connectionName: "gone",
|
|
151
|
+
});
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it("quotes resolvable entries and degrades an unresolvable one in the same bind", async () => {
|
|
155
|
+
const pkg = makePackage();
|
|
156
|
+
const lookup = sinon.stub(pkg, "getMalloyConnection");
|
|
157
|
+
lookup
|
|
158
|
+
.withArgs("sf")
|
|
159
|
+
.resolves({ dialectName: "snowflake" } as Connection);
|
|
160
|
+
lookup.withArgs("gone").rejects(new Error("missing"));
|
|
161
|
+
sinon.stub(logger, "error");
|
|
162
|
+
|
|
163
|
+
const out = await quote(pkg, {
|
|
164
|
+
good: { tableName: "schema.kept", connectionName: "sf" },
|
|
165
|
+
bad: { tableName: "schema.orphaned", connectionName: "gone" },
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
expect(out.good.tableName).toBe('"schema"."kept"');
|
|
169
|
+
expect(out.bad.tableName).toBe("schema.orphaned");
|
|
170
|
+
});
|
|
171
|
+
});
|
|
@@ -209,6 +209,37 @@ source: f is duckdb.sql("SELECT 1 as x")
|
|
|
209
209
|
{ timeout: 30000 },
|
|
210
210
|
);
|
|
211
211
|
|
|
212
|
+
// ── Rule 4: the cron must be a valid 5-field UNIX expression ──────────
|
|
213
|
+
|
|
214
|
+
it(
|
|
215
|
+
"rejects an unparseable schedule cron",
|
|
216
|
+
async () => {
|
|
217
|
+
const pkg = await loadPackage(PLAIN_MODEL, {
|
|
218
|
+
scope: "version",
|
|
219
|
+
materialization: { schedule: "not a cron" },
|
|
220
|
+
});
|
|
221
|
+
const joined = pkg.formatInvalidPersistencePolicy();
|
|
222
|
+
expect(joined).toContain("valid");
|
|
223
|
+
expect(joined).toContain("UNIX cron");
|
|
224
|
+
},
|
|
225
|
+
{ timeout: 30000 },
|
|
226
|
+
);
|
|
227
|
+
|
|
228
|
+
it(
|
|
229
|
+
"rejects a cron-parser extension the UNIX grammar lacks (L)",
|
|
230
|
+
async () => {
|
|
231
|
+
// Guards config-parity: cron-parser accepts `0 6 L * *` but the control
|
|
232
|
+
// plane's UNIX parser does not, so publish must reject it too rather
|
|
233
|
+
// than let it silently never arm in production.
|
|
234
|
+
const pkg = await loadPackage(PLAIN_MODEL, {
|
|
235
|
+
scope: "version",
|
|
236
|
+
materialization: { schedule: "0 6 L * *" },
|
|
237
|
+
});
|
|
238
|
+
expect(pkg.formatInvalidPersistencePolicy()).toContain("UNIX cron");
|
|
239
|
+
},
|
|
240
|
+
{ timeout: 30000 },
|
|
241
|
+
);
|
|
242
|
+
|
|
212
243
|
// ── inert when nothing is declared ────────────────────────────────────
|
|
213
244
|
|
|
214
245
|
it(
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { describe, expect, it } from "bun:test";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
bareTableName,
|
|
4
|
+
isQuotedIdentifierPath,
|
|
5
|
+
quoteIdentifier,
|
|
6
|
+
quoteManifestTablePath,
|
|
7
|
+
quoteTablePath,
|
|
8
|
+
} from "./quoting";
|
|
3
9
|
|
|
4
10
|
describe("bareTableName", () => {
|
|
5
11
|
it("returns the segment after the last dot for a qualified name", () => {
|
|
@@ -77,3 +83,53 @@ describe("quoteTablePath", () => {
|
|
|
77
83
|
expect(quoteTablePath("t_ab12_v0", "postgres")).toBe('"t_ab12_v0"');
|
|
78
84
|
});
|
|
79
85
|
});
|
|
86
|
+
|
|
87
|
+
describe("isQuotedIdentifierPath", () => {
|
|
88
|
+
it("is false for a logical, unquoted path (the control-plane form)", () => {
|
|
89
|
+
expect(isQuotedIdentifierPath("schema.order_summary__g0")).toBe(false);
|
|
90
|
+
expect(isQuotedIdentifierPath("order_summary")).toBe(false);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("is true once any segment carries a dialect quote char", () => {
|
|
94
|
+
expect(isQuotedIdentifierPath('"schema"."order_summary"')).toBe(true);
|
|
95
|
+
expect(isQuotedIdentifierPath("`ds`.`events`")).toBe(true);
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
// quoteManifestTablePath is the single quoting authority both the serve-side
|
|
100
|
+
// bind and the build-side manifest route through. It runs the real
|
|
101
|
+
// quoteTablePath (not a re-implementation), so this pins the exact per-dialect
|
|
102
|
+
// contract that keeps read (a Malloy FROM) byte-identical to write (the CREATE
|
|
103
|
+
// DDL, which also quotes via quoteTablePath) on every dialect — case-folding or
|
|
104
|
+
// not.
|
|
105
|
+
describe("quoteManifestTablePath", () => {
|
|
106
|
+
it("double-quotes each segment on a case-folding double-quote dialect (Snowflake)", () => {
|
|
107
|
+
expect(
|
|
108
|
+
quoteManifestTablePath("schema.order_summary__g0", "snowflake"),
|
|
109
|
+
).toBe('"schema"."order_summary__g0"');
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
it("double-quotes on Postgres and DuckDB (unchanged for lowercase names, correct for mixed case)", () => {
|
|
113
|
+
expect(quoteManifestTablePath("public.daily", "postgres")).toBe(
|
|
114
|
+
'"public"."daily"',
|
|
115
|
+
);
|
|
116
|
+
expect(quoteManifestTablePath("main.orders_mz", "duckdb")).toBe(
|
|
117
|
+
'"main"."orders_mz"',
|
|
118
|
+
);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it("backticks each segment on BigQuery (standardsql) — required for hyphenated ids", () => {
|
|
122
|
+
expect(quoteManifestTablePath("my-proj.ds.events", "standardsql")).toBe(
|
|
123
|
+
"`my-proj`.`ds`.`events`",
|
|
124
|
+
);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("passes an already-quoted name through unchanged (any dialect)", () => {
|
|
128
|
+
expect(quoteManifestTablePath('"schema"."already"', "snowflake")).toBe(
|
|
129
|
+
'"schema"."already"',
|
|
130
|
+
);
|
|
131
|
+
expect(quoteManifestTablePath("`ds`.`events`", "standardsql")).toBe(
|
|
132
|
+
"`ds`.`events`",
|
|
133
|
+
);
|
|
134
|
+
});
|
|
135
|
+
});
|