@malloy-publisher/server 0.0.161 → 0.0.165
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +78 -0
- package/dist/app/assets/HomePage-QekMXs8r.js +1 -0
- package/dist/app/assets/{MainPage-CzAy2LSX.js → MainPage-DAyUfYba.js} +1 -1
- package/dist/app/assets/{ModelPage-uZf0eDBu.js → ModelPage-CrMryV1s.js} +1 -1
- package/dist/app/assets/PackagePage-DDaABD2A.js +1 -0
- package/dist/app/assets/ProjectPage-FAYUFGhL.js +1 -0
- package/dist/app/assets/RouteError-BKYctANX.js +1 -0
- package/dist/app/assets/{WorkbookPage-CBJ0x84n.js → WorkbookPage-DZEVYGW3.js} +1 -1
- package/dist/app/assets/{index-BoduKqUa.js → index-BvVmB5sv.js} +98 -98
- package/dist/app/assets/{index-D_8e14NE.js → index-CsC07BYd.js} +1 -1
- package/dist/app/assets/{index-Bl1fe3A7.js → index-DWhjtyBB.js} +1 -1
- package/dist/app/assets/{index.umd-DRsjT6d4.js → index.umd-DvM-lTQa.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.js +4 -2
- package/dist/server.js +150 -18
- package/k6-tests/load-test/load-test.ts +30 -0
- package/package.json +5 -1
- package/src/health.ts +181 -0
- package/src/logger.ts +9 -1
- package/src/server.ts +42 -1
- package/src/service/model.spec.ts +5 -5
- package/src/service/model.ts +56 -19
- package/src/service/project_store.ts +5 -0
- package/dist/app/assets/HomePage-CgSTNEQW.js +0 -1
- package/dist/app/assets/PackagePage-BHvPFsmZ.js +0 -1
- package/dist/app/assets/ProjectPage-ndnv5nGp.js +0 -1
- package/dist/app/assets/RouteError-CZC9OOKh.js +0 -1
package/dist/app/index.html
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
href="https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap"
|
|
13
13
|
/>
|
|
14
14
|
<title>Malloy Publisher</title>
|
|
15
|
-
<script type="module" crossorigin src="/assets/index-
|
|
15
|
+
<script type="module" crossorigin src="/assets/index-BvVmB5sv.js"></script>
|
|
16
16
|
<link rel="stylesheet" crossorigin href="/assets/index-CMlGQMcl.css">
|
|
17
17
|
</head>
|
|
18
18
|
<body>
|
package/dist/instrumentation.js
CHANGED
|
@@ -106335,7 +106335,7 @@ var require_styles = __commonJS((exports2, module2) => {
|
|
|
106335
106335
|
// ../../node_modules/@colors/colors/lib/system/has-flag.js
|
|
106336
106336
|
var require_has_flag2 = __commonJS((exports2, module2) => {
|
|
106337
106337
|
module2.exports = function(flag, argv) {
|
|
106338
|
-
argv = argv || process.argv
|
|
106338
|
+
argv = argv || process.argv;
|
|
106339
106339
|
var terminatorPos = argv.indexOf("--");
|
|
106340
106340
|
var prefix = /^-{1,2}/.test(flag) ? "" : "--";
|
|
106341
106341
|
var pos = argv.indexOf(prefix + flag);
|
|
@@ -116569,7 +116569,9 @@ var loggerMiddleware = (req, res, next) => {
|
|
|
116569
116569
|
if (traceId) {
|
|
116570
116570
|
logMetadata.traceId = traceId;
|
|
116571
116571
|
}
|
|
116572
|
-
|
|
116572
|
+
if (req.url !== "/metrics" && req.url !== "/health" && req.url !== "/health/liveness" && req.url !== "/health/readiness") {
|
|
116573
|
+
logger.info(`${req.method} ${req.url}`, logMetadata);
|
|
116574
|
+
}
|
|
116573
116575
|
});
|
|
116574
116576
|
next();
|
|
116575
116577
|
};
|
package/dist/server.js
CHANGED
|
@@ -33301,7 +33301,7 @@ var require_styles = __commonJS((exports2, module2) => {
|
|
|
33301
33301
|
// ../../node_modules/@colors/colors/lib/system/has-flag.js
|
|
33302
33302
|
var require_has_flag2 = __commonJS((exports2, module2) => {
|
|
33303
33303
|
module2.exports = function(flag, argv) {
|
|
33304
|
-
argv = argv || process.argv
|
|
33304
|
+
argv = argv || process.argv;
|
|
33305
33305
|
var terminatorPos = argv.indexOf("--");
|
|
33306
33306
|
var prefix = /^-{1,2}/.test(flag) ? "" : "--";
|
|
33307
33307
|
var pos = argv.indexOf(prefix + flag);
|
|
@@ -124960,7 +124960,9 @@ var loggerMiddleware = (req, res, next) => {
|
|
|
124960
124960
|
if (traceId) {
|
|
124961
124961
|
logMetadata.traceId = traceId;
|
|
124962
124962
|
}
|
|
124963
|
-
|
|
124963
|
+
if (req.url !== "/metrics" && req.url !== "/health" && req.url !== "/health/liveness" && req.url !== "/health/readiness") {
|
|
124964
|
+
logger.info(`${req.method} ${req.url}`, logMetadata);
|
|
124965
|
+
}
|
|
124964
124966
|
});
|
|
124965
124967
|
next();
|
|
124966
124968
|
};
|
|
@@ -135906,6 +135908,96 @@ var getProcessedPublisherConfig = (serverRoot) => {
|
|
|
135906
135908
|
};
|
|
135907
135909
|
};
|
|
135908
135910
|
|
|
135911
|
+
// src/health.ts
|
|
135912
|
+
var operationalState = "initializing";
|
|
135913
|
+
var ready = false;
|
|
135914
|
+
var preGracefulShutdownCompleted = false;
|
|
135915
|
+
function getOperationalState() {
|
|
135916
|
+
return operationalState;
|
|
135917
|
+
}
|
|
135918
|
+
function markReady() {
|
|
135919
|
+
if (operationalState !== "draining") {
|
|
135920
|
+
operationalState = "serving";
|
|
135921
|
+
ready = true;
|
|
135922
|
+
logger.info("Service marked as ready");
|
|
135923
|
+
} else {
|
|
135924
|
+
logger.error("Service is already draining - cannot mark as ready");
|
|
135925
|
+
}
|
|
135926
|
+
}
|
|
135927
|
+
function markNotReady() {
|
|
135928
|
+
ready = false;
|
|
135929
|
+
logger.info("Service marked as not ready - readiness probe will fail");
|
|
135930
|
+
}
|
|
135931
|
+
function registerSignalHandlers(server, mcpServer, shutdownDrainDurationSeconds = 0, shutdownGracefulCloseTimeoutSeconds = 0) {
|
|
135932
|
+
process.once("SIGTERM", async () => {
|
|
135933
|
+
logger.info("========== SIGTERM RECEIVED ==========");
|
|
135934
|
+
markNotReady();
|
|
135935
|
+
operationalState = "draining";
|
|
135936
|
+
logger.info(`Service entering draining state for ${shutdownDrainDurationSeconds} seconds before closing servers...`);
|
|
135937
|
+
await new Promise((resolve3) => setTimeout(() => {
|
|
135938
|
+
preGracefulShutdownCompleted = true;
|
|
135939
|
+
resolve3(true);
|
|
135940
|
+
}, shutdownDrainDurationSeconds * 1000));
|
|
135941
|
+
const closeServer = (server2, name) => new Promise((resolve3) => {
|
|
135942
|
+
if (server2 && server2.listening) {
|
|
135943
|
+
server2.close((err) => {
|
|
135944
|
+
if (err) {
|
|
135945
|
+
logger.error(`${name} close error:`, err);
|
|
135946
|
+
} else {
|
|
135947
|
+
logger.info(`${name} closed`);
|
|
135948
|
+
}
|
|
135949
|
+
resolve3();
|
|
135950
|
+
});
|
|
135951
|
+
} else {
|
|
135952
|
+
resolve3();
|
|
135953
|
+
}
|
|
135954
|
+
});
|
|
135955
|
+
await Promise.all([
|
|
135956
|
+
closeServer(server, "Main server"),
|
|
135957
|
+
closeServer(mcpServer, "MCP server")
|
|
135958
|
+
]);
|
|
135959
|
+
try {
|
|
135960
|
+
logger.close();
|
|
135961
|
+
} catch (_error) {}
|
|
135962
|
+
if (shutdownGracefulCloseTimeoutSeconds > 0) {
|
|
135963
|
+
logger.info(`Waiting ${shutdownGracefulCloseTimeoutSeconds} seconds after server close before exit...`);
|
|
135964
|
+
await new Promise((resolve3) => setTimeout(resolve3, shutdownGracefulCloseTimeoutSeconds * 1000));
|
|
135965
|
+
}
|
|
135966
|
+
process.exit(0);
|
|
135967
|
+
});
|
|
135968
|
+
}
|
|
135969
|
+
function drainingGuard(req, res, next) {
|
|
135970
|
+
if (operationalState === "draining" && preGracefulShutdownCompleted && !req.path.startsWith("/health") && !req.path.startsWith("/metrics")) {
|
|
135971
|
+
res.status(503).json({
|
|
135972
|
+
message: "Service is draining",
|
|
135973
|
+
details: "The service is shutting down and hence is not available."
|
|
135974
|
+
});
|
|
135975
|
+
return;
|
|
135976
|
+
}
|
|
135977
|
+
next();
|
|
135978
|
+
}
|
|
135979
|
+
function registerHealthEndpoints(app) {
|
|
135980
|
+
app.get("/health", (_req, res) => {
|
|
135981
|
+
res.status(200).json({
|
|
135982
|
+
status: "UP",
|
|
135983
|
+
components: {
|
|
135984
|
+
livenessState: { status: "UP" },
|
|
135985
|
+
readinessState: { status: ready ? "UP" : "DOWN" }
|
|
135986
|
+
},
|
|
135987
|
+
groups: ["livenessState", "readinessState"]
|
|
135988
|
+
});
|
|
135989
|
+
});
|
|
135990
|
+
app.get("/health/liveness", (_req, res) => {
|
|
135991
|
+
res.status(200).json({ status: "UP" });
|
|
135992
|
+
});
|
|
135993
|
+
app.get("/health/readiness", (_req, res) => {
|
|
135994
|
+
const isReady = ready;
|
|
135995
|
+
res.status(isReady ? 200 : 503).json({
|
|
135996
|
+
status: isReady ? "UP" : "DOWN"
|
|
135997
|
+
});
|
|
135998
|
+
});
|
|
135999
|
+
}
|
|
136000
|
+
|
|
135909
136001
|
// src/storage/duckdb/DuckDBConnection.ts
|
|
135910
136002
|
var duckdb = __toESM(require("duckdb"));
|
|
135911
136003
|
var path4 = __toESM(require("path"));
|
|
@@ -136747,28 +136839,51 @@ class Model {
|
|
|
136747
136839
|
async getQueryResults(sourceName, queryName, query) {
|
|
136748
136840
|
const startTime = performance.now();
|
|
136749
136841
|
if (this.compilationError) {
|
|
136750
|
-
|
|
136842
|
+
if (this.compilationError instanceof import_malloy2.MalloyError || this.compilationError instanceof ModelCompilationError) {
|
|
136843
|
+
throw this.compilationError;
|
|
136844
|
+
}
|
|
136845
|
+
throw new BadRequestError(`Model compilation failed: ${this.compilationError.message}`);
|
|
136751
136846
|
}
|
|
136752
136847
|
let runnable;
|
|
136753
136848
|
if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
|
|
136754
136849
|
throw new BadRequestError("Model has no queryable entities.");
|
|
136755
|
-
|
|
136756
|
-
|
|
136850
|
+
try {
|
|
136851
|
+
if (!sourceName && !queryName && query) {
|
|
136852
|
+
runnable = this.modelMaterializer.loadQuery(`
|
|
136757
136853
|
` + query);
|
|
136758
|
-
|
|
136759
|
-
|
|
136854
|
+
} else if (queryName && !query) {
|
|
136855
|
+
runnable = this.modelMaterializer.loadQuery(`
|
|
136760
136856
|
run: ${sourceName ? sourceName + "->" : ""}${queryName}`);
|
|
136761
|
-
|
|
136762
|
-
|
|
136763
|
-
|
|
136764
|
-
|
|
136765
|
-
|
|
136766
|
-
|
|
136767
|
-
|
|
136768
|
-
|
|
136769
|
-
|
|
136857
|
+
} else {
|
|
136858
|
+
const endTime2 = performance.now();
|
|
136859
|
+
const executionTime2 = endTime2 - startTime;
|
|
136860
|
+
this.queryExecutionHistogram.record(executionTime2, {
|
|
136861
|
+
"malloy.model.path": this.modelPath,
|
|
136862
|
+
"malloy.model.query.name": queryName,
|
|
136863
|
+
"malloy.model.query.source": sourceName,
|
|
136864
|
+
"malloy.model.query.query": query,
|
|
136865
|
+
"malloy.model.query.status": "error"
|
|
136866
|
+
});
|
|
136867
|
+
throw new BadRequestError("Invalid query request. (Query AND !sourceName) OR (queryName AND sourceName) must be defined.");
|
|
136868
|
+
}
|
|
136869
|
+
} catch (error) {
|
|
136870
|
+
if (error instanceof BadRequestError) {
|
|
136871
|
+
throw error;
|
|
136872
|
+
}
|
|
136873
|
+
if (error instanceof import_malloy2.MalloyError) {
|
|
136874
|
+
throw error;
|
|
136875
|
+
}
|
|
136876
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
136877
|
+
logger.error("Query parsing error", {
|
|
136878
|
+
error,
|
|
136879
|
+
errorMessage,
|
|
136880
|
+
projectName: this.packageName,
|
|
136881
|
+
modelPath: this.modelPath,
|
|
136882
|
+
query,
|
|
136883
|
+
queryName,
|
|
136884
|
+
sourceName
|
|
136770
136885
|
});
|
|
136771
|
-
throw new BadRequestError(
|
|
136886
|
+
throw new BadRequestError(`Invalid query: ${errorMessage}`);
|
|
136772
136887
|
}
|
|
136773
136888
|
const rowLimit = (await runnable.getPreparedResult()).resultExplore.limit || ROW_LIMIT;
|
|
136774
136889
|
const endTime = performance.now();
|
|
@@ -137749,9 +137864,11 @@ class ProjectStore {
|
|
|
137749
137864
|
}
|
|
137750
137865
|
}
|
|
137751
137866
|
this.isInitialized = true;
|
|
137867
|
+
markReady();
|
|
137752
137868
|
const initializationDuration = performance.now() - initialTime;
|
|
137753
137869
|
logger.info(`Project store successfully initialized in ${formatDuration(initializationDuration)}`);
|
|
137754
137870
|
} catch (error) {
|
|
137871
|
+
markNotReady();
|
|
137755
137872
|
logger.error("Error initializing project store", { error });
|
|
137756
137873
|
process.exit(1);
|
|
137757
137874
|
}
|
|
@@ -137984,7 +138101,8 @@ class ProjectStore {
|
|
|
137984
138101
|
timestamp: Date.now(),
|
|
137985
138102
|
projects: [],
|
|
137986
138103
|
initialized: this.isInitialized,
|
|
137987
|
-
frozenConfig: isPublisherConfigFrozen(this.serverRootPath)
|
|
138104
|
+
frozenConfig: isPublisherConfigFrozen(this.serverRootPath),
|
|
138105
|
+
operationalState: getOperationalState()
|
|
137988
138106
|
};
|
|
137989
138107
|
const projects = await this.listProjects(true);
|
|
137990
138108
|
await Promise.all(projects.map(async (project) => {
|
|
@@ -142457,6 +142575,12 @@ function parseArgs() {
|
|
|
142457
142575
|
} else if (arg === "--mcp_port" && args[i + 1]) {
|
|
142458
142576
|
process.env.MCP_PORT = args[i + 1];
|
|
142459
142577
|
i++;
|
|
142578
|
+
} else if (arg === "--shutdown_drain_duration_seconds" && args[i + 1]) {
|
|
142579
|
+
process.env.SHUTDOWN_DRAIN_DURATION_SECONDS = args[i + 1];
|
|
142580
|
+
i++;
|
|
142581
|
+
} else if (arg === "--shutdown_graceful_close_timeout_seconds" && args[i + 1]) {
|
|
142582
|
+
process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = args[i + 1];
|
|
142583
|
+
i++;
|
|
142460
142584
|
} else if (arg === "--init") {
|
|
142461
142585
|
process.env.INITIALIZE_STORAGE = "true";
|
|
142462
142586
|
} else if (arg === "--help" || arg === "-h") {
|
|
@@ -142469,6 +142593,9 @@ function parseArgs() {
|
|
|
142469
142593
|
console.log(" --host <string> Host to bind the server to (default: localhost)");
|
|
142470
142594
|
console.log(" --server_root <path> Root directory to serve files from (default: .)");
|
|
142471
142595
|
console.log(" --mcp_port <number> Port for MCP server (default: 4040)");
|
|
142596
|
+
console.log(" --shutdown_drain_duration_seconds <number> Time in seconds to keep service in draining state before closing servers (default: 0)");
|
|
142597
|
+
console.log(" --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)");
|
|
142598
|
+
console.log(" --init Initialize the storage (default: false)");
|
|
142472
142599
|
console.log(" --help, -h Show this help message");
|
|
142473
142600
|
process.exit(0);
|
|
142474
142601
|
}
|
|
@@ -142479,6 +142606,8 @@ var PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
|
|
|
142479
142606
|
var PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
|
|
142480
142607
|
var MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
142481
142608
|
var MCP_ENDPOINT = "/mcp";
|
|
142609
|
+
var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
|
|
142610
|
+
var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
|
|
142482
142611
|
var ROOT;
|
|
142483
142612
|
if (require.main) {
|
|
142484
142613
|
ROOT = path10.join(path10.dirname(require.main.filename), "app");
|
|
@@ -142578,6 +142707,8 @@ app.use(import_cors.default({
|
|
|
142578
142707
|
credentials: true
|
|
142579
142708
|
}));
|
|
142580
142709
|
app.use(bodyParser.json());
|
|
142710
|
+
registerHealthEndpoints(app);
|
|
142711
|
+
app.use(drainingGuard);
|
|
142581
142712
|
app.get(`${API_PREFIX2}/status`, async (_req, res) => {
|
|
142582
142713
|
try {
|
|
142583
142714
|
const status = await projectStore.getStatus();
|
|
@@ -142974,3 +143105,4 @@ var mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
|
|
|
142974
143105
|
mcpServer.timeout = 600000;
|
|
142975
143106
|
mcpServer.keepAliveTimeout = 600000;
|
|
142976
143107
|
mcpServer.headersTimeout = 600000;
|
|
143108
|
+
registerSignalHandlers(mainServer, mcpServer, SHUTDOWN_DRAIN_DURATION_SECONDS, SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS);
|
|
@@ -440,6 +440,11 @@ export const loadTest: TestPreset = {
|
|
|
440
440
|
check(response.response, {
|
|
441
441
|
"get connection request successful": (r) => r.status === 200,
|
|
442
442
|
});
|
|
443
|
+
if (response.response.status !== 200) {
|
|
444
|
+
logger.error(
|
|
445
|
+
`get connection failed: status=${response.response.status}, body=${response.response.body}`,
|
|
446
|
+
);
|
|
447
|
+
}
|
|
443
448
|
});
|
|
444
449
|
}
|
|
445
450
|
});
|
|
@@ -590,6 +595,11 @@ export const loadTest: TestPreset = {
|
|
|
590
595
|
"post query data request successful": (r) =>
|
|
591
596
|
r.status === 200,
|
|
592
597
|
});
|
|
598
|
+
if (response.response.status !== 200) {
|
|
599
|
+
logger.error(
|
|
600
|
+
`post query data failed: status=${response.response.status}, body=${response.response.body}`,
|
|
601
|
+
);
|
|
602
|
+
}
|
|
593
603
|
} catch (error) {
|
|
594
604
|
logger.warn(`Failed to post query data: ${error}`);
|
|
595
605
|
}
|
|
@@ -619,6 +629,11 @@ export const loadTest: TestPreset = {
|
|
|
619
629
|
check(schemasResponse.response, {
|
|
620
630
|
"list schemas request successful": (r) => r.status === 200,
|
|
621
631
|
});
|
|
632
|
+
if (schemasResponse.response.status !== 200) {
|
|
633
|
+
logger.error(
|
|
634
|
+
`list schemas failed: status=${schemasResponse.response.status}, body=${schemasResponse.response.body}`,
|
|
635
|
+
);
|
|
636
|
+
}
|
|
622
637
|
|
|
623
638
|
sleep(0.1);
|
|
624
639
|
|
|
@@ -646,6 +661,11 @@ export const loadTest: TestPreset = {
|
|
|
646
661
|
"list tables request successful": (r) =>
|
|
647
662
|
r.status === 200,
|
|
648
663
|
});
|
|
664
|
+
if (tablesResponse.response.status !== 200) {
|
|
665
|
+
logger.error(
|
|
666
|
+
`list tables failed: status=${tablesResponse.response.status}, body=${tablesResponse.response.body}`,
|
|
667
|
+
);
|
|
668
|
+
}
|
|
649
669
|
|
|
650
670
|
sleep(0.1);
|
|
651
671
|
|
|
@@ -717,6 +737,11 @@ export const loadTest: TestPreset = {
|
|
|
717
737
|
"post sql source request successful": (r) =>
|
|
718
738
|
r.status === 200,
|
|
719
739
|
});
|
|
740
|
+
if (response.response.status !== 200) {
|
|
741
|
+
logger.error(
|
|
742
|
+
`post sql source failed: status=${response.response.status}, body=${response.response.body}`,
|
|
743
|
+
);
|
|
744
|
+
}
|
|
720
745
|
} catch (error) {
|
|
721
746
|
logger.warn(`Failed to post sql source: ${error}`);
|
|
722
747
|
}
|
|
@@ -739,6 +764,11 @@ export const loadTest: TestPreset = {
|
|
|
739
764
|
"post temporary table request successful": (r) =>
|
|
740
765
|
r.status === 200,
|
|
741
766
|
});
|
|
767
|
+
if (response.response.status !== 200) {
|
|
768
|
+
logger.error(
|
|
769
|
+
`post temporary table failed: status=${response.response.status}, body=${response.response.body}`,
|
|
770
|
+
);
|
|
771
|
+
}
|
|
742
772
|
} catch (error) {
|
|
743
773
|
logger.warn(`Failed to post temporary table: ${error}`);
|
|
744
774
|
}
|
package/package.json
CHANGED
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@malloy-publisher/server",
|
|
3
3
|
"description": "Malloy Publisher Server",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.165",
|
|
5
5
|
"main": "dist/server.js",
|
|
6
6
|
"bin": {
|
|
7
7
|
"malloy-publisher": "dist/server.js"
|
|
8
8
|
},
|
|
9
|
+
"repository": {
|
|
10
|
+
"type": "git",
|
|
11
|
+
"url": "https://github.com/malloydata/publisher.git"
|
|
12
|
+
},
|
|
9
13
|
"publishConfig": {
|
|
10
14
|
"access": "public"
|
|
11
15
|
},
|
package/src/health.ts
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Health check endpoints and graceful shutdown handling.
|
|
3
|
+
*
|
|
4
|
+
* Provides Kubernetes-compatible health probes:
|
|
5
|
+
* - /health - Overall health status
|
|
6
|
+
* - /health/liveness - Always returns 200 (process is alive)
|
|
7
|
+
* - /health/readiness - Returns 200 when ready, 503 when not ready
|
|
8
|
+
*
|
|
9
|
+
* On SIGTERM, the service enters draining state to allow in-flight requests
|
|
10
|
+
* to complete before shutdown.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { Express, NextFunction, Request, Response } from "express";
|
|
14
|
+
import { Server } from "http";
|
|
15
|
+
import { components } from "./api";
|
|
16
|
+
import { logger } from "./logger";
|
|
17
|
+
export type OperationalState =
|
|
18
|
+
components["schemas"]["ServerStatus"]["operationalState"];
|
|
19
|
+
|
|
20
|
+
let operationalState: OperationalState = "initializing" as OperationalState;
|
|
21
|
+
let ready: boolean = false;
|
|
22
|
+
let preGracefulShutdownCompleted: boolean = false;
|
|
23
|
+
/**
|
|
24
|
+
* Returns the current operational state of the service.
|
|
25
|
+
*/
|
|
26
|
+
export function getOperationalState(): OperationalState {
|
|
27
|
+
return operationalState;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Marks the service as ready to serve traffic.
|
|
32
|
+
*/
|
|
33
|
+
export function markReady(): void {
|
|
34
|
+
if (operationalState !== "draining") {
|
|
35
|
+
operationalState = "serving";
|
|
36
|
+
ready = true;
|
|
37
|
+
logger.info("Service marked as ready");
|
|
38
|
+
} else {
|
|
39
|
+
logger.error("Service is already draining - cannot mark as ready");
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Marks the service as not ready (readiness probe will return 503).
|
|
45
|
+
*/
|
|
46
|
+
export function markNotReady(): void {
|
|
47
|
+
ready = false;
|
|
48
|
+
logger.info("Service marked as not ready - readiness probe will fail");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Registers SIGTERM handler for graceful shutdown.
|
|
53
|
+
*
|
|
54
|
+
* Shutdown sequence:
|
|
55
|
+
* 1. Marks service as not ready (readiness probe returns 503) and enters draining state
|
|
56
|
+
* 2. Waits shutdownDrainDurationSeconds to allow in-flight requests to complete
|
|
57
|
+
* 3. Sets preGracefulShutdownCompleted flag (enables drainingGuard middleware to reject new requests)
|
|
58
|
+
* 4. Closes main server and MCP server (stops accepting new connections)
|
|
59
|
+
* 5. Closes logger
|
|
60
|
+
* 6. Waits shutdownGracefulCloseTimeoutSeconds (if > 0) for final cleanup
|
|
61
|
+
* 7. Exits process
|
|
62
|
+
*
|
|
63
|
+
* Note: drainingGuard only rejects requests after step 3 completes. During step 2,
|
|
64
|
+
* the service is draining but still accepts requests (readiness probe returns 503).
|
|
65
|
+
*
|
|
66
|
+
* @param server - Main HTTP server instance
|
|
67
|
+
* @param mcpServer - MCP server instance
|
|
68
|
+
* @param shutdownDrainDurationSeconds - Duration in seconds to wait before closing servers
|
|
69
|
+
* @param shutdownGracefulCloseTimeoutSeconds - Duration in seconds to wait after closing servers before exit
|
|
70
|
+
*/
|
|
71
|
+
export function registerSignalHandlers(
|
|
72
|
+
server: Server,
|
|
73
|
+
mcpServer: Server,
|
|
74
|
+
shutdownDrainDurationSeconds: number = 0,
|
|
75
|
+
shutdownGracefulCloseTimeoutSeconds: number = 0,
|
|
76
|
+
): void {
|
|
77
|
+
// Keep the process alive on SIGTERM — do not close the server.
|
|
78
|
+
// K8s will SIGKILL after terminationGracePeriodSeconds (which cannot be caught).
|
|
79
|
+
process.once("SIGTERM", async () => {
|
|
80
|
+
logger.info("========== SIGTERM RECEIVED ==========");
|
|
81
|
+
markNotReady();
|
|
82
|
+
operationalState = "draining" as OperationalState;
|
|
83
|
+
logger.info(
|
|
84
|
+
`Service entering draining state for ${shutdownDrainDurationSeconds} seconds before closing servers...`,
|
|
85
|
+
);
|
|
86
|
+
|
|
87
|
+
await new Promise((resolve) =>
|
|
88
|
+
setTimeout(() => {
|
|
89
|
+
preGracefulShutdownCompleted = true;
|
|
90
|
+
resolve(true);
|
|
91
|
+
}, shutdownDrainDurationSeconds * 1000),
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
const closeServer = (server: Server, name: string) =>
|
|
95
|
+
new Promise<void>((resolve) => {
|
|
96
|
+
if (server && server.listening) {
|
|
97
|
+
server.close((err) => {
|
|
98
|
+
if (err) {
|
|
99
|
+
logger.error(`${name} close error:`, err);
|
|
100
|
+
} else {
|
|
101
|
+
logger.info(`${name} closed`);
|
|
102
|
+
}
|
|
103
|
+
resolve();
|
|
104
|
+
});
|
|
105
|
+
} else {
|
|
106
|
+
resolve();
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
await Promise.all([
|
|
111
|
+
closeServer(server, "Main server"),
|
|
112
|
+
closeServer(mcpServer, "MCP server"),
|
|
113
|
+
]);
|
|
114
|
+
|
|
115
|
+
try {
|
|
116
|
+
logger.close();
|
|
117
|
+
} catch (_error) {
|
|
118
|
+
/* do nothing */
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (shutdownGracefulCloseTimeoutSeconds > 0) {
|
|
122
|
+
logger.info(
|
|
123
|
+
`Waiting ${shutdownGracefulCloseTimeoutSeconds} seconds after server close before exit...`,
|
|
124
|
+
);
|
|
125
|
+
await new Promise((resolve) =>
|
|
126
|
+
setTimeout(resolve, shutdownGracefulCloseTimeoutSeconds * 1000),
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
process.exit(0);
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Middleware that returns 503 for non-health and metrics requests when service is draining.
|
|
134
|
+
* Must be registered before application routes.
|
|
135
|
+
*/
|
|
136
|
+
export function drainingGuard(
|
|
137
|
+
req: Request,
|
|
138
|
+
res: Response,
|
|
139
|
+
next: NextFunction,
|
|
140
|
+
): void {
|
|
141
|
+
if (
|
|
142
|
+
operationalState === "draining" &&
|
|
143
|
+
preGracefulShutdownCompleted &&
|
|
144
|
+
!req.path.startsWith("/health") &&
|
|
145
|
+
!req.path.startsWith("/metrics")
|
|
146
|
+
) {
|
|
147
|
+
res.status(503).json({
|
|
148
|
+
message: "Service is draining",
|
|
149
|
+
details: "The service is shutting down and hence is not available.",
|
|
150
|
+
});
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
next();
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Registers health check endpoints
|
|
158
|
+
*/
|
|
159
|
+
export function registerHealthEndpoints(app: Express): void {
|
|
160
|
+
app.get("/health", (_req: Request, res: Response) => {
|
|
161
|
+
res.status(200).json({
|
|
162
|
+
status: "UP",
|
|
163
|
+
components: {
|
|
164
|
+
livenessState: { status: "UP" },
|
|
165
|
+
readinessState: { status: ready ? "UP" : "DOWN" },
|
|
166
|
+
},
|
|
167
|
+
groups: ["livenessState", "readinessState"],
|
|
168
|
+
});
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
app.get("/health/liveness", (_req: Request, res: Response) => {
|
|
172
|
+
res.status(200).json({ status: "UP" });
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
app.get("/health/readiness", (_req: Request, res: Response) => {
|
|
176
|
+
const isReady = ready;
|
|
177
|
+
res.status(isReady ? 200 : 503).json({
|
|
178
|
+
status: isReady ? "UP" : "DOWN",
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
}
|
package/src/logger.ts
CHANGED
|
@@ -119,7 +119,15 @@ export const loggerMiddleware: RequestHandler = (req, res, next) => {
|
|
|
119
119
|
logMetadata.traceId = traceId;
|
|
120
120
|
}
|
|
121
121
|
|
|
122
|
-
|
|
122
|
+
// Skip logging for metrics and health endpoints to reduce log noise
|
|
123
|
+
if (
|
|
124
|
+
req.url !== "/metrics" &&
|
|
125
|
+
req.url !== "/health" &&
|
|
126
|
+
req.url !== "/health/liveness" &&
|
|
127
|
+
req.url !== "/health/readiness"
|
|
128
|
+
) {
|
|
129
|
+
logger.info(`${req.method} ${req.url}`, logMetadata);
|
|
130
|
+
}
|
|
123
131
|
});
|
|
124
132
|
next();
|
|
125
133
|
};
|
package/src/server.ts
CHANGED
|
@@ -13,10 +13,14 @@ import { PackageController } from "./controller/package.controller";
|
|
|
13
13
|
import { QueryController } from "./controller/query.controller";
|
|
14
14
|
import { WatchModeController } from "./controller/watch-mode.controller";
|
|
15
15
|
import { internalErrorToHttpError, NotImplementedError } from "./errors";
|
|
16
|
+
import {
|
|
17
|
+
drainingGuard,
|
|
18
|
+
registerHealthEndpoints,
|
|
19
|
+
registerSignalHandlers,
|
|
20
|
+
} from "./health";
|
|
16
21
|
import { logger, loggerMiddleware } from "./logger";
|
|
17
22
|
import { initializeMcpServer } from "./mcp/server";
|
|
18
23
|
import { ProjectStore } from "./service/project_store";
|
|
19
|
-
|
|
20
24
|
// Parse command line arguments
|
|
21
25
|
function parseArgs() {
|
|
22
26
|
const args = process.argv.slice(2);
|
|
@@ -34,6 +38,15 @@ function parseArgs() {
|
|
|
34
38
|
} else if (arg === "--mcp_port" && args[i + 1]) {
|
|
35
39
|
process.env.MCP_PORT = args[i + 1];
|
|
36
40
|
i++;
|
|
41
|
+
} else if (arg === "--shutdown_drain_duration_seconds" && args[i + 1]) {
|
|
42
|
+
process.env.SHUTDOWN_DRAIN_DURATION_SECONDS = args[i + 1];
|
|
43
|
+
i++;
|
|
44
|
+
} else if (
|
|
45
|
+
arg === "--shutdown_graceful_close_timeout_seconds" &&
|
|
46
|
+
args[i + 1]
|
|
47
|
+
) {
|
|
48
|
+
process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = args[i + 1];
|
|
49
|
+
i++;
|
|
37
50
|
} else if (arg === "--init") {
|
|
38
51
|
process.env.INITIALIZE_STORAGE = "true";
|
|
39
52
|
} else if (arg === "--help" || arg === "-h") {
|
|
@@ -54,6 +67,15 @@ function parseArgs() {
|
|
|
54
67
|
console.log(
|
|
55
68
|
" --mcp_port <number> Port for MCP server (default: 4040)",
|
|
56
69
|
);
|
|
70
|
+
console.log(
|
|
71
|
+
" --shutdown_drain_duration_seconds <number> Time in seconds to keep service in draining state before closing servers (default: 0)",
|
|
72
|
+
);
|
|
73
|
+
console.log(
|
|
74
|
+
" --shutdown_graceful_close_timeout_seconds <number> Time in seconds to wait after closing servers before exit (default: 0)",
|
|
75
|
+
);
|
|
76
|
+
console.log(
|
|
77
|
+
" --init Initialize the storage (default: false)",
|
|
78
|
+
);
|
|
57
79
|
console.log(" --help, -h Show this help message");
|
|
58
80
|
process.exit(0);
|
|
59
81
|
}
|
|
@@ -67,6 +89,12 @@ const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
|
|
|
67
89
|
const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
|
|
68
90
|
const MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
69
91
|
const MCP_ENDPOINT = "/mcp";
|
|
92
|
+
const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
|
|
93
|
+
process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
|
|
94
|
+
);
|
|
95
|
+
const SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(
|
|
96
|
+
process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0,
|
|
97
|
+
);
|
|
70
98
|
// Find the app directory - handle NPX vs local execution
|
|
71
99
|
let ROOT: string;
|
|
72
100
|
if (require.main) {
|
|
@@ -212,6 +240,12 @@ app.use(
|
|
|
212
240
|
);
|
|
213
241
|
app.use(bodyParser.json());
|
|
214
242
|
|
|
243
|
+
// Register health check endpoints
|
|
244
|
+
registerHealthEndpoints(app);
|
|
245
|
+
|
|
246
|
+
// Register draining guard middleware - must be after health endpoints but before other routes
|
|
247
|
+
app.use(drainingGuard);
|
|
248
|
+
|
|
215
249
|
app.get(`${API_PREFIX}/status`, async (_req, res) => {
|
|
216
250
|
try {
|
|
217
251
|
const status = await projectStore.getStatus();
|
|
@@ -905,3 +939,10 @@ const mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
|
|
|
905
939
|
mcpServer.timeout = 600000;
|
|
906
940
|
mcpServer.keepAliveTimeout = 600000;
|
|
907
941
|
mcpServer.headersTimeout = 600000;
|
|
942
|
+
|
|
943
|
+
registerSignalHandlers(
|
|
944
|
+
mainServer,
|
|
945
|
+
mcpServer,
|
|
946
|
+
SHUTDOWN_DRAIN_DURATION_SECONDS,
|
|
947
|
+
SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS,
|
|
948
|
+
);
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { expect, it, describe } from "bun:test";
|
|
2
1
|
import { MalloyError, Runtime } from "@malloydata/malloy";
|
|
3
|
-
import
|
|
2
|
+
import { describe, expect, it } from "bun:test";
|
|
4
3
|
import fs from "fs/promises";
|
|
4
|
+
import sinon from "sinon";
|
|
5
5
|
|
|
6
|
-
import { Model, ModelType } from "./model";
|
|
7
6
|
import { BadRequestError, ModelNotFoundError } from "../errors";
|
|
7
|
+
import { Model, ModelType } from "./model";
|
|
8
8
|
|
|
9
9
|
describe("service/model", () => {
|
|
10
10
|
const packageName = "test-package";
|
|
@@ -190,7 +190,7 @@ describe("service/model", () => {
|
|
|
190
190
|
});
|
|
191
191
|
|
|
192
192
|
describe("getQueryResults", () => {
|
|
193
|
-
it("should throw
|
|
193
|
+
it("should throw BadRequestError if a non-MalloyError compilation error exists", async () => {
|
|
194
194
|
const error = new Error("Compilation error");
|
|
195
195
|
const model = new Model(
|
|
196
196
|
packageName,
|
|
@@ -208,7 +208,7 @@ describe("service/model", () => {
|
|
|
208
208
|
|
|
209
209
|
await expect(async () => {
|
|
210
210
|
await model.getQueryResults();
|
|
211
|
-
}).toThrowError(
|
|
211
|
+
}).toThrowError(BadRequestError);
|
|
212
212
|
|
|
213
213
|
sinon.restore();
|
|
214
214
|
});
|