@shibbirweb/mcp-db-read-only 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.dockerhub.md +233 -234
- package/README.md +221 -269
- package/dist/ApplicationFactory.js +111 -14
- package/dist/cli/ViewerCommand.js +117 -0
- package/dist/config/EnvironmentConfigLoader.js +58 -0
- package/dist/drivers/BaseDriver.js +10 -1
- package/dist/drivers/document/MongoDriver.js +27 -30
- package/dist/drivers/keyvalue/RedisDriver.js +20 -5
- package/dist/drivers/search/ElasticsearchDriver.js +13 -9
- package/dist/drivers/sql/ClickHouseDriver.js +12 -13
- package/dist/drivers/sql/MsSqlDriver.js +6 -6
- package/dist/drivers/sql/MySqlDriver.js +7 -5
- package/dist/drivers/sql/MySqlSessionInitializer.js +17 -2
- package/dist/drivers/sql/PostgresDriver.js +6 -6
- package/dist/drivers/sql/SqliteDriver.js +3 -3
- package/dist/formatting/JsonSerializer.js +3 -2
- package/dist/index.js +16 -7
- package/dist/logging/CallLogger.js +139 -0
- package/dist/logging/LogChannel.js +18 -0
- package/dist/logging/LogFormatter.js +80 -0
- package/dist/logging/LogRecords.js +7 -0
- package/dist/logging/LogSink.js +38 -0
- package/dist/logging/RecordJson.js +39 -0
- package/dist/logging/Redactor.js +95 -0
- package/dist/logging/StatementTracer.js +9 -0
- package/dist/logging/ToolCallObserver.js +6 -0
- package/dist/logging/store/FolderLogChannel.js +56 -0
- package/dist/logging/store/FolderLogStore.js +214 -0
- package/dist/logging/store/LogFileNames.js +57 -0
- package/dist/logging/store/LogStore.js +18 -0
- package/dist/logging/store/MemoryLogStore.js +70 -0
- package/dist/logging/viewer/LiveLogViewer.js +264 -0
- package/dist/logging/viewer/LiveViewerObserver.js +62 -0
- package/dist/logging/viewer/ViewerAssets.js +625 -0
- package/dist/server/BackgroundService.js +1 -0
- package/dist/server/McpDbServer.js +16 -2
- package/dist/tools/BaseTool.js +8 -2
- package/dist/tools/connection/CurrentConnectionTool.js +11 -2
- package/package.json +1 -1
|
@@ -16,6 +16,17 @@ import { MySqlDriver } from "./drivers/sql/MySqlDriver.js";
|
|
|
16
16
|
import { PostgresDriver } from "./drivers/sql/PostgresDriver.js";
|
|
17
17
|
import { SqliteDriver } from "./drivers/sql/SqliteDriver.js";
|
|
18
18
|
import { RowFormatter } from "./formatting/RowFormatter.js";
|
|
19
|
+
import { CallLogger } from "./logging/CallLogger.js";
|
|
20
|
+
import { TextLogChannel } from "./logging/LogChannel.js";
|
|
21
|
+
import { LiveLogViewer } from "./logging/viewer/LiveLogViewer.js";
|
|
22
|
+
import { LiveViewerObserver } from "./logging/viewer/LiveViewerObserver.js";
|
|
23
|
+
import { FolderLogChannel } from "./logging/store/FolderLogChannel.js";
|
|
24
|
+
import { FolderLogStore } from "./logging/store/FolderLogStore.js";
|
|
25
|
+
import { MemoryLogStore } from "./logging/store/MemoryLogStore.js";
|
|
26
|
+
import { JsonLogFormatter, PrettyLogFormatter } from "./logging/LogFormatter.js";
|
|
27
|
+
import { FileSink, StderrSink } from "./logging/LogSink.js";
|
|
28
|
+
import { SilentTracer } from "./logging/StatementTracer.js";
|
|
29
|
+
import { SilentObserver } from "./logging/ToolCallObserver.js";
|
|
19
30
|
import { McpDbServer } from "./server/McpDbServer.js";
|
|
20
31
|
import { DescribeTableTool } from "./tools/browse/DescribeTableTool.js";
|
|
21
32
|
import { GetForeignKeysTool } from "./tools/browse/GetForeignKeysTool.js";
|
|
@@ -63,6 +74,11 @@ export class ApplicationFactory {
|
|
|
63
74
|
* total open connections at MAX_DRIVERS * CONNECTION_LIMIT.
|
|
64
75
|
*/
|
|
65
76
|
static MAX_DRIVERS = 8;
|
|
77
|
+
/**
|
|
78
|
+
* Set once built, for the call log's client name: the logger has to exist
|
|
79
|
+
* before the server that will learn the name at handshake.
|
|
80
|
+
*/
|
|
81
|
+
server = null;
|
|
66
82
|
constructor(configLoader = new EnvironmentConfigLoader(), logger = (message) => console.error(`[mcp-db-ro] ${message}`), versionLoader = new PackageVersionLoader()) {
|
|
67
83
|
this.configLoader = configLoader;
|
|
68
84
|
this.logger = logger;
|
|
@@ -86,31 +102,112 @@ export class ApplicationFactory {
|
|
|
86
102
|
connectTimeoutMs: config.connectTimeoutMs,
|
|
87
103
|
queryTimeoutMs: config.queryTimeoutMs,
|
|
88
104
|
};
|
|
89
|
-
const
|
|
105
|
+
const callLog = this.createCallLog(config.logging, registry);
|
|
106
|
+
const cache = new DriverCache(this.createDriverRegistry(tuning, callLog.tracer), ApplicationFactory.MAX_DRIVERS);
|
|
90
107
|
const connections = new ConnectionManager(registry, cache);
|
|
91
108
|
const drivers = new DriverProvider(registry, cache, QUERY_TOOLS);
|
|
92
|
-
const tools = this.createTools(connections, drivers);
|
|
93
|
-
|
|
109
|
+
const tools = this.createTools(connections, drivers, callLog.viewerStatus);
|
|
110
|
+
const server = new McpDbServer(tools, cache, this.logger, callLog.observer, callLog.services, this.versionLoader.load());
|
|
111
|
+
this.server = server;
|
|
112
|
+
return server;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* The call log when DB_LOG or DB_LOG_FILE asks for it, silent stand-ins
|
|
116
|
+
* otherwise. One CallLogger plays both roles, observer for the tools and
|
|
117
|
+
* tracer for the drivers, which is what lets it nest each driver statement
|
|
118
|
+
* under the call that caused it.
|
|
119
|
+
*/
|
|
120
|
+
createCallLog(settings, registry) {
|
|
121
|
+
if (!settings.enabled) {
|
|
122
|
+
return { observer: new SilentObserver(), tracer: new SilentTracer(), services: [], viewerStatus: () => null };
|
|
123
|
+
}
|
|
124
|
+
const channels = [];
|
|
125
|
+
const services = [];
|
|
126
|
+
let viewer = null;
|
|
127
|
+
const outputs = [];
|
|
128
|
+
const sink = settings.file ? new FileSink(settings.file) : new StderrSink();
|
|
129
|
+
if (settings.text) {
|
|
130
|
+
const formatter = settings.format === "json" ? new JsonLogFormatter() : new PrettyLogFormatter();
|
|
131
|
+
channels.push(new TextLogChannel(sink, formatter));
|
|
132
|
+
outputs.push(`${settings.format} text to ${sink.description}`);
|
|
133
|
+
}
|
|
134
|
+
// The viewer reads the folder when there is one, which also shows the
|
|
135
|
+
// calls of every other copy of the server saving there, and otherwise
|
|
136
|
+
// keeps this process's recent entries in memory.
|
|
137
|
+
let store;
|
|
138
|
+
if (settings.directory) {
|
|
139
|
+
const folderStore = new FolderLogStore(settings.directory);
|
|
140
|
+
store = folderStore;
|
|
141
|
+
channels.push(new FolderLogChannel(settings.directory, (entry) => folderStore.noteWritten(entry)));
|
|
142
|
+
outputs.push(`one JSON file per entry in ${settings.directory}`);
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
const memoryStore = new MemoryLogStore(settings.viewerHistory);
|
|
146
|
+
store = memoryStore;
|
|
147
|
+
if (settings.viewerPort !== null) {
|
|
148
|
+
channels.push(memoryStore);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
if (settings.viewerPort !== null) {
|
|
152
|
+
// Always all interfaces, as configured: reachable from other machines
|
|
153
|
+
// and from a Docker host without extra settings. The viewer announces
|
|
154
|
+
// that it has no access control when it starts, which is on the first
|
|
155
|
+
// tool call rather than here.
|
|
156
|
+
viewer = new LiveLogViewer("0.0.0.0", settings.viewerPort, store, this.logger);
|
|
157
|
+
services.push(viewer);
|
|
158
|
+
}
|
|
159
|
+
const logger = new CallLogger(channels, () => {
|
|
160
|
+
const target = registry.getActiveTarget();
|
|
161
|
+
return target
|
|
162
|
+
? `${registry.getActiveName()} (${EngineCatalog.label(target.engine)}) ${target.describe()}`
|
|
163
|
+
: null;
|
|
164
|
+
}, this.logger, () => this.clientName());
|
|
165
|
+
// Said once at startup, because with logging on every query and every
|
|
166
|
+
// result is being written somewhere, and the operator should know where.
|
|
167
|
+
this.logger(`call logging on: every tool call, its statements and its full output are written as ${outputs.join(" and ")}`);
|
|
168
|
+
const fallback = outputs.length > 0 ? outputs.join(" and ") : "nowhere else";
|
|
169
|
+
const observer = viewer ? new LiveViewerObserver(logger, viewer, fallback, this.logger) : logger;
|
|
170
|
+
return {
|
|
171
|
+
observer,
|
|
172
|
+
tracer: logger,
|
|
173
|
+
services,
|
|
174
|
+
viewerStatus: () => (viewer ? this.describeViewer(viewer) : null),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
clientName() {
|
|
178
|
+
return this.server?.clientName() ?? null;
|
|
179
|
+
}
|
|
180
|
+
/** One line for current_connection. */
|
|
181
|
+
describeViewer(viewer) {
|
|
182
|
+
const status = viewer.status;
|
|
183
|
+
switch (status.state) {
|
|
184
|
+
case "running":
|
|
185
|
+
return `running at ${status.url}`;
|
|
186
|
+
case "unavailable":
|
|
187
|
+
return `unavailable, ${status.reason} (free it or set DB_LOG_PORT to another port)`;
|
|
188
|
+
case "idle":
|
|
189
|
+
return "starts on the first tool call";
|
|
190
|
+
}
|
|
94
191
|
}
|
|
95
192
|
/** One factory per engine. None of them does I/O; drivers connect on first use. */
|
|
96
|
-
createDriverRegistry(tuning) {
|
|
193
|
+
createDriverRegistry(tuning, tracer) {
|
|
97
194
|
return new DriverRegistry()
|
|
98
|
-
.register("mysql", (target) => new MySqlDriver(target, tuning, this.logger))
|
|
99
|
-
.register("postgres", (target) => new PostgresDriver(target, tuning))
|
|
100
|
-
.register("sqlite", (target) => new SqliteDriver(target, tuning))
|
|
101
|
-
.register("mssql", (target) => new MsSqlDriver(target, tuning))
|
|
102
|
-
.register("clickhouse", (target) => new ClickHouseDriver(target, tuning))
|
|
103
|
-
.register("mongodb", (target) => new MongoDriver(target, tuning))
|
|
104
|
-
.register("redis", (target) => new RedisDriver(target, tuning, this.logger))
|
|
105
|
-
.register("elasticsearch", (target) => new ElasticsearchDriver(target, tuning));
|
|
195
|
+
.register("mysql", (target) => new MySqlDriver(target, tuning, tracer, this.logger))
|
|
196
|
+
.register("postgres", (target) => new PostgresDriver(target, tuning, tracer))
|
|
197
|
+
.register("sqlite", (target) => new SqliteDriver(target, tuning, tracer))
|
|
198
|
+
.register("mssql", (target) => new MsSqlDriver(target, tuning, tracer))
|
|
199
|
+
.register("clickhouse", (target) => new ClickHouseDriver(target, tuning, tracer))
|
|
200
|
+
.register("mongodb", (target) => new MongoDriver(target, tuning, tracer))
|
|
201
|
+
.register("redis", (target) => new RedisDriver(target, tuning, tracer, this.logger))
|
|
202
|
+
.register("elasticsearch", (target) => new ElasticsearchDriver(target, tuning, tracer));
|
|
106
203
|
}
|
|
107
|
-
createTools(connections, drivers) {
|
|
204
|
+
createTools(connections, drivers, viewerStatus) {
|
|
108
205
|
const names = new NamePolicyRegistry();
|
|
109
206
|
const targetFactory = new ConnectionTargetFactory();
|
|
110
207
|
const rows = new RowFormatter();
|
|
111
208
|
const mongoGuard = new MongoOperatorGuard();
|
|
112
209
|
const tools = [
|
|
113
|
-
new CurrentConnectionTool(connections),
|
|
210
|
+
new CurrentConnectionTool(connections, viewerStatus),
|
|
114
211
|
new ListConnectionsTool(connections),
|
|
115
212
|
new ListDatabasesTool(drivers),
|
|
116
213
|
new UseDatabaseTool(connections, names),
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { mkdirSync } from "node:fs";
|
|
2
|
+
import { resolve } from "node:path";
|
|
3
|
+
import { FolderLogStore } from "../logging/store/FolderLogStore.js";
|
|
4
|
+
import { LiveLogViewer } from "../logging/viewer/LiveLogViewer.js";
|
|
5
|
+
/**
|
|
6
|
+
* `mcp-db-read-only viewer --dir <folder> [--port 4800] [--host 0.0.0.0]`
|
|
7
|
+
*
|
|
8
|
+
* The live log viewer on its own, with no MCP server, no database and no
|
|
9
|
+
* credentials: it only reads the log folder that MCP servers configured with
|
|
10
|
+
* DB_LOG_DIR write into, and serves the same page as DB_LOG_PORT does.
|
|
11
|
+
*
|
|
12
|
+
* Run separately, in a terminal, so the servers an MCP client starts stay as
|
|
13
|
+
* light as they can be and never compete for a port. It shows the calls of
|
|
14
|
+
* every copy of the server writing to the folder, across restarts.
|
|
15
|
+
*
|
|
16
|
+
* Unlike the in-server viewer, it binds at once and exits if it cannot: here
|
|
17
|
+
* a person is watching the terminal, so a clear error and a non-zero exit is
|
|
18
|
+
* the useful answer to a taken port, where inside an MCP server it would have
|
|
19
|
+
* cost the chat its tools.
|
|
20
|
+
*/
|
|
21
|
+
export class ViewerCommand {
|
|
22
|
+
output;
|
|
23
|
+
env;
|
|
24
|
+
static DEFAULT_PORT = 4800;
|
|
25
|
+
static DEFAULT_HOST = "0.0.0.0";
|
|
26
|
+
static USAGE = [
|
|
27
|
+
"Usage: mcp-db-read-only viewer --dir <folder> [--port <port>] [--host <address>]",
|
|
28
|
+
"",
|
|
29
|
+
"Serves the live log viewer for a log folder written by MCP servers run with DB_LOG_DIR.",
|
|
30
|
+
"",
|
|
31
|
+
` --dir, -d The log folder (required; DB_LOG_DIR is used when omitted)`,
|
|
32
|
+
` --port, -p Port to listen on (default ${ViewerCommand.DEFAULT_PORT})`,
|
|
33
|
+
` --host Address to listen on (default ${ViewerCommand.DEFAULT_HOST}, every interface)`,
|
|
34
|
+
" --help, -h Show this help",
|
|
35
|
+
"",
|
|
36
|
+
"The viewer has no access control: anyone who can reach the port can read the log.",
|
|
37
|
+
].join("\n");
|
|
38
|
+
constructor(output = {
|
|
39
|
+
out: (message) => process.stdout.write(`${message}\n`),
|
|
40
|
+
error: (message) => process.stderr.write(`${message}\n`),
|
|
41
|
+
}, env = process.env) {
|
|
42
|
+
this.output = output;
|
|
43
|
+
this.env = env;
|
|
44
|
+
}
|
|
45
|
+
/** Pure: argument text in, options or an exit out. */
|
|
46
|
+
parse(args) {
|
|
47
|
+
let directory = this.env.DB_LOG_DIR ?? "";
|
|
48
|
+
let port = ViewerCommand.DEFAULT_PORT;
|
|
49
|
+
let host = ViewerCommand.DEFAULT_HOST;
|
|
50
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
51
|
+
const [flag, inline] = args[index].split(/=(.*)/s, 2);
|
|
52
|
+
const value = () => inline ?? args[++index];
|
|
53
|
+
switch (flag) {
|
|
54
|
+
case "--help":
|
|
55
|
+
case "-h":
|
|
56
|
+
return { kind: "exit", code: 0, message: ViewerCommand.USAGE };
|
|
57
|
+
case "--dir":
|
|
58
|
+
case "-d":
|
|
59
|
+
directory = value() ?? "";
|
|
60
|
+
break;
|
|
61
|
+
case "--port":
|
|
62
|
+
case "-p": {
|
|
63
|
+
const text = value() ?? "";
|
|
64
|
+
const parsed = Number(text);
|
|
65
|
+
if (!/^\d+$/.test(text) || parsed < 1 || parsed > 65535) {
|
|
66
|
+
return { kind: "exit", code: 2, message: `--port must be a number from 1 to 65535, got "${text}".\n\n${ViewerCommand.USAGE}` };
|
|
67
|
+
}
|
|
68
|
+
port = parsed;
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
case "--host":
|
|
72
|
+
host = value() ?? "";
|
|
73
|
+
break;
|
|
74
|
+
default:
|
|
75
|
+
return { kind: "exit", code: 2, message: `Unknown option "${args[index]}".\n\n${ViewerCommand.USAGE}` };
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
if (!directory) {
|
|
79
|
+
return { kind: "exit", code: 2, message: `--dir is required: the folder your MCP servers write with DB_LOG_DIR.\n\n${ViewerCommand.USAGE}` };
|
|
80
|
+
}
|
|
81
|
+
if (!host) {
|
|
82
|
+
return { kind: "exit", code: 2, message: `--host needs an address.\n\n${ViewerCommand.USAGE}` };
|
|
83
|
+
}
|
|
84
|
+
return { kind: "run", options: { directory: resolve(directory), port, host } };
|
|
85
|
+
}
|
|
86
|
+
/** @returns the exit code: 0 after a clean shutdown, non-zero when it could not start. */
|
|
87
|
+
async run(args) {
|
|
88
|
+
const parsed = this.parse(args);
|
|
89
|
+
if (parsed.kind === "exit") {
|
|
90
|
+
(parsed.code === 0 ? this.output.out : this.output.error)(parsed.message);
|
|
91
|
+
return parsed.code;
|
|
92
|
+
}
|
|
93
|
+
const { directory, port, host } = parsed.options;
|
|
94
|
+
// Created if missing, owner-only, so the viewer can be started before the
|
|
95
|
+
// first call is ever logged.
|
|
96
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
97
|
+
const viewer = new LiveLogViewer(host, port, new FolderLogStore(directory), (message) => this.output.out(message));
|
|
98
|
+
const status = await viewer.ensureRunning();
|
|
99
|
+
if (status.state !== "running") {
|
|
100
|
+
const reason = status.state === "unavailable" ? status.reason : "it could not start";
|
|
101
|
+
this.output.error(`Could not start the viewer: ${reason}. Free that port, or choose another with --port.`);
|
|
102
|
+
await viewer.stop();
|
|
103
|
+
return 1;
|
|
104
|
+
}
|
|
105
|
+
// The in-server viewer never holds its process open; here it is the
|
|
106
|
+
// whole point of the process.
|
|
107
|
+
viewer.holdProcessOpen();
|
|
108
|
+
this.output.out(`Reading ${directory}. Press Ctrl+C to stop.`);
|
|
109
|
+
return new Promise((done) => {
|
|
110
|
+
const stop = () => {
|
|
111
|
+
void viewer.stop().then(() => done(0));
|
|
112
|
+
};
|
|
113
|
+
process.once("SIGINT", stop);
|
|
114
|
+
process.once("SIGTERM", stop);
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { resolve } from "node:path";
|
|
1
2
|
import { ConnectionProfile } from "../domain/ConnectionProfile.js";
|
|
2
3
|
import { ConnectionTargetFactory } from "../connections/ConnectionTargetFactory.js";
|
|
3
4
|
/**
|
|
@@ -30,6 +31,7 @@ export class EnvironmentConfigLoader {
|
|
|
30
31
|
targetFactory;
|
|
31
32
|
static DEFAULT_QUERY_TIMEOUT_MS = 30000;
|
|
32
33
|
static DEFAULT_CONNECT_TIMEOUT_MS = 10000;
|
|
34
|
+
static DEFAULT_VIEWER_HISTORY = 500;
|
|
33
35
|
constructor(env = process.env, targetFactory = new ConnectionTargetFactory()) {
|
|
34
36
|
this.env = env;
|
|
35
37
|
this.targetFactory = targetFactory;
|
|
@@ -46,9 +48,65 @@ export class EnvironmentConfigLoader {
|
|
|
46
48
|
defaultProfileName: this.env.DB_DEFAULT_PROFILE || this.env.MYSQL_DEFAULT_PROFILE || null,
|
|
47
49
|
queryTimeoutMs: this.readTimeout(this.env.DB_QUERY_TIMEOUT_MS ?? this.env.MYSQL_QUERY_TIMEOUT_MS, EnvironmentConfigLoader.DEFAULT_QUERY_TIMEOUT_MS),
|
|
48
50
|
connectTimeoutMs: this.readTimeout(this.env.DB_CONNECT_TIMEOUT_MS ?? this.env.MYSQL_CONNECT_TIMEOUT_MS, EnvironmentConfigLoader.DEFAULT_CONNECT_TIMEOUT_MS),
|
|
51
|
+
logging: this.readLogging(warnings),
|
|
49
52
|
warnings,
|
|
50
53
|
};
|
|
51
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* DB_LOG=true logs text to stderr; DB_LOG_FILE=/path logs text to that
|
|
57
|
+
* file. DB_LOG_DIR=/folder saves every entry as its own JSON file there.
|
|
58
|
+
* Each of the three turns logging on by itself, and they combine.
|
|
59
|
+
* DB_LOG_FORMAT picks pretty (the default) or json for the text output.
|
|
60
|
+
*
|
|
61
|
+
* Relative paths are made absolute here, against the directory the server
|
|
62
|
+
* was started in, so the startup message names the real location.
|
|
63
|
+
*/
|
|
64
|
+
readLogging(warnings) {
|
|
65
|
+
const file = this.env.DB_LOG_FILE ? resolve(this.env.DB_LOG_FILE) : null;
|
|
66
|
+
const directory = this.env.DB_LOG_DIR ? resolve(this.env.DB_LOG_DIR) : null;
|
|
67
|
+
const stderr = ["true", "1", "yes", "on"].includes((this.env.DB_LOG ?? "").toLowerCase());
|
|
68
|
+
const text = stderr || file !== null;
|
|
69
|
+
const enabled = text || directory !== null;
|
|
70
|
+
const requested = (this.env.DB_LOG_FORMAT ?? "").toLowerCase();
|
|
71
|
+
let format = "pretty";
|
|
72
|
+
if (requested === "json") {
|
|
73
|
+
format = "json";
|
|
74
|
+
}
|
|
75
|
+
else if (requested && requested !== "pretty") {
|
|
76
|
+
warnings.push(`DB_LOG_FORMAT="${this.env.DB_LOG_FORMAT}" is not pretty or json, using pretty.`);
|
|
77
|
+
}
|
|
78
|
+
return { enabled, text, file, directory, format, ...this.readViewer(enabled, warnings) };
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* DB_LOG_PORT starts the live browser viewer, but only while logging is on:
|
|
82
|
+
* the viewer shows the call log, so without one it has nothing to show, and
|
|
83
|
+
* it says so rather than silently doing nothing.
|
|
84
|
+
*/
|
|
85
|
+
readViewer(loggingEnabled, warnings) {
|
|
86
|
+
const history = this.readCount(this.env.DB_LOG_HISTORY, EnvironmentConfigLoader.DEFAULT_VIEWER_HISTORY);
|
|
87
|
+
const raw = this.env.DB_LOG_PORT;
|
|
88
|
+
if (!raw) {
|
|
89
|
+
return { viewerPort: null, viewerHistory: history };
|
|
90
|
+
}
|
|
91
|
+
const port = Number(raw);
|
|
92
|
+
if (!/^\d+$/.test(raw) || port < 1 || port > 65535) {
|
|
93
|
+
warnings.push(`DB_LOG_PORT="${raw}" is not a port from 1 to 65535, so the live viewer is off.`);
|
|
94
|
+
return { viewerPort: null, viewerHistory: history };
|
|
95
|
+
}
|
|
96
|
+
if (!loggingEnabled) {
|
|
97
|
+
warnings.push("DB_LOG_PORT is set but call logging is off. Set DB_LOG_DIR, DB_LOG=true or DB_LOG_FILE to use the live viewer.");
|
|
98
|
+
return { viewerPort: null, viewerHistory: history };
|
|
99
|
+
}
|
|
100
|
+
return { viewerPort: port, viewerHistory: history };
|
|
101
|
+
}
|
|
102
|
+
/** A non-negative integer, or the fallback. 0 is allowed and means none. */
|
|
103
|
+
readCount(value, fallback) {
|
|
104
|
+
if (value === undefined || value === "") {
|
|
105
|
+
return fallback;
|
|
106
|
+
}
|
|
107
|
+
const parsed = Number(value);
|
|
108
|
+
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
|
|
109
|
+
}
|
|
52
110
|
/**
|
|
53
111
|
* DB_PROFILES: `{"name": "url"}` or `{"name": {"url": "...", "password": "..."}}`.
|
|
54
112
|
*
|
|
@@ -7,11 +7,16 @@ import { UnsupportedOperationError } from "../errors/UnsupportedOperationError.j
|
|
|
7
7
|
* The two optional capabilities, indexes and foreign keys, default to a
|
|
8
8
|
* refusal here, so an engine without them needs no code to say so, and an
|
|
9
9
|
* engine that has them overrides the one method.
|
|
10
|
+
*
|
|
11
|
+
* Every statement a driver sends goes through `traced`, which reports it to
|
|
12
|
+
* the call log when logging is on and is a plain call when it is off.
|
|
10
13
|
*/
|
|
11
14
|
export class BaseDriver {
|
|
12
15
|
target;
|
|
13
|
-
|
|
16
|
+
tracer;
|
|
17
|
+
constructor(target, tracer) {
|
|
14
18
|
this.target = target;
|
|
19
|
+
this.tracer = tracer;
|
|
15
20
|
}
|
|
16
21
|
async listIndexes(_name) {
|
|
17
22
|
throw new UnsupportedOperationError(this.label, "indexes to list");
|
|
@@ -19,6 +24,10 @@ export class BaseDriver {
|
|
|
19
24
|
async listForeignKeys(_name) {
|
|
20
25
|
throw new UnsupportedOperationError(this.label, "foreign keys");
|
|
21
26
|
}
|
|
27
|
+
/** Run one statement through the tracer, labelled with this engine. */
|
|
28
|
+
traced(text, params, run, describe) {
|
|
29
|
+
return this.tracer.trace(this.label, { text, params }, run, describe);
|
|
30
|
+
}
|
|
22
31
|
get label() {
|
|
23
32
|
return EngineCatalog.label(this.target.engine);
|
|
24
33
|
}
|
|
@@ -25,8 +25,8 @@ export class MongoDriver extends BaseDriver {
|
|
|
25
25
|
static SYSTEM_DATABASES = new Set(["admin", "local", "config"]);
|
|
26
26
|
static SCHEMA_SAMPLE_SIZE = 100;
|
|
27
27
|
client;
|
|
28
|
-
constructor(target, tuning, stages = new MongoStageAllowlist(), sampler = new MongoSchemaSampler()) {
|
|
29
|
-
super(target);
|
|
28
|
+
constructor(target, tuning, tracer, stages = new MongoStageAllowlist(), sampler = new MongoSchemaSampler()) {
|
|
29
|
+
super(target, tracer);
|
|
30
30
|
this.tuning = tuning;
|
|
31
31
|
this.stages = stages;
|
|
32
32
|
this.sampler = sampler;
|
|
@@ -34,7 +34,8 @@ export class MongoDriver extends BaseDriver {
|
|
|
34
34
|
}
|
|
35
35
|
async verify() {
|
|
36
36
|
const { client } = await this.client.get();
|
|
37
|
-
|
|
37
|
+
const database = this.target.database || "admin";
|
|
38
|
+
await this.traced(`${database}.ping`, undefined, () => client.db(database).command({ ping: 1 }), () => "ok");
|
|
38
39
|
}
|
|
39
40
|
close() {
|
|
40
41
|
return this.client.close();
|
|
@@ -45,7 +46,7 @@ export class MongoDriver extends BaseDriver {
|
|
|
45
46
|
*/
|
|
46
47
|
async listDatabases() {
|
|
47
48
|
const { client } = await this.client.get();
|
|
48
|
-
const result = await client.db("admin").admin().listDatabases({ nameOnly: true, authorizedDatabases: true });
|
|
49
|
+
const result = await this.traced("admin.listDatabases", { nameOnly: true, authorizedDatabases: true }, () => client.db("admin").admin().listDatabases({ nameOnly: true, authorizedDatabases: true }), (answer) => `${answer.databases.length} databases`);
|
|
49
50
|
return result.databases.map((entry) => ({
|
|
50
51
|
name: entry.name,
|
|
51
52
|
system: MongoDriver.SYSTEM_DATABASES.has(entry.name),
|
|
@@ -53,22 +54,18 @@ export class MongoDriver extends BaseDriver {
|
|
|
53
54
|
}
|
|
54
55
|
async listObjects(pattern, limit) {
|
|
55
56
|
const database = await this.database();
|
|
56
|
-
const collections = await database
|
|
57
|
-
.listCollections({}, { nameOnly: true, authorizedCollections: true })
|
|
58
|
-
.toArray();
|
|
57
|
+
const collections = await this.traced(`${database.databaseName}.listCollections`, undefined, () => database.listCollections({}, { nameOnly: true, authorizedCollections: true }).toArray());
|
|
59
58
|
return new GlobPattern(pattern).apply(collections.map((entry) => entry.name).sort(), limit);
|
|
60
59
|
}
|
|
61
60
|
/** The inferred shape of a sample, plus any JSON Schema validator the collection declares. */
|
|
62
61
|
async describeObject(name) {
|
|
63
62
|
const database = await this.database();
|
|
64
|
-
const [info] = await database.listCollections({ name }).toArray();
|
|
63
|
+
const [info] = await this.traced(`${database.databaseName}.listCollections`, { name }, () => database.listCollections({ name }).toArray());
|
|
65
64
|
if (!info) {
|
|
66
65
|
throw new ObjectNotFoundError(this.objectNoun, name);
|
|
67
66
|
}
|
|
68
|
-
const
|
|
69
|
-
|
|
70
|
-
.aggregate([{ $sample: { size: MongoDriver.SCHEMA_SAMPLE_SIZE } }], { maxTimeMS: this.tuning.queryTimeoutMs })
|
|
71
|
-
.toArray();
|
|
67
|
+
const sampling = [{ $sample: { size: MongoDriver.SCHEMA_SAMPLE_SIZE } }];
|
|
68
|
+
const documents = await this.traced(`${database.databaseName}.${name}.aggregate`, sampling, () => database.collection(name).aggregate(sampling, { maxTimeMS: this.tuning.queryTimeoutMs }).toArray());
|
|
72
69
|
return {
|
|
73
70
|
collection: name,
|
|
74
71
|
sampled_documents: documents.length,
|
|
@@ -79,7 +76,7 @@ export class MongoDriver extends BaseDriver {
|
|
|
79
76
|
async listIndexes(name) {
|
|
80
77
|
await this.assertCollection(name);
|
|
81
78
|
const database = await this.database();
|
|
82
|
-
return database.collection(name).indexes();
|
|
79
|
+
return this.traced(`${database.databaseName}.${name}.indexes`, undefined, () => database.collection(name).indexes());
|
|
83
80
|
}
|
|
84
81
|
async sample(name, limit) {
|
|
85
82
|
await this.assertCollection(name);
|
|
@@ -95,7 +92,8 @@ export class MongoDriver extends BaseDriver {
|
|
|
95
92
|
skip: request.skip,
|
|
96
93
|
maxTimeMS: this.tuning.queryTimeoutMs,
|
|
97
94
|
});
|
|
98
|
-
|
|
95
|
+
const documents = await this.traced(`${database.databaseName}.${collection}.find`, request, () => cursor.toArray());
|
|
96
|
+
return this.toJson(bson, documents);
|
|
99
97
|
}
|
|
100
98
|
/**
|
|
101
99
|
* Reads at most `limit + 1` documents and closes the cursor, so a pipeline
|
|
@@ -112,32 +110,31 @@ export class MongoDriver extends BaseDriver {
|
|
|
112
110
|
batchSize: limit + 1,
|
|
113
111
|
});
|
|
114
112
|
const rows = [];
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
113
|
+
await this.traced(`${database.databaseName}.${collection}.aggregate`, pipeline, async () => {
|
|
114
|
+
try {
|
|
115
|
+
for await (const document of cursor) {
|
|
116
|
+
rows.push(document);
|
|
117
|
+
if (rows.length > limit) {
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
120
|
}
|
|
121
121
|
}
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
122
|
+
finally {
|
|
123
|
+
await cursor.close();
|
|
124
|
+
}
|
|
125
|
+
return rows;
|
|
126
|
+
});
|
|
126
127
|
return { rows: this.toJson(bson, rows.slice(0, limit)), truncated: rows.length > limit };
|
|
127
128
|
}
|
|
128
129
|
async count(collection, filter) {
|
|
129
130
|
const { bson } = await this.client.get();
|
|
130
131
|
const database = await this.database();
|
|
131
|
-
return database
|
|
132
|
-
.collection(collection)
|
|
133
|
-
.countDocuments(this.fromJson(bson, filter), { maxTimeMS: this.tuning.queryTimeoutMs });
|
|
132
|
+
return this.traced(`${database.databaseName}.${collection}.countDocuments`, filter, () => database.collection(collection).countDocuments(this.fromJson(bson, filter), { maxTimeMS: this.tuning.queryTimeoutMs }));
|
|
134
133
|
}
|
|
135
134
|
async distinct(collection, field, filter) {
|
|
136
135
|
const { bson } = await this.client.get();
|
|
137
136
|
const database = await this.database();
|
|
138
|
-
const values = await database
|
|
139
|
-
.collection(collection)
|
|
140
|
-
.distinct(field, this.fromJson(bson, filter), { maxTimeMS: this.tuning.queryTimeoutMs });
|
|
137
|
+
const values = await this.traced(`${database.databaseName}.${collection}.distinct`, { field, filter }, () => database.collection(collection).distinct(field, this.fromJson(bson, filter), { maxTimeMS: this.tuning.queryTimeoutMs }));
|
|
141
138
|
return this.toJson(bson, values);
|
|
142
139
|
}
|
|
143
140
|
async database() {
|
|
@@ -146,7 +143,7 @@ export class MongoDriver extends BaseDriver {
|
|
|
146
143
|
}
|
|
147
144
|
async assertCollection(name) {
|
|
148
145
|
const database = await this.database();
|
|
149
|
-
const found = await database.listCollections({ name }, { nameOnly: true }).toArray();
|
|
146
|
+
const found = await this.traced(`${database.databaseName}.listCollections`, { name }, () => database.listCollections({ name }, { nameOnly: true }).toArray());
|
|
150
147
|
if (found.length === 0) {
|
|
151
148
|
throw new ObjectNotFoundError(this.objectNoun, name);
|
|
152
149
|
}
|
|
@@ -25,8 +25,8 @@ export class RedisDriver extends BaseDriver {
|
|
|
25
25
|
/** Long strings are cut in samples; the length is reported so nothing is hidden silently. */
|
|
26
26
|
static MAX_STRING_PREVIEW = 4096;
|
|
27
27
|
client;
|
|
28
|
-
constructor(target, tuning, logger, guard = new RedisCommandFlagsGuard()) {
|
|
29
|
-
super(target);
|
|
28
|
+
constructor(target, tuning, tracer, logger, guard = new RedisCommandFlagsGuard()) {
|
|
29
|
+
super(target, tracer);
|
|
30
30
|
this.tuning = tuning;
|
|
31
31
|
this.logger = logger;
|
|
32
32
|
this.guard = guard;
|
|
@@ -111,9 +111,12 @@ export class RedisDriver extends BaseDriver {
|
|
|
111
111
|
return { key: name, type, note: "No sample reader for this type; use redis_command." };
|
|
112
112
|
}
|
|
113
113
|
}
|
|
114
|
+
/**
|
|
115
|
+
* The guard's COMMAND INFO lookups go through `call` too, so the log shows
|
|
116
|
+
* the server being asked before the command itself is sent.
|
|
117
|
+
*/
|
|
114
118
|
async command(name, args) {
|
|
115
|
-
const
|
|
116
|
-
const call = (command, ...rest) => client.call(command, ...rest);
|
|
119
|
+
const call = (command, ...rest) => this.call(command, ...rest);
|
|
117
120
|
const subcommand = this.isContainer(name) ? args[0]?.toUpperCase() : undefined;
|
|
118
121
|
await this.guard.assertReadOnly(call, name, subcommand);
|
|
119
122
|
return call(name, ...args);
|
|
@@ -172,9 +175,21 @@ export class RedisDriver extends BaseDriver {
|
|
|
172
175
|
isContainer(name) {
|
|
173
176
|
return ["OBJECT", "MEMORY", "XINFO"].includes(name);
|
|
174
177
|
}
|
|
178
|
+
/** Every command this driver sends passes through here, so every one is traced. */
|
|
175
179
|
async call(command, ...args) {
|
|
176
180
|
const client = await this.client.get();
|
|
177
|
-
return client.call(command, ...args);
|
|
181
|
+
return this.traced([command, ...args].join(" "), undefined, () => client.call(command, ...args), RedisDriver.describeReply);
|
|
182
|
+
}
|
|
183
|
+
/** Arrays by length, short scalars by value, so `GET` and `DBSIZE` read naturally in the log. */
|
|
184
|
+
static describeReply(reply) {
|
|
185
|
+
if (Array.isArray(reply)) {
|
|
186
|
+
return `${reply.length} item${reply.length === 1 ? "" : "s"}`;
|
|
187
|
+
}
|
|
188
|
+
if (reply === null) {
|
|
189
|
+
return "nil";
|
|
190
|
+
}
|
|
191
|
+
const text = String(reply);
|
|
192
|
+
return text.length <= 40 ? text : `${text.length} characters`;
|
|
178
193
|
}
|
|
179
194
|
databaseIndex() {
|
|
180
195
|
const parsed = Number.parseInt(this.target.database, 10);
|
|
@@ -18,10 +18,10 @@ export class ElasticsearchDriver extends BaseDriver {
|
|
|
18
18
|
tuning;
|
|
19
19
|
fetcher;
|
|
20
20
|
family = "search";
|
|
21
|
-
constructor(target, tuning,
|
|
21
|
+
constructor(target, tuning, tracer,
|
|
22
22
|
// Wrapped so fetch is never invoked with this driver as its receiver.
|
|
23
23
|
fetcher = (input, init) => fetch(input, init)) {
|
|
24
|
-
super(target);
|
|
24
|
+
super(target, tracer);
|
|
25
25
|
this.tuning = tuning;
|
|
26
26
|
this.fetcher = fetcher;
|
|
27
27
|
}
|
|
@@ -80,13 +80,17 @@ export class ElasticsearchDriver extends BaseDriver {
|
|
|
80
80
|
}
|
|
81
81
|
async send(request, timeoutMs = this.tuning.queryTimeoutMs) {
|
|
82
82
|
const { method, path, body } = this.route(request);
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
83
|
+
// Traced as the request line, with the body as its parameters. The
|
|
84
|
+
// authorization header is never part of what is traced.
|
|
85
|
+
const { response, text } = await this.traced(`${method} ${path}`, body, async () => {
|
|
86
|
+
const answer = await this.fetcher(`${this.baseUrl()}${path}`, {
|
|
87
|
+
method,
|
|
88
|
+
headers: this.headers(),
|
|
89
|
+
body: body === undefined ? undefined : JSON.stringify(body),
|
|
90
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
91
|
+
});
|
|
92
|
+
return { response: answer, text: await answer.text() };
|
|
93
|
+
}, (result) => `HTTP ${result.response.status}`);
|
|
90
94
|
const parsed = text ? this.parse(text) : null;
|
|
91
95
|
if (response.status === 404 && request.kind !== "cluster" && request.kind !== "indices") {
|
|
92
96
|
throw new ObjectNotFoundError(this.objectNoun, request.index);
|
|
@@ -29,8 +29,8 @@ export class ClickHouseDriver extends BaseDriver {
|
|
|
29
29
|
*/
|
|
30
30
|
static MAX_RESULT_ROWS = 10000;
|
|
31
31
|
client;
|
|
32
|
-
constructor(target, tuning) {
|
|
33
|
-
super(target);
|
|
32
|
+
constructor(target, tuning, tracer) {
|
|
33
|
+
super(target, tracer);
|
|
34
34
|
this.tuning = tuning;
|
|
35
35
|
this.client = new LazyResource(() => this.open(), (opened) => opened.client.close());
|
|
36
36
|
}
|
|
@@ -77,13 +77,15 @@ export class ClickHouseDriver extends BaseDriver {
|
|
|
77
77
|
}
|
|
78
78
|
async run(sql, params) {
|
|
79
79
|
const { client, settings } = await this.client.get();
|
|
80
|
-
|
|
81
|
-
query
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
80
|
+
return this.traced(sql, params, async () => {
|
|
81
|
+
const result = await client.query({
|
|
82
|
+
query: sql,
|
|
83
|
+
format: "JSONEachRow",
|
|
84
|
+
query_params: params,
|
|
85
|
+
clickhouse_settings: settings,
|
|
86
|
+
});
|
|
87
|
+
return (await result.json());
|
|
85
88
|
});
|
|
86
|
-
return (await result.json());
|
|
87
89
|
}
|
|
88
90
|
async assertExists(database, table) {
|
|
89
91
|
const rows = await this.run("SELECT 1 AS found FROM system.tables WHERE database = {database:String} AND name = {table:String}", { database, table });
|
|
@@ -133,11 +135,8 @@ export class ClickHouseDriver extends BaseDriver {
|
|
|
133
135
|
* attaching any is exactly what a read-only account refuses.
|
|
134
136
|
*/
|
|
135
137
|
async negotiateSettings(client) {
|
|
136
|
-
const
|
|
137
|
-
|
|
138
|
-
format: "JSONEachRow",
|
|
139
|
-
});
|
|
140
|
-
const rows = (await result.json());
|
|
138
|
+
const probe = "SELECT toUInt8(getSetting('readonly')) AS readonly";
|
|
139
|
+
const rows = (await this.traced(probe, undefined, async () => (await client.query({ query: probe, format: "JSONEachRow" })).json()));
|
|
141
140
|
const level = Number(rows[0]?.readonly ?? 0);
|
|
142
141
|
const limits = {
|
|
143
142
|
max_execution_time: Math.max(1, Math.ceil(this.tuning.queryTimeoutMs / 1000)),
|