@malloy-publisher/server 0.0.231 → 0.0.232
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +4 -0
- package/dist/app/api-doc.yaml +74 -3
- package/dist/app/assets/{EnvironmentPage-wa_EPkwK.js → EnvironmentPage-DXEaZIPx.js} +1 -1
- package/dist/app/assets/{HomePage-jnCrupQp.js → HomePage-kofsqpZt.js} +1 -1
- package/dist/app/assets/{LightMode-DYbwNULZ.js → LightMode-CNhIlIlJ.js} +1 -1
- package/dist/app/assets/{MainPage-CuJLrPNI.js → MainPage-Bgqo8jCy.js} +1 -1
- package/dist/app/assets/{MaterializationsPage-D_67x2ee.js → MaterializationsPage-CgBlgGz2.js} +1 -1
- package/dist/app/assets/{ModelPage-D5JtAWqR.js → ModelPage-B0TjoDtf.js} +1 -1
- package/dist/app/assets/{PackagePage-BRwtqUSG.js → PackagePage-BL8vnFj1.js} +1 -1
- package/dist/app/assets/{RouteError-CBNNrnSD.js → RouteError-BzPby0X2.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-CTCeBneA.js → ThemeEditorPage-CTEP_9r3.js} +1 -1
- package/dist/app/assets/{WorkbookPage-SN6f1RBm.js → WorkbookPage-BwM3BmKw.js} +1 -1
- package/dist/app/assets/{core-Dp3q5Ieu.es-CD5FvM2s.js → core-CK68iv6w.es-CpRxXBt7.js} +1 -1
- package/dist/app/assets/{index-C_tJstcx.js → index-B33zGctF.js} +15 -15
- package/dist/app/assets/{index-DU4r7GdU.js → index-BabP-V-S.js} +346 -321
- package/dist/app/assets/{index-B3Nn8Vm2.js → index-BkiWKaAF.js} +266 -265
- package/dist/app/assets/{index-BLCx1EdC.js → index-CmkW1MiE.js} +1 -1
- package/dist/app/assets/{index-CfmBVB6M.js → index-tXJXwdyj.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +2 -0
- package/dist/package_load_worker.mjs +11 -1
- package/dist/server.mjs +3186 -433
- package/package.json +12 -12
- package/scripts/bake-duckdb-extensions.js +4 -1
- package/src/config.spec.ts +39 -0
- package/src/config.ts +135 -0
- package/src/controller/materialization.controller.spec.ts +62 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/controller/package.controller.spec.ts +6 -0
- package/src/controller/package.controller.ts +7 -2
- package/src/errors.ts +19 -0
- package/src/logger.spec.ts +18 -1
- package/src/logger.ts +3 -1
- package/src/materialization_metrics.spec.ts +89 -4
- package/src/materialization_metrics.ts +155 -5
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/embedding_index.spec.ts +1236 -0
- package/src/mcp/tools/embedding_index.ts +808 -0
- package/src/mcp/tools/get_context_eval.ts +194 -45
- package/src/mcp/tools/get_context_tool.spec.ts +295 -2
- package/src/mcp/tools/get_context_tool.ts +159 -10
- package/src/pg_helpers.spec.ts +201 -0
- package/src/pg_helpers.ts +44 -5
- package/src/server.ts +24 -0
- package/src/service/build_plan.spec.ts +128 -2
- package/src/service/build_plan.ts +239 -17
- package/src/service/connection.ts +263 -7
- package/src/service/connection_config.spec.ts +48 -0
- package/src/service/connection_config.ts +21 -8
- package/src/service/connection_federation.spec.ts +184 -0
- package/src/service/embedding_provider.spec.ts +329 -0
- package/src/service/embedding_provider.ts +236 -0
- package/src/service/environment.ts +274 -12
- package/src/service/environment_store.spec.ts +678 -3
- package/src/service/environment_store.ts +449 -33
- package/src/service/environment_store_clone.spec.ts +350 -0
- package/src/service/manifest_loader.spec.ts +68 -13
- package/src/service/manifest_loader.ts +67 -19
- package/src/service/materialization_build_session.spec.ts +435 -0
- package/src/service/materialization_build_session.ts +681 -0
- package/src/service/materialization_eligibility.spec.ts +158 -0
- package/src/service/materialization_eligibility.ts +305 -0
- package/src/service/materialization_serve_transform.spec.ts +1003 -0
- package/src/service/materialization_serve_transform.ts +779 -0
- package/src/service/materialization_service.spec.ts +774 -7
- package/src/service/materialization_service.ts +1107 -42
- package/src/service/materialization_test_fixtures.ts +7 -0
- package/src/service/model.spec.ts +207 -0
- package/src/service/model.ts +540 -52
- package/src/service/model_storage_serve.spec.ts +193 -0
- package/src/service/model_storage_serve_joins.spec.ts +193 -0
- package/src/service/package.spec.ts +196 -0
- package/src/service/package.ts +385 -17
- package/src/service/persistence_policy.spec.ts +109 -0
- package/src/storage/duckdb/schema.ts +37 -0
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +177 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +104 -0
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +235 -0
|
@@ -6,7 +6,7 @@ import extract from "extract-zip";
|
|
|
6
6
|
import * as fs from "fs";
|
|
7
7
|
import * as os from "os";
|
|
8
8
|
import * as path from "path";
|
|
9
|
-
import simpleGit from "simple-git";
|
|
9
|
+
import simpleGit, { type SimpleGitProgressEvent } from "simple-git";
|
|
10
10
|
import { Writable } from "stream";
|
|
11
11
|
import { components } from "../api";
|
|
12
12
|
import {
|
|
@@ -29,6 +29,10 @@ import {
|
|
|
29
29
|
} from "../errors";
|
|
30
30
|
import { getOperationalState, markNotReady, markReady } from "../health";
|
|
31
31
|
import { formatDuration, logger } from "../logger";
|
|
32
|
+
import {
|
|
33
|
+
deleteEnvironmentEmbeddings,
|
|
34
|
+
deletePackageEmbeddings,
|
|
35
|
+
} from "../mcp/tools/embedding_index";
|
|
32
36
|
import { redactPgSecrets } from "../pg_helpers";
|
|
33
37
|
import {
|
|
34
38
|
assertSafeEnvironmentPath,
|
|
@@ -151,6 +155,281 @@ export function resolvePackageLocation(
|
|
|
151
155
|
return path.isAbsolute(expanded) ? expanded : path.join(anchorDir, expanded);
|
|
152
156
|
}
|
|
153
157
|
|
|
158
|
+
/**
|
|
159
|
+
* Options for every package git clone. Packages are served from the default
|
|
160
|
+
* branch's working tree only (no ref is ever checked out afterwards, and no
|
|
161
|
+
* later git operation touches the clone: a reload deletes and re-clones), so
|
|
162
|
+
* history is pure download cost. Exported for tests.
|
|
163
|
+
*/
|
|
164
|
+
export const GIT_CLONE_OPTIONS: Record<string, string | number | null> = {
|
|
165
|
+
"--depth": 1,
|
|
166
|
+
"--single-branch": null,
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* With a progress handler configured, simple-git forces `--progress`, and a
|
|
171
|
+
* failed clone's error message then carries every progress line git wrote to
|
|
172
|
+
* stderr. Those messages surface in API error responses on the add/update
|
|
173
|
+
* package path and in the failure logs (the boot path's `/status` loadErrors
|
|
174
|
+
* carries a location-level summary instead), so drop the noise and keep the
|
|
175
|
+
* lines that say what went wrong. Returns the input when everything matched,
|
|
176
|
+
* rather than an empty error.
|
|
177
|
+
*/
|
|
178
|
+
export function stripGitProgressNoise(message: string): string {
|
|
179
|
+
const kept = message
|
|
180
|
+
.split(/\r\n|\r|\n/)
|
|
181
|
+
.filter(
|
|
182
|
+
(line) =>
|
|
183
|
+
!/\d+% \(\d+\/\d+\)/.test(line) &&
|
|
184
|
+
// Also matches the "Error: Cloning into ..." shape V8 seeds
|
|
185
|
+
// into err.stack from the message's first line.
|
|
186
|
+
!/^(\w+: )?Cloning into /.test(line.trim()),
|
|
187
|
+
)
|
|
188
|
+
.join("\n")
|
|
189
|
+
.trim();
|
|
190
|
+
return kept.length > 0 ? kept : message.trim();
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** e.g. `[examples] cloning malloydata/publisher (storefront, +2 more)`. */
|
|
194
|
+
export function cloneProgressLabel(
|
|
195
|
+
repo: string,
|
|
196
|
+
context?: { environmentName?: string; packageNames?: string[] },
|
|
197
|
+
): string {
|
|
198
|
+
const env = context?.environmentName ? `[${context.environmentName}] ` : "";
|
|
199
|
+
const names = context?.packageNames ?? [];
|
|
200
|
+
const shown = names.length > 4 ? names.slice(0, 3) : names;
|
|
201
|
+
const more = names.length - shown.length;
|
|
202
|
+
const pkgs =
|
|
203
|
+
shown.length > 0
|
|
204
|
+
? ` (${shown.join(", ")}${more > 0 ? `, +${more} more` : ""})`
|
|
205
|
+
: "";
|
|
206
|
+
return `${env}cloning ${repo}${pkgs}`;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Streams git clone progress to stderr: off winston (whose format changes
|
|
211
|
+
* under OTEL and whose level can hide info lines) and off stdout (which an
|
|
212
|
+
* MCP stdio transport would own), and it is where git itself reports
|
|
213
|
+
* progress. A TTY gets one line rewritten in place; anything else (CI logs)
|
|
214
|
+
* gets a line per stage and per 25-point step, so a slow clone shows life
|
|
215
|
+
* without flooding. Exported for tests.
|
|
216
|
+
*/
|
|
217
|
+
export class CloneProgressReporter {
|
|
218
|
+
private lastStage: string | undefined;
|
|
219
|
+
private lastMilestone = -1;
|
|
220
|
+
private wroteInPlace = false;
|
|
221
|
+
private finished = false;
|
|
222
|
+
/**
|
|
223
|
+
* One reporter at a time owns a TTY's in-place line: environments load
|
|
224
|
+
* concurrently at boot, and two reporters `\r`-rewriting the same line
|
|
225
|
+
* would overwrite each other. Later reporters fall back to milestone
|
|
226
|
+
* lines until the owner finishes. Keyed by stream so tests with fake
|
|
227
|
+
* streams cannot collide.
|
|
228
|
+
*/
|
|
229
|
+
private static ttyOwners = new WeakMap<object, CloneProgressReporter>();
|
|
230
|
+
|
|
231
|
+
constructor(
|
|
232
|
+
private readonly label: string,
|
|
233
|
+
private readonly out: NodeJS.WriteStream = process.stderr,
|
|
234
|
+
) {}
|
|
235
|
+
|
|
236
|
+
onProgress(event: SimpleGitProgressEvent): void {
|
|
237
|
+
if (this.finished) {
|
|
238
|
+
// A late event after done() must not re-claim the TTY line the
|
|
239
|
+
// finishing newline just released.
|
|
240
|
+
return;
|
|
241
|
+
}
|
|
242
|
+
const { stage, progress, processed, total } = event;
|
|
243
|
+
const counts = total > 0 ? ` (${processed}/${total})` : "";
|
|
244
|
+
const text = `${this.label}: ${stage} ${progress}%${counts}`;
|
|
245
|
+
if (this.out.isTTY === true && this.claimTtyLine()) {
|
|
246
|
+
// Clamped to the terminal width: an auto-wrapped line breaks the
|
|
247
|
+
// \r rewrite (git clamps its own progress output for the same
|
|
248
|
+
// reason). The moving part sits at the end, so shorten the label
|
|
249
|
+
// and keep the stage/percentage visible; only when even the
|
|
250
|
+
// suffix cannot fit does a plain head slice apply. \x1b[K clears
|
|
251
|
+
// the tail of a longer previous line.
|
|
252
|
+
const budget = Math.max(1, (this.out.columns || 80) - 1);
|
|
253
|
+
const suffix = `: ${stage} ${progress}%${counts}`;
|
|
254
|
+
let line: string;
|
|
255
|
+
if (text.length <= budget) {
|
|
256
|
+
line = text;
|
|
257
|
+
} else if (suffix.length + 4 <= budget) {
|
|
258
|
+
const labelBudget = budget - suffix.length;
|
|
259
|
+
line = `${this.label.slice(0, labelBudget - 3)}...${suffix}`;
|
|
260
|
+
} else {
|
|
261
|
+
line = text.slice(0, budget);
|
|
262
|
+
}
|
|
263
|
+
this.out.write(`\r${line}\x1b[K`);
|
|
264
|
+
this.wroteInPlace = true;
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
const milestone = Math.floor(progress / 25);
|
|
268
|
+
if (stage !== this.lastStage || milestone < this.lastMilestone) {
|
|
269
|
+
// A new stage, or a percentage restart inside one reported stage:
|
|
270
|
+
// git's server-side counting and compressing phases both arrive as
|
|
271
|
+
// stage "remote:", so a drop in progress marks the next sub-phase.
|
|
272
|
+
this.lastStage = stage;
|
|
273
|
+
this.lastMilestone = milestone;
|
|
274
|
+
} else if (milestone > this.lastMilestone) {
|
|
275
|
+
this.lastMilestone = milestone;
|
|
276
|
+
} else {
|
|
277
|
+
return;
|
|
278
|
+
}
|
|
279
|
+
if (this.out.isTTY === true) {
|
|
280
|
+
// A non-owner on a TTY: the cursor may sit mid-line on the owner's
|
|
281
|
+
// in-place output, so take over the row cleanly and scroll it.
|
|
282
|
+
this.out.write(`\r${text}\x1b[K\n`);
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
this.out.write(`${text}\n`);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
done(): void {
|
|
289
|
+
this.finished = true;
|
|
290
|
+
if (this.wroteInPlace) {
|
|
291
|
+
this.out.write("\n");
|
|
292
|
+
}
|
|
293
|
+
if (CloneProgressReporter.ttyOwners.get(this.out) === this) {
|
|
294
|
+
CloneProgressReporter.ttyOwners.delete(this.out);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
private claimTtyLine(): boolean {
|
|
299
|
+
const owner = CloneProgressReporter.ttyOwners.get(this.out);
|
|
300
|
+
if (owner === undefined) {
|
|
301
|
+
CloneProgressReporter.ttyOwners.set(this.out, this);
|
|
302
|
+
return true;
|
|
303
|
+
}
|
|
304
|
+
return owner === this;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* The last `/segment` of a run of non-empty path segments at the start of
|
|
310
|
+
* `afterRepo`, or undefined. Reproduces the capture of the old
|
|
311
|
+
* `(\/[^/]+)*` group, which retained only its final iteration and stopped at
|
|
312
|
+
* the first empty segment.
|
|
313
|
+
*/
|
|
314
|
+
function lastGitHubPathSegment(afterRepo: string): string | undefined {
|
|
315
|
+
if (!afterRepo.startsWith("/")) return undefined;
|
|
316
|
+
const run: string[] = [];
|
|
317
|
+
for (const seg of afterRepo.slice(1).split("/")) {
|
|
318
|
+
if (seg === "") break;
|
|
319
|
+
run.push(seg);
|
|
320
|
+
}
|
|
321
|
+
return run.length > 0 ? `/${run[run.length - 1]}` : undefined;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
/**
|
|
325
|
+
* Parse a GitHub package location into owner, repo, and (optionally) the
|
|
326
|
+
* subdirectory path segment.
|
|
327
|
+
*
|
|
328
|
+
* Uses a repetition-free regex for owner/repo plus plain string ops for the
|
|
329
|
+
* path, NOT a single `(\/[^/]+)*` regex: those shapes were flagged as
|
|
330
|
+
* polynomial-backtracking (ReDoS) on adversarial input, and the location is
|
|
331
|
+
* user/config supplied. Behavior is preserved exactly, including
|
|
332
|
+
* `packagePath` being only the LAST path segment after the repo (a quirk the
|
|
333
|
+
* callers rely on); environment_store.spec.ts fuzzes the old regexes as an
|
|
334
|
+
* oracle to prove equivalence. Exported for that test.
|
|
335
|
+
*/
|
|
336
|
+
export function parseGitHubUrl(
|
|
337
|
+
githubUrl: string,
|
|
338
|
+
): { owner: string; repoName: string; packagePath?: string } | null {
|
|
339
|
+
// HTTPS: https://github.com/owner/repo/tree/branch/subdir. The host may
|
|
340
|
+
// appear anywhere (the old regex was unanchored); the owner/repo pattern
|
|
341
|
+
// has no repetition group, so it cannot backtrack polynomially.
|
|
342
|
+
const httpsMatch = /github\.com\/([^/]+)\/([^/]+)/.exec(githubUrl);
|
|
343
|
+
if (httpsMatch) {
|
|
344
|
+
const afterRepo = githubUrl.slice(
|
|
345
|
+
httpsMatch.index + httpsMatch[0].length,
|
|
346
|
+
);
|
|
347
|
+
return {
|
|
348
|
+
owner: httpsMatch[1],
|
|
349
|
+
repoName: httpsMatch[2],
|
|
350
|
+
packagePath: lastGitHubPathSegment(afterRepo),
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
// SSH: git@github.com:owner/repo(.git)?(/subdir)*. The old regex was
|
|
355
|
+
// $-anchored, so the whole tail after the repo had to be a run of
|
|
356
|
+
// non-empty `/segment`s; a trailing slash or empty segment matched nothing
|
|
357
|
+
// and failed the parse.
|
|
358
|
+
// `git@github.com:` may appear anywhere (the old regex was not anchored at
|
|
359
|
+
// the start), though in practice isGitHubURL only routes locations that
|
|
360
|
+
// begin with it here.
|
|
361
|
+
const SSH_PREFIX = "git@github.com:";
|
|
362
|
+
const sshAt = githubUrl.indexOf(SSH_PREFIX);
|
|
363
|
+
if (sshAt !== -1) {
|
|
364
|
+
const rest = githubUrl.slice(sshAt + SSH_PREFIX.length);
|
|
365
|
+
const firstSlash = rest.indexOf("/");
|
|
366
|
+
if (firstSlash > 0) {
|
|
367
|
+
const owner = rest.slice(0, firstSlash);
|
|
368
|
+
const afterOwner = rest.slice(firstSlash + 1);
|
|
369
|
+
const secondSlash = afterOwner.indexOf("/");
|
|
370
|
+
const repoSeg =
|
|
371
|
+
secondSlash === -1 ? afterOwner : afterOwner.slice(0, secondSlash);
|
|
372
|
+
const afterRepo =
|
|
373
|
+
secondSlash === -1 ? "" : afterOwner.slice(secondSlash);
|
|
374
|
+
// repo was `[^/\s]+` (no whitespace); anything else falls through to
|
|
375
|
+
// "not a GitHub URL", as the old regex did.
|
|
376
|
+
if (repoSeg.length > 0 && !/\s/.test(repoSeg)) {
|
|
377
|
+
const repoName =
|
|
378
|
+
repoSeg.length > 4 && repoSeg.endsWith(".git")
|
|
379
|
+
? repoSeg.slice(0, -4)
|
|
380
|
+
: repoSeg;
|
|
381
|
+
if (afterRepo === "") {
|
|
382
|
+
return { owner, repoName };
|
|
383
|
+
}
|
|
384
|
+
const segments = afterRepo.slice(1).split("/");
|
|
385
|
+
// $-anchored: every trailing segment had to be non-empty.
|
|
386
|
+
if (!segments.includes("")) {
|
|
387
|
+
return {
|
|
388
|
+
owner,
|
|
389
|
+
repoName,
|
|
390
|
+
packagePath: `/${segments[segments.length - 1]}`,
|
|
391
|
+
};
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
return null;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/** `0.0.0.0`/`::` bind every interface; a script needs a host it can dial. */
|
|
401
|
+
function displayHost(host: string): string {
|
|
402
|
+
if (host === "0.0.0.0" || host === "::") {
|
|
403
|
+
return "localhost";
|
|
404
|
+
}
|
|
405
|
+
// An IPv6 literal needs brackets to be dialable inside a URL.
|
|
406
|
+
return host.includes(":") ? `[${host}]` : host;
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* The one-line machine-readable boot signal. Scripts grep for the
|
|
411
|
+
* `PUBLISHER_READY` prefix instead of polling `/api/v0/status`, and the
|
|
412
|
+
* counts carry what `operationalState: "serving"` alone does not say: how
|
|
413
|
+
* much of the configured world actually loaded. Host and ports mirror
|
|
414
|
+
* server.ts's use-site defaults; parseArgs writes the flag values into
|
|
415
|
+
* process.env before the store is constructed. Exported for tests.
|
|
416
|
+
*/
|
|
417
|
+
export function formatReadinessLine(counts: {
|
|
418
|
+
environments: number;
|
|
419
|
+
packages: number;
|
|
420
|
+
loadErrors: number;
|
|
421
|
+
}): string {
|
|
422
|
+
const host = displayHost(process.env.PUBLISHER_HOST || "0.0.0.0");
|
|
423
|
+
const port = Number(process.env.PUBLISHER_PORT || 4000);
|
|
424
|
+
const mcpPort = Number(process.env.MCP_PORT || 4040);
|
|
425
|
+
return (
|
|
426
|
+
`PUBLISHER_READY url=http://${host}:${port} ` +
|
|
427
|
+
`mcp=http://${host}:${mcpPort} ` +
|
|
428
|
+
`environments=${counts.environments} packages=${counts.packages} ` +
|
|
429
|
+
`load_errors=${counts.loadErrors}`
|
|
430
|
+
);
|
|
431
|
+
}
|
|
432
|
+
|
|
154
433
|
export class EnvironmentStore {
|
|
155
434
|
public serverRootPath: string;
|
|
156
435
|
private environments: Map<string, Environment> = new Map();
|
|
@@ -404,6 +683,28 @@ export class EnvironmentStore {
|
|
|
404
683
|
environmentInstance.setMemoryGovernor(
|
|
405
684
|
this.memoryGovernor,
|
|
406
685
|
);
|
|
686
|
+
// Re-establish serve routing when a package loads, from
|
|
687
|
+
// its latest successful materialization — so serving
|
|
688
|
+
// survives a restart, not only a fresh build. The full
|
|
689
|
+
// entry map is returned; the environment splits it by
|
|
690
|
+
// tier (colocated + storage=) in
|
|
691
|
+
// rebindServeBindingsFromLocalStore.
|
|
692
|
+
const envId = dbEnvironment.id;
|
|
693
|
+
environmentInstance.setStorageBindingResolver(
|
|
694
|
+
async (packageName) => {
|
|
695
|
+
const runs =
|
|
696
|
+
await repository.listMaterializations(
|
|
697
|
+
envId,
|
|
698
|
+
packageName,
|
|
699
|
+
);
|
|
700
|
+
const latest = runs.find(
|
|
701
|
+
(m) =>
|
|
702
|
+
m.status === "MANIFEST_FILE_READY" &&
|
|
703
|
+
m.manifest?.entries,
|
|
704
|
+
);
|
|
705
|
+
return latest?.manifest?.entries ?? {};
|
|
706
|
+
},
|
|
707
|
+
);
|
|
407
708
|
|
|
408
709
|
// Get packages from database
|
|
409
710
|
const packages = await repository.listPackages(
|
|
@@ -446,10 +747,55 @@ export class EnvironmentStore {
|
|
|
446
747
|
logger.info(
|
|
447
748
|
`Environment store successfully initialized in ${formatDuration(initializationDuration)}`,
|
|
448
749
|
);
|
|
750
|
+
this.emitReadinessLine();
|
|
449
751
|
} catch (error) {
|
|
450
752
|
markNotReady();
|
|
451
753
|
const errorData = this.extractErrorDataFromError(error);
|
|
452
754
|
logger.error("Error initializing environment store", errorData);
|
|
755
|
+
try {
|
|
756
|
+
// The failure counterpart of PUBLISHER_READY, so a script
|
|
757
|
+
// waiting on that token fails fast instead of hanging on a
|
|
758
|
+
// broken config. Redacted because initialization errors can
|
|
759
|
+
// carry connection strings.
|
|
760
|
+
process.stderr.write(
|
|
761
|
+
`PUBLISHER_INIT_FAILED error=${JSON.stringify(
|
|
762
|
+
redactPgSecrets(
|
|
763
|
+
error instanceof Error ? error.message : String(error),
|
|
764
|
+
),
|
|
765
|
+
)}\n`,
|
|
766
|
+
);
|
|
767
|
+
} catch {
|
|
768
|
+
// A failed stderr write must not mask the logged error above.
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
/**
|
|
774
|
+
* Printed exactly once, from initialize()'s success tail: markReady() has
|
|
775
|
+
* just flipped /api/v0/status to "serving", and both HTTP listeners bound
|
|
776
|
+
* long before the downloads finished, so this is the same moment every
|
|
777
|
+
* existing readiness consumer (CI status polls, MCP /health/readiness)
|
|
778
|
+
* fires on. The catch above swallows a failed boot after markNotReady(),
|
|
779
|
+
* so the line never prints for a server that is not actually serving.
|
|
780
|
+
*/
|
|
781
|
+
private emitReadinessLine(): void {
|
|
782
|
+
let packages = 0;
|
|
783
|
+
let loadErrors = this.failedEnvironments.size;
|
|
784
|
+
for (const environment of this.environments.values()) {
|
|
785
|
+
packages += environment.getServingPackageCount();
|
|
786
|
+
loadErrors += environment.getFailedPackages().size;
|
|
787
|
+
}
|
|
788
|
+
try {
|
|
789
|
+
process.stderr.write(
|
|
790
|
+
formatReadinessLine({
|
|
791
|
+
environments: this.environments.size,
|
|
792
|
+
packages,
|
|
793
|
+
loadErrors,
|
|
794
|
+
}) + "\n",
|
|
795
|
+
);
|
|
796
|
+
} catch {
|
|
797
|
+
// A failed stderr write must not fail a boot that is already
|
|
798
|
+
// serving.
|
|
453
799
|
}
|
|
454
800
|
}
|
|
455
801
|
|
|
@@ -486,6 +832,10 @@ export class EnvironmentStore {
|
|
|
486
832
|
public async deleteEnvironmentFromDatabase(
|
|
487
833
|
environmentName: string,
|
|
488
834
|
): Promise<void> {
|
|
835
|
+
// Before the metadata lookup, for the same reason as the package
|
|
836
|
+
// variant: the cleanup needs only the name, never the row.
|
|
837
|
+
this.cleanupEnvironmentEmbeddings(environmentName);
|
|
838
|
+
|
|
489
839
|
const repository = this.storageManager.getRepository();
|
|
490
840
|
|
|
491
841
|
// Get the environment from database
|
|
@@ -755,6 +1105,11 @@ export class EnvironmentStore {
|
|
|
755
1105
|
environmentName: string,
|
|
756
1106
|
packageName: string,
|
|
757
1107
|
): Promise<void> {
|
|
1108
|
+
// Before the metadata lookups: embeddings are written off the
|
|
1109
|
+
// in-memory environment, which can outlive a lost metadata row,
|
|
1110
|
+
// so the cleanup must not sit behind the missing-row early return.
|
|
1111
|
+
this.cleanupPackageEmbeddings(environmentName, packageName);
|
|
1112
|
+
|
|
758
1113
|
const repository = this.storageManager.getRepository();
|
|
759
1114
|
|
|
760
1115
|
// Get the environment ID from database
|
|
@@ -778,6 +1133,64 @@ export class EnvironmentStore {
|
|
|
778
1133
|
}
|
|
779
1134
|
}
|
|
780
1135
|
|
|
1136
|
+
/**
|
|
1137
|
+
* Best-effort, non-blocking: drop a deleted package's cached entity
|
|
1138
|
+
* embeddings so package churn does not grow publisher.db forever.
|
|
1139
|
+
* Not awaited by callers: the cleanup queues on the package's sync
|
|
1140
|
+
* mutex, which an in-flight bulk embed can hold for minutes, and a
|
|
1141
|
+
* DELETE response must not stall behind it. Needs only the names, so
|
|
1142
|
+
* it runs regardless of whether the metadata row still exists. An
|
|
1143
|
+
* orphaned row is inert (reads are scoped by environment + package),
|
|
1144
|
+
* so failure is logged, never propagated.
|
|
1145
|
+
*/
|
|
1146
|
+
private cleanupPackageEmbeddings(
|
|
1147
|
+
environmentName: string,
|
|
1148
|
+
packageName: string,
|
|
1149
|
+
): void {
|
|
1150
|
+
try {
|
|
1151
|
+
deletePackageEmbeddings(
|
|
1152
|
+
this.storageManager.getDuckDbConnection(),
|
|
1153
|
+
environmentName,
|
|
1154
|
+
packageName,
|
|
1155
|
+
).catch((error: unknown) => {
|
|
1156
|
+
logger.warn("Failed to clean up entity embeddings for package", {
|
|
1157
|
+
environmentName,
|
|
1158
|
+
packageName,
|
|
1159
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1160
|
+
});
|
|
1161
|
+
});
|
|
1162
|
+
} catch (error) {
|
|
1163
|
+
logger.warn("Failed to clean up entity embeddings for package", {
|
|
1164
|
+
environmentName,
|
|
1165
|
+
packageName,
|
|
1166
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1167
|
+
});
|
|
1168
|
+
}
|
|
1169
|
+
}
|
|
1170
|
+
|
|
1171
|
+
/** Environment-wide variant of {@link cleanupPackageEmbeddings}. */
|
|
1172
|
+
private cleanupEnvironmentEmbeddings(environmentName: string): void {
|
|
1173
|
+
try {
|
|
1174
|
+
deleteEnvironmentEmbeddings(
|
|
1175
|
+
this.storageManager.getDuckDbConnection(),
|
|
1176
|
+
environmentName,
|
|
1177
|
+
).catch((error: unknown) => {
|
|
1178
|
+
logger.warn(
|
|
1179
|
+
"Failed to clean up entity embeddings for environment",
|
|
1180
|
+
{
|
|
1181
|
+
environmentName,
|
|
1182
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1183
|
+
},
|
|
1184
|
+
);
|
|
1185
|
+
});
|
|
1186
|
+
} catch (error) {
|
|
1187
|
+
logger.warn("Failed to clean up entity embeddings for environment", {
|
|
1188
|
+
environmentName,
|
|
1189
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1190
|
+
});
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
|
|
781
1194
|
private async cleanupAndCreatePublisherPath() {
|
|
782
1195
|
const reInit = process.env.INITIALIZE_STORAGE === "true";
|
|
783
1196
|
|
|
@@ -1347,7 +1760,7 @@ export class EnvironmentStore {
|
|
|
1347
1760
|
// For GitHub URLs, group by base repository URL to optimize downloads
|
|
1348
1761
|
let locationKey = _package.location;
|
|
1349
1762
|
if (this.isGitHubURL(_package.location)) {
|
|
1350
|
-
const githubInfo =
|
|
1763
|
+
const githubInfo = parseGitHubUrl(_package.location);
|
|
1351
1764
|
if (githubInfo) {
|
|
1352
1765
|
// Always use HTTPS format for grouping to ensure consistency
|
|
1353
1766
|
locationKey = `https://github.com/${githubInfo.owner}/${githubInfo.repoName}`;
|
|
@@ -1364,6 +1777,8 @@ export class EnvironmentStore {
|
|
|
1364
1777
|
}
|
|
1365
1778
|
|
|
1366
1779
|
// Processing by each unique location
|
|
1780
|
+
const totalPackages = packages.length;
|
|
1781
|
+
let mountedCount = 0;
|
|
1367
1782
|
for (const [groupedLocation, packagesForLocation] of locationGroups) {
|
|
1368
1783
|
// Use a hash instead of base64 to keep paths short (Windows MAX_PATH limit)
|
|
1369
1784
|
// This works for both local and remote paths
|
|
@@ -1390,6 +1805,7 @@ export class EnvironmentStore {
|
|
|
1390
1805
|
tempDownloadPath,
|
|
1391
1806
|
environmentName,
|
|
1392
1807
|
"shared",
|
|
1808
|
+
packagesForLocation.map((p) => p.name),
|
|
1393
1809
|
);
|
|
1394
1810
|
} catch (error) {
|
|
1395
1811
|
downloaded = false;
|
|
@@ -1419,7 +1835,7 @@ export class EnvironmentStore {
|
|
|
1419
1835
|
// For GitHub URLs, extract the subdirectory path from the original location
|
|
1420
1836
|
let sourcePath: string;
|
|
1421
1837
|
if (this.isGitHubURL(_package.location)) {
|
|
1422
|
-
const githubInfo =
|
|
1838
|
+
const githubInfo = parseGitHubUrl(_package.location);
|
|
1423
1839
|
if (githubInfo && githubInfo.packagePath) {
|
|
1424
1840
|
// Extract subdirectory from the original GitHub URL
|
|
1425
1841
|
// Handle both /tree/main/subdir and /tree/branch/subdir cases
|
|
@@ -1490,7 +1906,7 @@ export class EnvironmentStore {
|
|
|
1490
1906
|
linkType,
|
|
1491
1907
|
);
|
|
1492
1908
|
logger.info(
|
|
1493
|
-
`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`,
|
|
1909
|
+
`In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}" (${++mountedCount}/${totalPackages})`,
|
|
1494
1910
|
);
|
|
1495
1911
|
} catch (linkError) {
|
|
1496
1912
|
// Degrade gracefully instead of failing the env load:
|
|
@@ -1511,6 +1927,9 @@ export class EnvironmentStore {
|
|
|
1511
1927
|
await fs.promises.cp(sourcePath, absolutePackagePath, {
|
|
1512
1928
|
recursive: true,
|
|
1513
1929
|
});
|
|
1930
|
+
logger.info(
|
|
1931
|
+
`Copied package "${packageDir}" (${++mountedCount}/${totalPackages})`,
|
|
1932
|
+
);
|
|
1514
1933
|
}
|
|
1515
1934
|
} else {
|
|
1516
1935
|
if (
|
|
@@ -1532,7 +1951,7 @@ export class EnvironmentStore {
|
|
|
1532
1951
|
recursive: true,
|
|
1533
1952
|
});
|
|
1534
1953
|
logger.info(
|
|
1535
|
-
`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`,
|
|
1954
|
+
`Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"} (${++mountedCount}/${totalPackages})`,
|
|
1536
1955
|
);
|
|
1537
1956
|
}
|
|
1538
1957
|
} else {
|
|
@@ -1545,7 +1964,7 @@ export class EnvironmentStore {
|
|
|
1545
1964
|
recursive: true,
|
|
1546
1965
|
});
|
|
1547
1966
|
logger.info(
|
|
1548
|
-
`Copied entire download as package "${packageDir}"`,
|
|
1967
|
+
`Copied entire download as package "${packageDir}" (${++mountedCount}/${totalPackages})`,
|
|
1549
1968
|
);
|
|
1550
1969
|
}
|
|
1551
1970
|
} catch (error) {
|
|
@@ -1590,6 +2009,7 @@ export class EnvironmentStore {
|
|
|
1590
2009
|
targetPath: string,
|
|
1591
2010
|
environmentName: string,
|
|
1592
2011
|
packageName: string,
|
|
2012
|
+
packageNames?: string[],
|
|
1593
2013
|
) {
|
|
1594
2014
|
const isCompressedFile = location.endsWith(".zip");
|
|
1595
2015
|
// Handle GCS paths
|
|
@@ -1623,7 +2043,10 @@ export class EnvironmentStore {
|
|
|
1623
2043
|
logger.info(
|
|
1624
2044
|
`Cloning GitHub repository from "${location}" to "${targetPath}"`,
|
|
1625
2045
|
);
|
|
1626
|
-
await this.downloadGitHubDirectory(location, targetPath
|
|
2046
|
+
await this.downloadGitHubDirectory(location, targetPath, {
|
|
2047
|
+
environmentName,
|
|
2048
|
+
packageNames,
|
|
2049
|
+
});
|
|
1627
2050
|
return;
|
|
1628
2051
|
} catch (error) {
|
|
1629
2052
|
const errorData = this.extractErrorDataFromError(error);
|
|
@@ -1876,35 +2299,15 @@ export class EnvironmentStore {
|
|
|
1876
2299
|
logger.info(`Downloaded S3 directory ${s3Path} to ${absoluteDirPath}`);
|
|
1877
2300
|
}
|
|
1878
2301
|
|
|
1879
|
-
|
|
2302
|
+
async downloadGitHubDirectory(
|
|
1880
2303
|
githubUrl: string,
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
/github\.com\/(?<owner>[^/]+)\/(?<repoName>[^/]+)(?<packagePath>\/[^/]+)*/;
|
|
1885
|
-
const httpsMatch = githubUrl.match(httpsRegex);
|
|
1886
|
-
if (httpsMatch) {
|
|
1887
|
-
const { owner, repoName, packagePath } = httpsMatch.groups!;
|
|
1888
|
-
return { owner, repoName, packagePath };
|
|
1889
|
-
}
|
|
1890
|
-
|
|
1891
|
-
// Handle SSH format: git@github.com:owner/repo.git or git@github.com:owner/repo
|
|
1892
|
-
const sshRegex =
|
|
1893
|
-
/git@github\.com:(?<owner>[^/]+)\/(?<repoName>[^/\s]+?)(?:\.git)?(?<packagePath>\/[^/]+)*$/;
|
|
1894
|
-
const sshMatch = githubUrl.match(sshRegex);
|
|
1895
|
-
if (sshMatch) {
|
|
1896
|
-
const { owner, repoName, packagePath } = sshMatch.groups!;
|
|
1897
|
-
return { owner, repoName, packagePath };
|
|
1898
|
-
}
|
|
1899
|
-
|
|
1900
|
-
return null;
|
|
1901
|
-
}
|
|
1902
|
-
|
|
1903
|
-
async downloadGitHubDirectory(githubUrl: string, absoluteDirPath: string) {
|
|
2304
|
+
absoluteDirPath: string,
|
|
2305
|
+
progressContext?: { environmentName?: string; packageNames?: string[] },
|
|
2306
|
+
) {
|
|
1904
2307
|
assertSafeEnvironmentPath(absoluteDirPath);
|
|
1905
2308
|
// First we'll clone the repo without the additional path
|
|
1906
2309
|
// E.g. we're removing `/tree/main/imdb` from https://github.com/credibledata/malloy-samples/tree/main/imdb
|
|
1907
|
-
const githubInfo =
|
|
2310
|
+
const githubInfo = parseGitHubUrl(githubUrl);
|
|
1908
2311
|
if (!githubInfo) {
|
|
1909
2312
|
throw new Error(`Invalid GitHub URL: ${githubUrl}`);
|
|
1910
2313
|
}
|
|
@@ -1923,17 +2326,30 @@ export class EnvironmentStore {
|
|
|
1923
2326
|
});
|
|
1924
2327
|
await fs.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
1925
2328
|
const repoUrl = `https://github.com/${owner}/${repoName}`;
|
|
2329
|
+
const reporter = new CloneProgressReporter(
|
|
2330
|
+
cloneProgressLabel(`${owner}/${repoName}`, progressContext),
|
|
2331
|
+
);
|
|
1926
2332
|
|
|
1927
2333
|
// We'll clone the repo into absoluteDirPath
|
|
1928
2334
|
await new Promise<void>((resolve, reject) => {
|
|
1929
|
-
simpleGit(
|
|
2335
|
+
simpleGit({
|
|
2336
|
+
progress: (event) => reporter.onProgress(event),
|
|
2337
|
+
}).clone(repoUrl, absoluteDirPath, GIT_CLONE_OPTIONS, (err) => {
|
|
2338
|
+
reporter.done();
|
|
1930
2339
|
if (err) {
|
|
2340
|
+
err.message = stripGitProgressNoise(err.message);
|
|
2341
|
+
if (err.stack) {
|
|
2342
|
+
// V8 captures the message into the stack at construction,
|
|
2343
|
+
// so the spew survives there unless stripped too.
|
|
2344
|
+
err.stack = stripGitProgressNoise(err.stack);
|
|
2345
|
+
}
|
|
1931
2346
|
const errorData = this.extractErrorDataFromError(err);
|
|
1932
2347
|
logger.error(
|
|
1933
2348
|
`Failed to clone GitHub repository "${repoUrl}"`,
|
|
1934
2349
|
errorData,
|
|
1935
2350
|
);
|
|
1936
2351
|
reject(err);
|
|
2352
|
+
return;
|
|
1937
2353
|
}
|
|
1938
2354
|
resolve();
|
|
1939
2355
|
});
|