@malloy-publisher/server 0.0.230 → 0.0.232

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/README.docker.md +4 -0
  2. package/dist/app/api-doc.yaml +74 -3
  3. package/dist/app/assets/{EnvironmentPage-wa_EPkwK.js → EnvironmentPage-DXEaZIPx.js} +1 -1
  4. package/dist/app/assets/{HomePage-jnCrupQp.js → HomePage-kofsqpZt.js} +1 -1
  5. package/dist/app/assets/{LightMode-DYbwNULZ.js → LightMode-CNhIlIlJ.js} +1 -1
  6. package/dist/app/assets/{MainPage-CuJLrPNI.js → MainPage-Bgqo8jCy.js} +1 -1
  7. package/dist/app/assets/{MaterializationsPage-D_67x2ee.js → MaterializationsPage-CgBlgGz2.js} +1 -1
  8. package/dist/app/assets/{ModelPage-D5JtAWqR.js → ModelPage-B0TjoDtf.js} +1 -1
  9. package/dist/app/assets/{PackagePage-BRwtqUSG.js → PackagePage-BL8vnFj1.js} +1 -1
  10. package/dist/app/assets/{RouteError-CBNNrnSD.js → RouteError-BzPby0X2.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-CTCeBneA.js → ThemeEditorPage-CTEP_9r3.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-SN6f1RBm.js → WorkbookPage-BwM3BmKw.js} +1 -1
  13. package/dist/app/assets/{core-Dp3q5Ieu.es-CD5FvM2s.js → core-CK68iv6w.es-CpRxXBt7.js} +1 -1
  14. package/dist/app/assets/{index-C_tJstcx.js → index-B33zGctF.js} +15 -15
  15. package/dist/app/assets/{index-DU4r7GdU.js → index-BabP-V-S.js} +346 -321
  16. package/dist/app/assets/{index-B3Nn8Vm2.js → index-BkiWKaAF.js} +266 -265
  17. package/dist/app/assets/{index-BLCx1EdC.js → index-CmkW1MiE.js} +1 -1
  18. package/dist/app/assets/{index-CfmBVB6M.js → index-tXJXwdyj.js} +1 -1
  19. package/dist/app/index.html +1 -1
  20. package/dist/instrumentation.mjs +2 -0
  21. package/dist/package_load_worker.mjs +11 -1
  22. package/dist/server.mjs +3186 -433
  23. package/package.json +12 -12
  24. package/scripts/bake-duckdb-extensions.js +4 -1
  25. package/src/config.spec.ts +39 -0
  26. package/src/config.ts +135 -0
  27. package/src/controller/materialization.controller.spec.ts +62 -0
  28. package/src/controller/materialization.controller.ts +15 -0
  29. package/src/controller/package.controller.spec.ts +6 -0
  30. package/src/controller/package.controller.ts +7 -2
  31. package/src/errors.ts +19 -0
  32. package/src/logger.spec.ts +18 -1
  33. package/src/logger.ts +3 -1
  34. package/src/materialization_metrics.spec.ts +89 -4
  35. package/src/materialization_metrics.ts +155 -5
  36. package/src/mcp/skills/skills_bundle.json +1 -1
  37. package/src/mcp/tools/embedding_index.spec.ts +1236 -0
  38. package/src/mcp/tools/embedding_index.ts +808 -0
  39. package/src/mcp/tools/get_context_eval.ts +194 -45
  40. package/src/mcp/tools/get_context_tool.spec.ts +295 -2
  41. package/src/mcp/tools/get_context_tool.ts +159 -10
  42. package/src/pg_helpers.spec.ts +201 -0
  43. package/src/pg_helpers.ts +44 -5
  44. package/src/server.ts +24 -0
  45. package/src/service/build_plan.spec.ts +128 -2
  46. package/src/service/build_plan.ts +239 -17
  47. package/src/service/connection.ts +263 -7
  48. package/src/service/connection_config.spec.ts +48 -0
  49. package/src/service/connection_config.ts +21 -8
  50. package/src/service/connection_federation.spec.ts +184 -0
  51. package/src/service/embedding_provider.spec.ts +329 -0
  52. package/src/service/embedding_provider.ts +236 -0
  53. package/src/service/environment.ts +274 -12
  54. package/src/service/environment_store.spec.ts +678 -3
  55. package/src/service/environment_store.ts +449 -33
  56. package/src/service/environment_store_clone.spec.ts +350 -0
  57. package/src/service/manifest_loader.spec.ts +68 -13
  58. package/src/service/manifest_loader.ts +67 -19
  59. package/src/service/materialization_build_session.spec.ts +435 -0
  60. package/src/service/materialization_build_session.ts +681 -0
  61. package/src/service/materialization_eligibility.spec.ts +158 -0
  62. package/src/service/materialization_eligibility.ts +305 -0
  63. package/src/service/materialization_serve_transform.spec.ts +1003 -0
  64. package/src/service/materialization_serve_transform.ts +779 -0
  65. package/src/service/materialization_service.spec.ts +774 -7
  66. package/src/service/materialization_service.ts +1107 -42
  67. package/src/service/materialization_test_fixtures.ts +7 -0
  68. package/src/service/model.spec.ts +207 -0
  69. package/src/service/model.ts +540 -52
  70. package/src/service/model_storage_serve.spec.ts +193 -0
  71. package/src/service/model_storage_serve_joins.spec.ts +193 -0
  72. package/src/service/package.spec.ts +196 -0
  73. package/src/service/package.ts +385 -17
  74. package/src/service/persistence_policy.spec.ts +109 -0
  75. package/src/storage/duckdb/schema.ts +37 -0
  76. package/tests/fixtures/xlsx/database.xlsx +0 -0
  77. package/tests/integration/first_boot/readiness_line.integration.spec.ts +177 -0
  78. package/tests/integration/materialization/manifest_binding.integration.spec.ts +104 -0
  79. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +235 -0
@@ -6,7 +6,7 @@ import extract from "extract-zip";
6
6
  import * as fs from "fs";
7
7
  import * as os from "os";
8
8
  import * as path from "path";
9
- import simpleGit from "simple-git";
9
+ import simpleGit, { type SimpleGitProgressEvent } from "simple-git";
10
10
  import { Writable } from "stream";
11
11
  import { components } from "../api";
12
12
  import {
@@ -29,6 +29,10 @@ import {
29
29
  } from "../errors";
30
30
  import { getOperationalState, markNotReady, markReady } from "../health";
31
31
  import { formatDuration, logger } from "../logger";
32
+ import {
33
+ deleteEnvironmentEmbeddings,
34
+ deletePackageEmbeddings,
35
+ } from "../mcp/tools/embedding_index";
32
36
  import { redactPgSecrets } from "../pg_helpers";
33
37
  import {
34
38
  assertSafeEnvironmentPath,
@@ -151,6 +155,281 @@ export function resolvePackageLocation(
151
155
  return path.isAbsolute(expanded) ? expanded : path.join(anchorDir, expanded);
152
156
  }
153
157
 
158
+ /**
159
+ * Options for every package git clone. Packages are served from the default
160
+ * branch's working tree only (no ref is ever checked out afterwards, and no
161
+ * later git operation touches the clone: a reload deletes and re-clones), so
162
+ * history is pure download cost. Exported for tests.
163
+ */
164
+ export const GIT_CLONE_OPTIONS: Record<string, string | number | null> = {
165
+ "--depth": 1,
166
+ "--single-branch": null,
167
+ };
168
+
169
+ /**
170
+ * With a progress handler configured, simple-git forces `--progress`, and a
171
+ * failed clone's error message then carries every progress line git wrote to
172
+ * stderr. Those messages surface in API error responses on the add/update
173
+ * package path and in the failure logs (the boot path's `/status` loadErrors
174
+ * carries a location-level summary instead), so drop the noise and keep the
175
+ * lines that say what went wrong. Returns the input when everything matched,
176
+ * rather than an empty error.
177
+ */
178
+ export function stripGitProgressNoise(message: string): string {
179
+ const kept = message
180
+ .split(/\r\n|\r|\n/)
181
+ .filter(
182
+ (line) =>
183
+ !/\d+% \(\d+\/\d+\)/.test(line) &&
184
+ // Also matches the "Error: Cloning into ..." shape V8 seeds
185
+ // into err.stack from the message's first line.
186
+ !/^(\w+: )?Cloning into /.test(line.trim()),
187
+ )
188
+ .join("\n")
189
+ .trim();
190
+ return kept.length > 0 ? kept : message.trim();
191
+ }
192
+
193
+ /** e.g. `[examples] cloning malloydata/publisher (storefront, +2 more)`. */
194
+ export function cloneProgressLabel(
195
+ repo: string,
196
+ context?: { environmentName?: string; packageNames?: string[] },
197
+ ): string {
198
+ const env = context?.environmentName ? `[${context.environmentName}] ` : "";
199
+ const names = context?.packageNames ?? [];
200
+ const shown = names.length > 4 ? names.slice(0, 3) : names;
201
+ const more = names.length - shown.length;
202
+ const pkgs =
203
+ shown.length > 0
204
+ ? ` (${shown.join(", ")}${more > 0 ? `, +${more} more` : ""})`
205
+ : "";
206
+ return `${env}cloning ${repo}${pkgs}`;
207
+ }
208
+
209
+ /**
210
+ * Streams git clone progress to stderr: off winston (whose format changes
211
+ * under OTEL and whose level can hide info lines) and off stdout (which an
212
+ * MCP stdio transport would own), and it is where git itself reports
213
+ * progress. A TTY gets one line rewritten in place; anything else (CI logs)
214
+ * gets a line per stage and per 25-point step, so a slow clone shows life
215
+ * without flooding. Exported for tests.
216
+ */
217
+ export class CloneProgressReporter {
218
+ private lastStage: string | undefined;
219
+ private lastMilestone = -1;
220
+ private wroteInPlace = false;
221
+ private finished = false;
222
+ /**
223
+ * One reporter at a time owns a TTY's in-place line: environments load
224
+ * concurrently at boot, and two reporters `\r`-rewriting the same line
225
+ * would overwrite each other. Later reporters fall back to milestone
226
+ * lines until the owner finishes. Keyed by stream so tests with fake
227
+ * streams cannot collide.
228
+ */
229
+ private static ttyOwners = new WeakMap<object, CloneProgressReporter>();
230
+
231
+ constructor(
232
+ private readonly label: string,
233
+ private readonly out: NodeJS.WriteStream = process.stderr,
234
+ ) {}
235
+
236
+ onProgress(event: SimpleGitProgressEvent): void {
237
+ if (this.finished) {
238
+ // A late event after done() must not re-claim the TTY line the
239
+ // finishing newline just released.
240
+ return;
241
+ }
242
+ const { stage, progress, processed, total } = event;
243
+ const counts = total > 0 ? ` (${processed}/${total})` : "";
244
+ const text = `${this.label}: ${stage} ${progress}%${counts}`;
245
+ if (this.out.isTTY === true && this.claimTtyLine()) {
246
+ // Clamped to the terminal width: an auto-wrapped line breaks the
247
+ // \r rewrite (git clamps its own progress output for the same
248
+ // reason). The moving part sits at the end, so shorten the label
249
+ // and keep the stage/percentage visible; only when even the
250
+ // suffix cannot fit does a plain head slice apply. \x1b[K clears
251
+ // the tail of a longer previous line.
252
+ const budget = Math.max(1, (this.out.columns || 80) - 1);
253
+ const suffix = `: ${stage} ${progress}%${counts}`;
254
+ let line: string;
255
+ if (text.length <= budget) {
256
+ line = text;
257
+ } else if (suffix.length + 4 <= budget) {
258
+ const labelBudget = budget - suffix.length;
259
+ line = `${this.label.slice(0, labelBudget - 3)}...${suffix}`;
260
+ } else {
261
+ line = text.slice(0, budget);
262
+ }
263
+ this.out.write(`\r${line}\x1b[K`);
264
+ this.wroteInPlace = true;
265
+ return;
266
+ }
267
+ const milestone = Math.floor(progress / 25);
268
+ if (stage !== this.lastStage || milestone < this.lastMilestone) {
269
+ // A new stage, or a percentage restart inside one reported stage:
270
+ // git's server-side counting and compressing phases both arrive as
271
+ // stage "remote:", so a drop in progress marks the next sub-phase.
272
+ this.lastStage = stage;
273
+ this.lastMilestone = milestone;
274
+ } else if (milestone > this.lastMilestone) {
275
+ this.lastMilestone = milestone;
276
+ } else {
277
+ return;
278
+ }
279
+ if (this.out.isTTY === true) {
280
+ // A non-owner on a TTY: the cursor may sit mid-line on the owner's
281
+ // in-place output, so take over the row cleanly and scroll it.
282
+ this.out.write(`\r${text}\x1b[K\n`);
283
+ return;
284
+ }
285
+ this.out.write(`${text}\n`);
286
+ }
287
+
288
+ done(): void {
289
+ this.finished = true;
290
+ if (this.wroteInPlace) {
291
+ this.out.write("\n");
292
+ }
293
+ if (CloneProgressReporter.ttyOwners.get(this.out) === this) {
294
+ CloneProgressReporter.ttyOwners.delete(this.out);
295
+ }
296
+ }
297
+
298
+ private claimTtyLine(): boolean {
299
+ const owner = CloneProgressReporter.ttyOwners.get(this.out);
300
+ if (owner === undefined) {
301
+ CloneProgressReporter.ttyOwners.set(this.out, this);
302
+ return true;
303
+ }
304
+ return owner === this;
305
+ }
306
+ }
307
+
308
+ /**
309
+ * The last `/segment` of a run of non-empty path segments at the start of
310
+ * `afterRepo`, or undefined. Reproduces the capture of the old
311
+ * `(\/[^/]+)*` group, which retained only its final iteration and stopped at
312
+ * the first empty segment.
313
+ */
314
+ function lastGitHubPathSegment(afterRepo: string): string | undefined {
315
+ if (!afterRepo.startsWith("/")) return undefined;
316
+ const run: string[] = [];
317
+ for (const seg of afterRepo.slice(1).split("/")) {
318
+ if (seg === "") break;
319
+ run.push(seg);
320
+ }
321
+ return run.length > 0 ? `/${run[run.length - 1]}` : undefined;
322
+ }
323
+
324
+ /**
325
+ * Parse a GitHub package location into owner, repo, and (optionally) the
326
+ * subdirectory path segment.
327
+ *
328
+ * Uses a repetition-free regex for owner/repo plus plain string ops for the
329
+ * path, NOT a single `(\/[^/]+)*` regex: those shapes were flagged as
330
+ * polynomial-backtracking (ReDoS) on adversarial input, and the location is
331
+ * user/config supplied. Behavior is preserved exactly, including
332
+ * `packagePath` being only the LAST path segment after the repo (a quirk the
333
+ * callers rely on); environment_store.spec.ts fuzzes the old regexes as an
334
+ * oracle to prove equivalence. Exported for that test.
335
+ */
336
+ export function parseGitHubUrl(
337
+ githubUrl: string,
338
+ ): { owner: string; repoName: string; packagePath?: string } | null {
339
+ // HTTPS: https://github.com/owner/repo/tree/branch/subdir. The host may
340
+ // appear anywhere (the old regex was unanchored); the owner/repo pattern
341
+ // has no repetition group, so it cannot backtrack polynomially.
342
+ const httpsMatch = /github\.com\/([^/]+)\/([^/]+)/.exec(githubUrl);
343
+ if (httpsMatch) {
344
+ const afterRepo = githubUrl.slice(
345
+ httpsMatch.index + httpsMatch[0].length,
346
+ );
347
+ return {
348
+ owner: httpsMatch[1],
349
+ repoName: httpsMatch[2],
350
+ packagePath: lastGitHubPathSegment(afterRepo),
351
+ };
352
+ }
353
+
354
+ // SSH: git@github.com:owner/repo(.git)?(/subdir)*. The old regex was
355
+ // $-anchored, so the whole tail after the repo had to be a run of
356
+ // non-empty `/segment`s; a trailing slash or empty segment matched nothing
357
+ // and failed the parse.
358
+ // `git@github.com:` may appear anywhere (the old regex was not anchored at
359
+ // the start), though in practice isGitHubURL only routes locations that
360
+ // begin with it here.
361
+ const SSH_PREFIX = "git@github.com:";
362
+ const sshAt = githubUrl.indexOf(SSH_PREFIX);
363
+ if (sshAt !== -1) {
364
+ const rest = githubUrl.slice(sshAt + SSH_PREFIX.length);
365
+ const firstSlash = rest.indexOf("/");
366
+ if (firstSlash > 0) {
367
+ const owner = rest.slice(0, firstSlash);
368
+ const afterOwner = rest.slice(firstSlash + 1);
369
+ const secondSlash = afterOwner.indexOf("/");
370
+ const repoSeg =
371
+ secondSlash === -1 ? afterOwner : afterOwner.slice(0, secondSlash);
372
+ const afterRepo =
373
+ secondSlash === -1 ? "" : afterOwner.slice(secondSlash);
374
+ // repo was `[^/\s]+` (no whitespace); anything else falls through to
375
+ // "not a GitHub URL", as the old regex did.
376
+ if (repoSeg.length > 0 && !/\s/.test(repoSeg)) {
377
+ const repoName =
378
+ repoSeg.length > 4 && repoSeg.endsWith(".git")
379
+ ? repoSeg.slice(0, -4)
380
+ : repoSeg;
381
+ if (afterRepo === "") {
382
+ return { owner, repoName };
383
+ }
384
+ const segments = afterRepo.slice(1).split("/");
385
+ // $-anchored: every trailing segment had to be non-empty.
386
+ if (!segments.includes("")) {
387
+ return {
388
+ owner,
389
+ repoName,
390
+ packagePath: `/${segments[segments.length - 1]}`,
391
+ };
392
+ }
393
+ }
394
+ }
395
+ }
396
+
397
+ return null;
398
+ }
399
+
400
+ /** `0.0.0.0`/`::` bind every interface; a script needs a host it can dial. */
401
+ function displayHost(host: string): string {
402
+ if (host === "0.0.0.0" || host === "::") {
403
+ return "localhost";
404
+ }
405
+ // An IPv6 literal needs brackets to be dialable inside a URL.
406
+ return host.includes(":") ? `[${host}]` : host;
407
+ }
408
+
409
+ /**
410
+ * The one-line machine-readable boot signal. Scripts grep for the
411
+ * `PUBLISHER_READY` prefix instead of polling `/api/v0/status`, and the
412
+ * counts carry what `operationalState: "serving"` alone does not say: how
413
+ * much of the configured world actually loaded. Host and ports mirror
414
+ * server.ts's use-site defaults; parseArgs writes the flag values into
415
+ * process.env before the store is constructed. Exported for tests.
416
+ */
417
+ export function formatReadinessLine(counts: {
418
+ environments: number;
419
+ packages: number;
420
+ loadErrors: number;
421
+ }): string {
422
+ const host = displayHost(process.env.PUBLISHER_HOST || "0.0.0.0");
423
+ const port = Number(process.env.PUBLISHER_PORT || 4000);
424
+ const mcpPort = Number(process.env.MCP_PORT || 4040);
425
+ return (
426
+ `PUBLISHER_READY url=http://${host}:${port} ` +
427
+ `mcp=http://${host}:${mcpPort} ` +
428
+ `environments=${counts.environments} packages=${counts.packages} ` +
429
+ `load_errors=${counts.loadErrors}`
430
+ );
431
+ }
432
+
154
433
  export class EnvironmentStore {
155
434
  public serverRootPath: string;
156
435
  private environments: Map<string, Environment> = new Map();
@@ -404,6 +683,28 @@ export class EnvironmentStore {
404
683
  environmentInstance.setMemoryGovernor(
405
684
  this.memoryGovernor,
406
685
  );
686
+ // Re-establish serve routing when a package loads, from
687
+ // its latest successful materialization — so serving
688
+ // survives a restart, not only a fresh build. The full
689
+ // entry map is returned; the environment splits it by
690
+ // tier (colocated + storage=) in
691
+ // rebindServeBindingsFromLocalStore.
692
+ const envId = dbEnvironment.id;
693
+ environmentInstance.setStorageBindingResolver(
694
+ async (packageName) => {
695
+ const runs =
696
+ await repository.listMaterializations(
697
+ envId,
698
+ packageName,
699
+ );
700
+ const latest = runs.find(
701
+ (m) =>
702
+ m.status === "MANIFEST_FILE_READY" &&
703
+ m.manifest?.entries,
704
+ );
705
+ return latest?.manifest?.entries ?? {};
706
+ },
707
+ );
407
708
 
408
709
  // Get packages from database
409
710
  const packages = await repository.listPackages(
@@ -446,10 +747,55 @@ export class EnvironmentStore {
446
747
  logger.info(
447
748
  `Environment store successfully initialized in ${formatDuration(initializationDuration)}`,
448
749
  );
750
+ this.emitReadinessLine();
449
751
  } catch (error) {
450
752
  markNotReady();
451
753
  const errorData = this.extractErrorDataFromError(error);
452
754
  logger.error("Error initializing environment store", errorData);
755
+ try {
756
+ // The failure counterpart of PUBLISHER_READY, so a script
757
+ // waiting on that token fails fast instead of hanging on a
758
+ // broken config. Redacted because initialization errors can
759
+ // carry connection strings.
760
+ process.stderr.write(
761
+ `PUBLISHER_INIT_FAILED error=${JSON.stringify(
762
+ redactPgSecrets(
763
+ error instanceof Error ? error.message : String(error),
764
+ ),
765
+ )}\n`,
766
+ );
767
+ } catch {
768
+ // A failed stderr write must not mask the logged error above.
769
+ }
770
+ }
771
+ }
772
+
773
+ /**
774
+ * Printed exactly once, from initialize()'s success tail: markReady() has
775
+ * just flipped /api/v0/status to "serving", and both HTTP listeners bound
776
+ * long before the downloads finished, so this is the same moment every
777
+ * existing readiness consumer (CI status polls, MCP /health/readiness)
778
+ * fires on. The catch above swallows a failed boot after markNotReady(),
779
+ * so the line never prints for a server that is not actually serving.
780
+ */
781
+ private emitReadinessLine(): void {
782
+ let packages = 0;
783
+ let loadErrors = this.failedEnvironments.size;
784
+ for (const environment of this.environments.values()) {
785
+ packages += environment.getServingPackageCount();
786
+ loadErrors += environment.getFailedPackages().size;
787
+ }
788
+ try {
789
+ process.stderr.write(
790
+ formatReadinessLine({
791
+ environments: this.environments.size,
792
+ packages,
793
+ loadErrors,
794
+ }) + "\n",
795
+ );
796
+ } catch {
797
+ // A failed stderr write must not fail a boot that is already
798
+ // serving.
453
799
  }
454
800
  }
455
801
 
@@ -486,6 +832,10 @@ export class EnvironmentStore {
486
832
  public async deleteEnvironmentFromDatabase(
487
833
  environmentName: string,
488
834
  ): Promise<void> {
835
+ // Before the metadata lookup, for the same reason as the package
836
+ // variant: the cleanup needs only the name, never the row.
837
+ this.cleanupEnvironmentEmbeddings(environmentName);
838
+
489
839
  const repository = this.storageManager.getRepository();
490
840
 
491
841
  // Get the environment from database
@@ -755,6 +1105,11 @@ export class EnvironmentStore {
755
1105
  environmentName: string,
756
1106
  packageName: string,
757
1107
  ): Promise<void> {
1108
+ // Before the metadata lookups: embeddings are written off the
1109
+ // in-memory environment, which can outlive a lost metadata row,
1110
+ // so the cleanup must not sit behind the missing-row early return.
1111
+ this.cleanupPackageEmbeddings(environmentName, packageName);
1112
+
758
1113
  const repository = this.storageManager.getRepository();
759
1114
 
760
1115
  // Get the environment ID from database
@@ -778,6 +1133,64 @@ export class EnvironmentStore {
778
1133
  }
779
1134
  }
780
1135
 
1136
+ /**
1137
+ * Best-effort, non-blocking: drop a deleted package's cached entity
1138
+ * embeddings so package churn does not grow publisher.db forever.
1139
+ * Not awaited by callers: the cleanup queues on the package's sync
1140
+ * mutex, which an in-flight bulk embed can hold for minutes, and a
1141
+ * DELETE response must not stall behind it. Needs only the names, so
1142
+ * it runs regardless of whether the metadata row still exists. An
1143
+ * orphaned row is inert (reads are scoped by environment + package),
1144
+ * so failure is logged, never propagated.
1145
+ */
1146
+ private cleanupPackageEmbeddings(
1147
+ environmentName: string,
1148
+ packageName: string,
1149
+ ): void {
1150
+ try {
1151
+ deletePackageEmbeddings(
1152
+ this.storageManager.getDuckDbConnection(),
1153
+ environmentName,
1154
+ packageName,
1155
+ ).catch((error: unknown) => {
1156
+ logger.warn("Failed to clean up entity embeddings for package", {
1157
+ environmentName,
1158
+ packageName,
1159
+ error: error instanceof Error ? error.message : String(error),
1160
+ });
1161
+ });
1162
+ } catch (error) {
1163
+ logger.warn("Failed to clean up entity embeddings for package", {
1164
+ environmentName,
1165
+ packageName,
1166
+ error: error instanceof Error ? error.message : String(error),
1167
+ });
1168
+ }
1169
+ }
1170
+
1171
+ /** Environment-wide variant of {@link cleanupPackageEmbeddings}. */
1172
+ private cleanupEnvironmentEmbeddings(environmentName: string): void {
1173
+ try {
1174
+ deleteEnvironmentEmbeddings(
1175
+ this.storageManager.getDuckDbConnection(),
1176
+ environmentName,
1177
+ ).catch((error: unknown) => {
1178
+ logger.warn(
1179
+ "Failed to clean up entity embeddings for environment",
1180
+ {
1181
+ environmentName,
1182
+ error: error instanceof Error ? error.message : String(error),
1183
+ },
1184
+ );
1185
+ });
1186
+ } catch (error) {
1187
+ logger.warn("Failed to clean up entity embeddings for environment", {
1188
+ environmentName,
1189
+ error: error instanceof Error ? error.message : String(error),
1190
+ });
1191
+ }
1192
+ }
1193
+
781
1194
  private async cleanupAndCreatePublisherPath() {
782
1195
  const reInit = process.env.INITIALIZE_STORAGE === "true";
783
1196
 
@@ -1347,7 +1760,7 @@ export class EnvironmentStore {
1347
1760
  // For GitHub URLs, group by base repository URL to optimize downloads
1348
1761
  let locationKey = _package.location;
1349
1762
  if (this.isGitHubURL(_package.location)) {
1350
- const githubInfo = this.parseGitHubUrl(_package.location);
1763
+ const githubInfo = parseGitHubUrl(_package.location);
1351
1764
  if (githubInfo) {
1352
1765
  // Always use HTTPS format for grouping to ensure consistency
1353
1766
  locationKey = `https://github.com/${githubInfo.owner}/${githubInfo.repoName}`;
@@ -1364,6 +1777,8 @@ export class EnvironmentStore {
1364
1777
  }
1365
1778
 
1366
1779
  // Processing by each unique location
1780
+ const totalPackages = packages.length;
1781
+ let mountedCount = 0;
1367
1782
  for (const [groupedLocation, packagesForLocation] of locationGroups) {
1368
1783
  // Use a hash instead of base64 to keep paths short (Windows MAX_PATH limit)
1369
1784
  // This works for both local and remote paths
@@ -1390,6 +1805,7 @@ export class EnvironmentStore {
1390
1805
  tempDownloadPath,
1391
1806
  environmentName,
1392
1807
  "shared",
1808
+ packagesForLocation.map((p) => p.name),
1393
1809
  );
1394
1810
  } catch (error) {
1395
1811
  downloaded = false;
@@ -1419,7 +1835,7 @@ export class EnvironmentStore {
1419
1835
  // For GitHub URLs, extract the subdirectory path from the original location
1420
1836
  let sourcePath: string;
1421
1837
  if (this.isGitHubURL(_package.location)) {
1422
- const githubInfo = this.parseGitHubUrl(_package.location);
1838
+ const githubInfo = parseGitHubUrl(_package.location);
1423
1839
  if (githubInfo && githubInfo.packagePath) {
1424
1840
  // Extract subdirectory from the original GitHub URL
1425
1841
  // Handle both /tree/main/subdir and /tree/branch/subdir cases
@@ -1490,7 +1906,7 @@ export class EnvironmentStore {
1490
1906
  linkType,
1491
1907
  );
1492
1908
  logger.info(
1493
- `In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}"`,
1909
+ `In-place mount (watch mode): linked package "${packageDir}" -> "${absoluteSourcePath}" (${++mountedCount}/${totalPackages})`,
1494
1910
  );
1495
1911
  } catch (linkError) {
1496
1912
  // Degrade gracefully instead of failing the env load:
@@ -1511,6 +1927,9 @@ export class EnvironmentStore {
1511
1927
  await fs.promises.cp(sourcePath, absolutePackagePath, {
1512
1928
  recursive: true,
1513
1929
  });
1930
+ logger.info(
1931
+ `Copied package "${packageDir}" (${++mountedCount}/${totalPackages})`,
1932
+ );
1514
1933
  }
1515
1934
  } else {
1516
1935
  if (
@@ -1532,7 +1951,7 @@ export class EnvironmentStore {
1532
1951
  recursive: true,
1533
1952
  });
1534
1953
  logger.info(
1535
- `Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"}`,
1954
+ `Extracted package "${packageDir}" from ${groupedLocation.startsWith("https://github.com/") && _package.location.includes("/tree/") ? "GitHub subdirectory" : "shared download"} (${++mountedCount}/${totalPackages})`,
1536
1955
  );
1537
1956
  }
1538
1957
  } else {
@@ -1545,7 +1964,7 @@ export class EnvironmentStore {
1545
1964
  recursive: true,
1546
1965
  });
1547
1966
  logger.info(
1548
- `Copied entire download as package "${packageDir}"`,
1967
+ `Copied entire download as package "${packageDir}" (${++mountedCount}/${totalPackages})`,
1549
1968
  );
1550
1969
  }
1551
1970
  } catch (error) {
@@ -1590,6 +2009,7 @@ export class EnvironmentStore {
1590
2009
  targetPath: string,
1591
2010
  environmentName: string,
1592
2011
  packageName: string,
2012
+ packageNames?: string[],
1593
2013
  ) {
1594
2014
  const isCompressedFile = location.endsWith(".zip");
1595
2015
  // Handle GCS paths
@@ -1623,7 +2043,10 @@ export class EnvironmentStore {
1623
2043
  logger.info(
1624
2044
  `Cloning GitHub repository from "${location}" to "${targetPath}"`,
1625
2045
  );
1626
- await this.downloadGitHubDirectory(location, targetPath);
2046
+ await this.downloadGitHubDirectory(location, targetPath, {
2047
+ environmentName,
2048
+ packageNames,
2049
+ });
1627
2050
  return;
1628
2051
  } catch (error) {
1629
2052
  const errorData = this.extractErrorDataFromError(error);
@@ -1876,35 +2299,15 @@ export class EnvironmentStore {
1876
2299
  logger.info(`Downloaded S3 directory ${s3Path} to ${absoluteDirPath}`);
1877
2300
  }
1878
2301
 
1879
- private parseGitHubUrl(
2302
+ async downloadGitHubDirectory(
1880
2303
  githubUrl: string,
1881
- ): { owner: string; repoName: string; packagePath?: string } | null {
1882
- // Handle HTTPS format: https://github.com/owner/repo/tree/branch/subdir
1883
- const httpsRegex =
1884
- /github\.com\/(?<owner>[^/]+)\/(?<repoName>[^/]+)(?<packagePath>\/[^/]+)*/;
1885
- const httpsMatch = githubUrl.match(httpsRegex);
1886
- if (httpsMatch) {
1887
- const { owner, repoName, packagePath } = httpsMatch.groups!;
1888
- return { owner, repoName, packagePath };
1889
- }
1890
-
1891
- // Handle SSH format: git@github.com:owner/repo.git or git@github.com:owner/repo
1892
- const sshRegex =
1893
- /git@github\.com:(?<owner>[^/]+)\/(?<repoName>[^/\s]+?)(?:\.git)?(?<packagePath>\/[^/]+)*$/;
1894
- const sshMatch = githubUrl.match(sshRegex);
1895
- if (sshMatch) {
1896
- const { owner, repoName, packagePath } = sshMatch.groups!;
1897
- return { owner, repoName, packagePath };
1898
- }
1899
-
1900
- return null;
1901
- }
1902
-
1903
- async downloadGitHubDirectory(githubUrl: string, absoluteDirPath: string) {
2304
+ absoluteDirPath: string,
2305
+ progressContext?: { environmentName?: string; packageNames?: string[] },
2306
+ ) {
1904
2307
  assertSafeEnvironmentPath(absoluteDirPath);
1905
2308
  // First we'll clone the repo without the additional path
1906
2309
  // E.g. we're removing `/tree/main/imdb` from https://github.com/credibledata/malloy-samples/tree/main/imdb
1907
- const githubInfo = this.parseGitHubUrl(githubUrl);
2310
+ const githubInfo = parseGitHubUrl(githubUrl);
1908
2311
  if (!githubInfo) {
1909
2312
  throw new Error(`Invalid GitHub URL: ${githubUrl}`);
1910
2313
  }
@@ -1923,17 +2326,30 @@ export class EnvironmentStore {
1923
2326
  });
1924
2327
  await fs.promises.mkdir(absoluteDirPath, { recursive: true });
1925
2328
  const repoUrl = `https://github.com/${owner}/${repoName}`;
2329
+ const reporter = new CloneProgressReporter(
2330
+ cloneProgressLabel(`${owner}/${repoName}`, progressContext),
2331
+ );
1926
2332
 
1927
2333
  // We'll clone the repo into absoluteDirPath
1928
2334
  await new Promise<void>((resolve, reject) => {
1929
- simpleGit().clone(repoUrl, absoluteDirPath, {}, (err) => {
2335
+ simpleGit({
2336
+ progress: (event) => reporter.onProgress(event),
2337
+ }).clone(repoUrl, absoluteDirPath, GIT_CLONE_OPTIONS, (err) => {
2338
+ reporter.done();
1930
2339
  if (err) {
2340
+ err.message = stripGitProgressNoise(err.message);
2341
+ if (err.stack) {
2342
+ // V8 captures the message into the stack at construction,
2343
+ // so the spew survives there unless stripped too.
2344
+ err.stack = stripGitProgressNoise(err.stack);
2345
+ }
1931
2346
  const errorData = this.extractErrorDataFromError(err);
1932
2347
  logger.error(
1933
2348
  `Failed to clone GitHub repository "${repoUrl}"`,
1934
2349
  errorData,
1935
2350
  );
1936
2351
  reject(err);
2352
+ return;
1937
2353
  }
1938
2354
  resolve();
1939
2355
  });