@superblocksteam/sdk 2.0.115-next.0 → 2.0.115-next.2

package/src/cli-replacement/dev.mts

@@ -58,6 +58,13 @@ import {
   ensureRemoteHasDefaultBranch,
   getGitErrorFields,
 } from "./git-repo-setup.mjs";
+import {
+  didPackageJsonSnapshotChange,
+  packageJsonSnapshot,
+  packageJsonSnapshotDiagnostic,
+  type PackageJsonSnapshot,
+  restoreManagedPackageDependencies,
+} from "./package-json-snapshot.mjs";
 import { getCurrentCliVersion } from "./version-detection.js";
 
 const exec = promisify(child_process.exec);
@@ -296,6 +303,15 @@ export async function dev(options: {
 
   /** Pre-existing HTTP server from warm standby mode (avoids port gap on transition). */
   existingServer?: HttpServer;
+
+  /** Force package installation when the caller has detected dependency drift. */
+  forcePackageInstall?: boolean;
+
+  /** Package snapshot before S3 workspace restore; used to refresh forced install after DBFS sync. */
+  packageJsonSnapshotBeforeRestore?: PackageJsonSnapshot | null;
+
+  /** Restore managed warm-template package pins after DBFS download in S3 warm restore flow. */
+  normalizeManagedPackageDependencies?: boolean;
 }) {
   const {
     cwd,
@@ -566,6 +582,9 @@ export async function dev(options: {
             }
 
             let hasPackageChanged = false;
+            let packageJsonRequiresInstall = false;
+            const hasPackageJsonSnapshotBeforeRestore =
+              options.packageJsonSnapshotBeforeRestore !== undefined;
 
             const packageJsonBefore = await readPkgJson(cwd);
 
@@ -576,6 +595,17 @@ export async function dev(options: {
                 );
 
                 await syncService!.downloadDirectory();
+                if (
+                  options.normalizeManagedPackageDependencies &&
+                  (await restoreManagedPackageDependencies(
+                    cwd,
+                    packageJsonBefore,
+                  ))
+                ) {
+                  logger.info(
+                    "Restored managed package dependencies to the warm template versions after DBFS download",
+                  );
+                }
                 span.end();
               });
             }
@@ -603,6 +633,100 @@ export async function dev(options: {
                     applicationId: applicationConfig.id,
                     workDir: cwd,
                   });
+
+                  // In CSB mode, schedule a background retry loop. The app's
+                  // git config may not be persisted yet at claim time (e.g. git
+                  // was connected moments before the CSB was assigned). Without
+                  // this, the CSB runs its entire lifetime without git — meaning
+                  // changes are only in DBFS and are lost if the CSB recycles.
+                  if (lockType === LockType.CSB) {
+                    const GIT_RETRY_DELAY_MS = 30_000;
+                    const GIT_RETRY_MAX_ATTEMPTS = 10;
+                    const scheduleRetry = (attempt: number): void => {
+                      if (attempt > GIT_RETRY_MAX_ATTEMPTS) {
+                        logger.info(
+                          "[git] background bootstrap retry exhausted, giving up",
+                          {
+                            gitCategory: "setup",
+                            gitOperation: "background-retry",
+                            gitOutcome: "exhausted",
+                            gitAttempt: attempt - 1,
+                            applicationId: applicationConfig.id,
+                          },
+                        );
+                        return;
+                      }
+                      const timer = setTimeout(() => {
+                        void (async () => {
+                          if (gitService) return;
+                          try {
+                            const svc = await bootstrapGitService({
+                              sdk,
+                              applicationId: applicationConfig.id,
+                              cwd,
+                              logger,
+                              userName: gitUserName,
+                              userEmail: gitUserEmail,
+                              superblocksBaseUrl:
+                                tokenConfig.superblocksBaseUrl,
+                            });
+                            if (!svc) {
+                              scheduleRetry(attempt + 1);
+                              return;
+                            }
+
+                            gitService = svc;
+
+                            try {
+                              await fetchAndEnsureLiveBranch(
+                                svc,
+                                "Git background retry fetch failed",
+                              );
+                            } catch {
+                              // non-fatal
+                            }
+
+                            activeDbfsBranchName =
+                              await ensureRuntimeDbfsBranchConsistency({
+                                sdk,
+                                applicationConfig,
+                                logger,
+                                lockService,
+                                syncService,
+                                currentBranchName: activeDbfsBranchName,
+                              });
+
+                            logger.info(
+                              "[git] background bootstrap retry succeeded",
+                              {
+                                gitCategory: "setup",
+                                gitOperation: "background-retry",
+                                gitOutcome: "success",
+                                gitAttempt: attempt,
+                                applicationId: applicationConfig.id,
+                              },
+                            );
+                          } catch (err) {
+                            logger.warn(
+                              "[git] background bootstrap retry failed",
+                              {
+                                gitCategory: "setup",
+                                gitOperation: "background-retry",
+                                gitOutcome: "failed",
+                                gitAttempt: attempt,
+                                applicationId: applicationConfig.id,
+                                ...getGitErrorFields(err),
+                              },
+                            );
+                            scheduleRetry(attempt + 1);
+                          }
+                        })();
+                      }, GIT_RETRY_DELAY_MS);
+                      timer.unref();
+                    };
+                    scheduleRetry(1);
+                  }
+
                   gitSpan.end();
                   return;
                 }
@@ -720,16 +844,73 @@ export async function dev(options: {
 
             const packageJsonAfter = await readPkgJson(cwd);
 
+            let packageJsonSnapshotBefore: PackageJsonSnapshot | null = null;
+            let packageJsonSnapshotAfter: PackageJsonSnapshot | null = null;
+            let packageJsonInstallBaselineSnapshot: PackageJsonSnapshot | null =
+              null;
+
             if (packageJsonBefore && packageJsonAfter) {
               hasPackageChanged =
                 JSON.stringify(packageJsonBefore, null, 2) !==
                 JSON.stringify(packageJsonAfter, null, 2);
             } else if (packageJsonAfter) {
               hasPackageChanged = true;
+            }
+
+            if (packageJsonBefore) {
+              packageJsonSnapshotBefore =
+                packageJsonSnapshot(packageJsonBefore);
+            }
+            if (packageJsonAfter) {
+              packageJsonSnapshotAfter = packageJsonSnapshot(packageJsonAfter);
+            }
+
+            packageJsonInstallBaselineSnapshot =
+              hasPackageJsonSnapshotBeforeRestore
+                ? (options.packageJsonSnapshotBeforeRestore ?? null)
+                : packageJsonSnapshotBefore;
+            if (packageJsonAfter) {
+              packageJsonRequiresInstall = didPackageJsonSnapshotChange(
+                packageJsonInstallBaselineSnapshot,
+                packageJsonSnapshotAfter,
+              );
+            }
+            if (!packageJsonBefore && packageJsonRequiresInstall) {
               logger.info("package.json was created, installing packages…");
             }
+            const forcePackageInstallRequested = !!options.forcePackageInstall;
+            let forcePackageInstall = forcePackageInstallRequested;
+            if (
+              forcePackageInstallRequested &&
+              hasPackageJsonSnapshotBeforeRestore
+            ) {
+              forcePackageInstall = packageJsonRequiresInstall;
+            }
+
+            logger.info("Package install decision", {
+              packageJsonBeforePresent: !!packageJsonBefore,
+              packageJsonAfterPresent: !!packageJsonAfter,
+              hasPackageChanged,
+              packageJsonRequiresInstall,
+              forcePackageInstall,
+              forcePackageInstallRequested,
+              upgradePromiseCount: upgradePromises.length,
+              packageJsonSnapshotBefore: packageJsonSnapshotDiagnostic(
+                packageJsonSnapshotBefore,
+              ),
+              packageJsonInstallBaselineSnapshot: packageJsonSnapshotDiagnostic(
+                packageJsonInstallBaselineSnapshot,
+              ),
+              packageJsonSnapshotAfter: packageJsonSnapshotDiagnostic(
+                packageJsonSnapshotAfter,
+              ),
+            });
 
-            if (hasPackageChanged || upgradePromises.length > 0) {
+            if (
+              packageJsonRequiresInstall ||
+              upgradePromises.length > 0 ||
+              forcePackageInstall
+            ) {
               logger.info("Installing packages…");
               await tracer.startActiveSpan("installPackages", async (span) => {
                 try {
@@ -748,7 +929,9 @@ export async function dev(options: {
               );
             }
 
-            if (hasPackageChanged || uploadFirst) {
+            const shouldUploadPackageState =
+              hasPackageChanged || forcePackageInstall;
+            if (shouldUploadPackageState || uploadFirst) {
               logger.info(
                 `Uploading local files to branch '${activeDbfsBranchName}' on server before starting`,
               );
@@ -1230,10 +1413,23 @@ async function ensureLiveBranchCheckedOutAfterFetch(
   // If remote live branch exists, initialize/switch local live branch from it.
   // This handles fresh repos where HEAD is unborn before first fetch.
   if (await canResolveRef(git, `origin/${SUPERBLOCKS_LIVE_GIT_BRANCH}`)) {
-    await git.checkoutOrCreate(
-      SUPERBLOCKS_LIVE_GIT_BRANCH,
-      `origin/${SUPERBLOCKS_LIVE_GIT_BRANCH}`,
-    );
+    try {
+      await git.checkoutOrCreate(
+        SUPERBLOCKS_LIVE_GIT_BRANCH,
+        `origin/${SUPERBLOCKS_LIVE_GIT_BRANCH}`,
+      );
+    } catch {
+      // Pre-warmed CSBs download DBFS files before git is set up, leaving
+      // untracked working tree files that collide with the branch contents.
+      // Force-checkout is safe here — the files are identical.
+      await git.raw([
+        "checkout",
+        "-f",
+        "-B",
+        SUPERBLOCKS_LIVE_GIT_BRANCH,
+        `origin/${SUPERBLOCKS_LIVE_GIT_BRANCH}`,
+      ]);
+    }
     return;
   }
 

package/src/cli-replacement/package-json-snapshot.mts

@@ -0,0 +1,328 @@
+import { createHash } from "node:crypto";
+import * as fs from "node:fs/promises";
+import { join } from "node:path";
+
+export const SUPERBLOCKS_LIBRARY_PACKAGE = "@superblocksteam/library";
+export const SUPERBLOCKS_SDK_API_PACKAGE = "@superblocksteam/sdk-api";
+
+export const MANAGED_PACKAGE_DEPENDENCIES = [
+  SUPERBLOCKS_LIBRARY_PACKAGE,
+  SUPERBLOCKS_SDK_API_PACKAGE,
+] as const;
+
+export const PACKAGE_DEPENDENCY_FIELDS = [
+  "dependencies",
+  "devDependencies",
+  "peerDependencies",
+  "optionalDependencies",
+] as const;
+
+const PACKAGE_INSTALL_METADATA_FIELDS = [
+  "dependenciesMeta",
+  "devEngines",
+  "engines",
+  "overrides",
+  "packageManager",
+  "peerDependenciesMeta",
+  "pnpm",
+  "resolutions",
+  "workspaces",
+] as const;
+
+const PACKAGE_BUNDLE_DEPENDENCIES_FIELDS = [
+  "bundleDependencies",
+  "bundledDependencies",
+] as const;
+
+export type PackageJsonSnapshot = {
+  value: string;
+  diagnostic: {
+    sha256: string;
+    bytes: number;
+  };
+};
+
+export type PackageJsonSnapshotReadResult = {
+  packageJson: unknown | null;
+  snapshot: PackageJsonSnapshot | null;
+};
+
+export function packageJsonSnapshot(packageJson: unknown): PackageJsonSnapshot {
+  const packageJsonObject = packageJsonRecord(packageJson);
+  const normalized = PACKAGE_DEPENDENCY_FIELDS.reduce<Record<string, unknown>>(
+    (acc, field) => {
+      const dependencies = packageJsonObject
+        ? dependencyMap(packageJsonObject, field)
+        : undefined;
+      if (dependencies) {
+        acc[field] = dependencies;
+      }
+      return acc;
+    },
+    {},
+  );
+  if (packageJsonObject) {
+    const bundleDependenciesField = PACKAGE_BUNDLE_DEPENDENCIES_FIELDS.find(
+      (field) => Object.prototype.hasOwnProperty.call(packageJsonObject, field),
+    );
+    if (bundleDependenciesField) {
+      normalized.bundleDependencies =
+        packageJsonObject[bundleDependenciesField];
+    }
+
+    for (const field of PACKAGE_INSTALL_METADATA_FIELDS) {
+      if (Object.prototype.hasOwnProperty.call(packageJsonObject, field)) {
+        normalized[field] = packageJsonObject[field];
+      }
+    }
+  }
+  const value = stableStringify(normalized);
+
+  return {
+    value,
+    diagnostic: {
+      sha256: createHash("sha256").update(value).digest("hex"),
+      bytes: Buffer.byteLength(value),
+    },
+  };
+}
+
+export function didPackageJsonSnapshotChange(
+  before: PackageJsonSnapshot | null,
+  after: PackageJsonSnapshot | null,
+): boolean {
+  if (before?.value === after?.value) {
+    return false;
+  }
+
+  // If restore removes package.json, there is no manifest to install from.
+  // Treat that as no install work instead of forcing an install that cannot
+  // converge until DBFS or another source recreates the manifest.
+  return after !== null;
+}
+
+export function packageJsonSnapshotDiagnostic(
+  snapshot: PackageJsonSnapshot | null,
+): { present: boolean; sha256?: string; bytes?: number } {
+  if (snapshot === null) {
+    return { present: false };
+  }
+
+  return {
+    present: true,
+    ...snapshot.diagnostic,
+  };
+}
+
+type PackageJsonObject = Record<string, unknown>;
+type PackageDependencySpec = { field: string; value: unknown };
+
+function stableStringify(value: unknown): string {
+  return stableStringifyValue(value) ?? "null";
+}
+
+function stableStringifyValue(value: unknown): string | undefined {
+  if (value === undefined) {
+    return undefined;
+  }
+
+  if (Array.isArray(value)) {
+    const items = value.map((item) => stableStringifyValue(item) ?? "null");
+    return `[${items.join(",")}]`;
+  }
+
+  if (value && typeof value === "object") {
+    const object = value as Record<string, unknown>;
+    const entries = Object.keys(object)
+      .sort()
+      .flatMap((key) => {
+        const serializedValue = stableStringifyValue(object[key]);
+        return serializedValue === undefined
+          ? []
+          : [`${JSON.stringify(key)}:${serializedValue}`];
+      });
+    return `{${entries.join(",")}}`;
+  }
+
+  const serialized = JSON.stringify(value);
+  return serialized === undefined ? undefined : serialized;
+}
+
+function packageJsonRecord(packageJson: unknown): PackageJsonObject | null {
+  if (
+    !packageJson ||
+    typeof packageJson !== "object" ||
+    Array.isArray(packageJson)
+  ) {
+    return null;
+  }
+
+  return packageJson as PackageJsonObject;
+}
+
+function dependencyMap(
+  packageJson: PackageJsonObject,
+  field: string,
+): Record<string, unknown> | undefined {
+  const dependencies = packageJson[field];
+  if (
+    dependencies &&
+    typeof dependencies === "object" &&
+    !Array.isArray(dependencies)
+  ) {
+    return dependencies as Record<string, unknown>;
+  }
+
+  return undefined;
+}
+
+function packageDependencySpecs(
+  packageJson: PackageJsonObject,
+  packageName: string,
+): PackageDependencySpec[] {
+  return PACKAGE_DEPENDENCY_FIELDS.flatMap((field) => {
+    const dependencies = dependencyMap(packageJson, field);
+    if (
+      dependencies &&
+      Object.prototype.hasOwnProperty.call(dependencies, packageName)
+    ) {
+      return [{ field, value: dependencies[packageName] }];
+    }
+
+    return [];
+  });
+}
+
+function dependencySpecEquals(a: unknown, b: unknown): boolean {
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+
+async function readPackageJson(cwd: string): Promise<unknown | null> {
+  return (await readPackageJsonFile(cwd))?.packageJson ?? null;
+}
+
+async function readPackageJsonFile(
+  cwd: string,
+): Promise<{ packageJson: unknown; source: string } | null> {
+  try {
+    const source = await fs.readFile(join(cwd, "package.json"), "utf-8");
+    return {
+      packageJson: JSON.parse(source),
+      source,
+    };
+  } catch {
+    return null;
+  }
+}
+
+function detectJsonIndent(source: string): number | string {
+  return source.match(/\n([ \t]+)"/)?.[1] ?? 2;
+}
+
+function hasFinalNewline(source: string): boolean {
+  return source.endsWith("\n");
+}
+
+export async function readPackageJsonSnapshot(
+  cwd: string,
+): Promise<PackageJsonSnapshot | null> {
+  const packageJson = await readPackageJson(cwd);
+  if (!packageJson) {
+    return null;
+  }
+  return packageJsonSnapshot(packageJson);
+}
+
+export async function readPackageJsonSnapshotWithSource(
+  cwd: string,
+): Promise<PackageJsonSnapshotReadResult> {
+  const packageJson = await readPackageJson(cwd);
+  return {
+    packageJson,
+    snapshot: packageJson ? packageJsonSnapshot(packageJson) : null,
+  };
+}
+
+export async function restoreManagedPackageDependencies(
+  cwd: string,
+  warmPackageJson: unknown | null,
+): Promise<boolean> {
+  const warmPackageJsonObject = packageJsonRecord(warmPackageJson);
+  if (!warmPackageJsonObject) {
+    return false;
+  }
+
+  const restoredPackageJsonFile = await readPackageJsonFile(cwd);
+  if (!restoredPackageJsonFile) {
+    return false;
+  }
+
+  const restoredPackageJson = packageJsonRecord(
+    restoredPackageJsonFile.packageJson,
+  );
+  if (!restoredPackageJson) {
+    return false;
+  }
+  const restoredPackageJsonSource = restoredPackageJsonFile.source;
+
+  let changed = false;
+  for (const packageName of MANAGED_PACKAGE_DEPENDENCIES) {
+    const [warmPackageSpec] = packageDependencySpecs(
+      warmPackageJsonObject,
+      packageName,
+    );
+    if (!warmPackageSpec) {
+      continue;
+    }
+
+    const restoredPackageSpecs = packageDependencySpecs(
+      restoredPackageJson,
+      packageName,
+    );
+    if (
+      restoredPackageSpecs.length === 1 &&
+      restoredPackageSpecs[0].field === warmPackageSpec.field &&
+      dependencySpecEquals(restoredPackageSpecs[0].value, warmPackageSpec.value)
+    ) {
+      continue;
+    }
+
+    if (
+      restoredPackageSpecs.length === 1 &&
+      restoredPackageSpecs[0].field === warmPackageSpec.field
+    ) {
+      dependencyMap(restoredPackageJson, warmPackageSpec.field)![packageName] =
+        warmPackageSpec.value;
+    } else {
+      for (const field of PACKAGE_DEPENDENCY_FIELDS) {
+        delete dependencyMap(restoredPackageJson, field)?.[packageName];
+      }
+
+      let dependencies = dependencyMap(
+        restoredPackageJson,
+        warmPackageSpec.field,
+      );
+      if (!dependencies) {
+        dependencies = {};
+        restoredPackageJson[warmPackageSpec.field] = dependencies;
+      }
+      dependencies[packageName] = warmPackageSpec.value;
+    }
+    changed = true;
+  }
+
+  if (!changed) {
+    return false;
+  }
+
+  await fs.writeFile(
+    join(cwd, "package.json"),
+    JSON.stringify(
+      restoredPackageJson,
+      null,
+      detectJsonIndent(restoredPackageJsonSource),
+    ) + (hasFinalNewline(restoredPackageJsonSource) ? "\n" : ""),
+  );
+
+  return true;
+}