@superblocksteam/sdk 1.14.2 → 2.0.3-next.46

package/src/client.ts

@@ -1,38 +1,42 @@
 import * as fs from "fs";
+import { Bucketeer, FileDescriptor } from "@superblocksteam/bucketeer-sdk";
+import { ExportViewMode } from "@superblocksteam/shared";
 import {
   COMPONENT_EVENT_HEADER,
   ComponentEvent,
   ForbiddenError,
   getBucketeerUrlFromSuperblocksUrl,
   getContentType,
-  LocalGitRepoState,
   NotFoundError,
-  SuperblocksResourceType,
   BadRequestError,
   unreachable,
-  ValidateGitSetupRequestBody,
 } from "@superblocksteam/util";
-import axios, { AxiosError, AxiosRequestConfig } from "axios";
+import axios, { AxiosError } from "axios";
 import FormData from "form-data";
-import { isEqual, isEmpty } from "lodash";
+import { isEqual, isEmpty } from "lodash-es";
 import {
   BranchNotCheckedOutError,
   CommitAlreadyExistsError,
   ValidateGitSetupError,
-} from "./errors";
-import { signingEnabled } from "./flag";
-import { connectToISocketRPCServer, StdISocketRPCClient } from "./socket";
-import {
-  AgentType,
-  Api,
+} from "./errors.js";
+import { signingEnabled } from "./flag.js";
+import { connectToISocketRPCServer } from "./socket/index.js";
+import { AgentType } from "./types/index.js";
+import { getAgentUrl } from "./utils.js";
+import type { StdISocketRPCClient } from "./socket/index.js";
+import type {
   ApiWithPb,
   Page,
   RemoteCommitDto,
-  UserMeDto,
-  ViewMode,
   DeploymentDto,
-} from "./types";
-import { getAgentUrl } from "./utils";
+  UserMeDto,
+} from "./types/index.js";
+import type {
+  LocalGitRepoState,
+  SuperblocksResourceType,
+  ValidateGitSetupRequestBody,
+} from "@superblocksteam/util";
+import type { AxiosRequestConfig } from "axios";
 
 const BASE_BUCKETEER_URL = "api";
 const BASE_SERVER_PUBLIC_API_URL_V1 = "api/v1/public";
@@ -63,23 +67,31 @@ export interface MultiPageApplicationWrapper {
   apis: Record<string, any>[];
 }
 
+export interface MultiPageApplicationWrapperWithComponents {
+  type: "multi-page";
+  application: Record<string, any>;
+  pages: Page[];
+  apis: Record<string, any>[];
+  componentFiles: any;
+}
+
+export interface CodeModeApplicationWrapper {
+  type: "code-mode";
+  application: Record<string, any>;
+}
+
 export type PushMultiPageApplicationWithCommitConfig =
   MultiPageApplicationWrapper & {
-    commitId?: string;
-    commitMessage?: string;
+    commitId: string;
+    commitMessage: string;
     gitState: LocalGitRepoState;
     skipCommit: boolean;
   };
 
-export interface ApiWrapper {
-  apiPb: Api;
-  name?: string; //@deprecated this attribute is used for getting a name for backends(jobs and workflows) that were never migrated to the new API version.
-}
-
 export type PushApiWithCommitConfig = {
   apiPb: Record<string, any>;
-  commitId?: string;
-  commitMessage?: string;
+  commitId: string;
+  commitMessage: string;
   gitState: LocalGitRepoState;
   skipCommit: boolean;
 };
@@ -136,12 +148,12 @@ export async function fetchApplication({
   branch?: string;
   token: string;
   superblocksBaseUrl: string;
-  viewMode: ViewMode;
+  viewMode: ExportViewMode;
   commitId?: string;
   skipSigningVerification?: boolean;
   injectedHeaders: Record<string, string>;
 }): Promise<MultiPageApplicationWrapper | undefined> {
-  if (commitId && viewMode !== "export-commit") {
+  if (commitId && viewMode !== ExportViewMode.EXPORT_COMMIT) {
     throw new Error(
       `If commitId ${commitId} is provided, viewMode cannot be ${viewMode}`,
     );
@@ -177,7 +189,7 @@ export async function fetchApplication({
           branchName: branch,
           commitId,
         });
-        return resp.data;
+        return resp.data as unknown as MultiPageApplicationWrapper;
       } finally {
         socket.close();
       }
@@ -263,14 +275,13 @@ export async function fetchApplicationWithComponents({
   branch: string;
   token: string;
   superblocksBaseUrl: string;
-  viewMode: ViewMode;
+  viewMode: ExportViewMode;
   commitId?: string;
   skipSigningVerification?: boolean;
   injectedHeaders: Record<string, string>;
 }): Promise<
-  | (MultiPageApplicationWrapper & {
-      componentFiles: any;
-    })
+  | MultiPageApplicationWrapperWithComponents
+  | CodeModeApplicationWrapper
   | undefined
 > {
   const applicationWrapper = await fetchApplication({
@@ -289,9 +300,48 @@ export async function fetchApplicationWithComponents({
     return;
   }
 
+  if (applicationWrapper.application.devEnvEnabled) {
+    return {
+      type: "code-mode",
+      ...applicationWrapper,
+    } satisfies CodeModeApplicationWrapper;
+  }
+
+  return await fetchApplicationWithComponentsFromBucketeer({
+    applicationWrapper,
+    superblocksBaseUrl,
+    applicationId,
+    branch,
+    commitId,
+    viewMode,
+    token,
+    injectedHeaders,
+  });
+}
+
+async function fetchApplicationWithComponentsFromBucketeer({
+  applicationWrapper,
+  superblocksBaseUrl,
+  applicationId,
+  branch,
+  commitId,
+  viewMode,
+  token,
+  injectedHeaders,
+}: {
+  applicationWrapper: MultiPageApplicationWrapper;
+  superblocksBaseUrl: string;
+  applicationId: string;
+  branch: string;
+  commitId?: string;
+  viewMode: ExportViewMode;
+  token: string;
+  injectedHeaders: Record<string, string>;
+}): Promise<MultiPageApplicationWrapperWithComponents> {
   // if there are no custom components, just return here without trying to initialize them using bucketeer
   if (isEmpty(applicationWrapper.application?.settings?.registeredComponents)) {
     return {
+      type: "multi-page",
       ...applicationWrapper,
       componentFiles: null,
     };
@@ -318,9 +368,7 @@ export async function fetchApplicationWithComponents({
       },
     };
     const bucketeerApp = (await axios(config))
-      .data as MultiPageApplicationWrapper & {
-      componentFiles: any;
-    };
+      .data as MultiPageApplicationWrapperWithComponents;
 
     if (
       !isEqual(
@@ -383,7 +431,7 @@ export async function fetchApi(
   apiId: string,
   token: string,
   superblocksBaseUrl: string,
-  viewMode: ViewMode,
+  viewMode: ExportViewMode,
   branch?: string,
   commitId?: string,
   skipSigningVerification = false,
@@ -716,7 +764,7 @@ You can reduce your component bundle size by uploading static assets to a separa
         branchName: branch ?? undefined,
         srcFiles: srcFiles.map((file) => file.filename),
         buildFiles: buildFiles.map((file) => file.filename),
-        registeredComponents: componentConfigs,
+        registeredComponents: componentConfigs as Record<string, never>,
         cliVersion,
         componentBaseUrl: uploadResponse.data.componentBaseUrl,
         signingRequired: !isEmpty(initialSocket),
@@ -858,7 +906,7 @@ export async function pushApplication({
       if (resp.responseMeta.status !== 200) {
         // Get the raw error message from the server and throw it. The outer try-catch block will wrap it in a nicer error message
         const message: string =
-          (resp?.responseMeta?.message as string) ?? JSON.stringify(resp?.data);
+          resp?.responseMeta?.message ?? JSON.stringify(resp?.data);
         throw new Error(message);
       }
       return resp.data;
@@ -968,7 +1016,7 @@ export async function pushApi({
       if (resp.responseMeta.status !== 200) {
         // Get the raw error message from the server and throw it. The outer try-catch block will wrap it in a nicer error message
         const message: string =
-          (resp?.responseMeta?.message as string) ?? JSON.stringify(resp?.data);
+          resp?.responseMeta?.message ?? JSON.stringify(resp?.data);
         throw new Error(message);
       }
       return resp.data;
@@ -1215,3 +1263,31 @@ async function deployResource(
     throw new Error(`${e.message}`);
   }
 }
+
+export async function uploadApplication({
+  files,
+  scopedJwt,
+  url,
+  cliVersion,
+}: {
+  files: string[];
+  scopedJwt: string;
+  url: string;
+  cliVersion: string;
+}) {
+  const fds = filesToFileDescriptors(files);
+  const bucketeer = new Bucketeer({
+    token: scopedJwt,
+    baseUrl: url,
+    cliVersion,
+    maxTotalFileSizeMB: SUPERBLOCKS_MAX_FILE_SIZE_MB,
+  });
+  await bucketeer.uploadApplication(fds);
+}
+
+function filesToFileDescriptors(files: string[]) {
+  const fds = files.map((file) => {
+    return new FileDescriptor(file, fs.createReadStream(file));
+  });
+  return fds;
+}

package/src/dbfs/client.ts

@@ -0,0 +1,162 @@
+import { unreachable } from "@superblocksteam/util";
+import { connectToISocketRPCServer } from "../socket/index.js";
+import { doDownloadDirectoryToLocal, doUploadLocalDirectory } from "./local.js";
+
+// TODO(code-mode): this is re-implemented in the vite-plugin-file-sync package
+// because SDK depends on vite-plugin-file-sync, but can't be circular
+export async function getApplicationDirectoryHash(
+  token: string,
+  superblocksBaseUrl: string,
+  applicationId: string,
+  branch: string | undefined,
+) {
+  const rpcClient = await connectToISocketRPCServer({
+    token,
+    superblocksBaseUrl,
+  });
+  try {
+    const response =
+      await rpcClient.call.v3.application.liveEditDirectoryContents.get({
+        applicationId,
+        branchName: branch,
+      });
+    const liveEditHash = response.data.hash;
+    if (!liveEditHash) {
+      throw new Error("No live edit hash found");
+    }
+    return liveEditHash;
+  } finally {
+    rpcClient.close();
+  }
+}
+
+export async function downloadApplicationDirectory(
+  token: string,
+  superblocksBaseUrl: string,
+  applicationId: string,
+  branch: string | undefined,
+  localDirectoryPath: string,
+): Promise<void> {
+  const rpcClient = await connectToISocketRPCServer({
+    token,
+    superblocksBaseUrl,
+  });
+  try {
+    const liveEditHash = await getApplicationDirectoryHash(
+      token,
+      superblocksBaseUrl,
+      applicationId,
+      branch,
+    );
+    await doDownloadDirectoryToLocal(
+      rpcClient,
+      liveEditHash,
+      localDirectoryPath,
+    );
+  } finally {
+    rpcClient.close();
+  }
+}
+
+export async function uploadLocalApplication(
+  token: string,
+  superblocksBaseUrl: string,
+  applicationId: string,
+  branch: string | undefined,
+  localDirectoryPath: string,
+): Promise<void> {
+  const rpcClient = await connectToISocketRPCServer({
+    token,
+    superblocksBaseUrl,
+  });
+  try {
+    const directoryHash = await doUploadLocalDirectory(
+      rpcClient,
+      localDirectoryPath,
+    );
+    console.log(`New application directory hash: ${directoryHash}`);
+    await rpcClient.call.v3.application.liveEditDirectoryContents.set({
+      applicationId,
+      branchName: branch,
+      hash: directoryHash,
+    });
+  } finally {
+    rpcClient.close();
+  }
+}
+
+export async function printDirectoryEntries(
+  token: string,
+  superblocksBaseUrl: string,
+  directoryHash: string,
+): Promise<void> {
+  const rpcClient = await connectToISocketRPCServer({
+    token,
+    superblocksBaseUrl,
+  });
+  try {
+    const directoryContentsResponse =
+      await rpcClient.call.v1.dbfs.directoryContents.get({
+        hash: directoryHash,
+      });
+    for (const entry of directoryContentsResponse.data.contents) {
+      let executable: boolean;
+      let hash: string;
+      let target: string | undefined;
+      switch (entry.type) {
+        case "-": {
+          executable = entry.executable;
+          hash = entry.hash;
+          break;
+        }
+        case "l": {
+          executable = false;
+          target = entry.target;
+          hash = " ".repeat(Math.ceil(256 / 24) * 4);
+          break;
+        }
+        case "d": {
+          executable = false;
+          hash = entry.hash;
+          break;
+        }
+        default: {
+          unreachable(entry);
+        }
+      }
+      console.log(
+        `${entry.type}${executable ? "x" : " "} ${hash} ${entry.name}${
+          target ? ` -> ${target}` : ""
+        }`,
+      );
+    }
+  } finally {
+    rpcClient.close();
+  }
+}
+
+export async function printFileContents(
+  token: string,
+  superblocksBaseUrl: string,
+  fileHash: string,
+): Promise<void> {
+  const rpcClient = await connectToISocketRPCServer({
+    token,
+    superblocksBaseUrl,
+  });
+  try {
+    const fileContentsResponse = await rpcClient.call.v1.dbfs.fileContents.get({
+      hash: fileHash,
+    });
+    // The file can contain binary data, do not interpret it as text
+    // Note: `Buffer.from` is a Node.js specific API
+    const fileContents = Buffer.from(
+      fileContentsResponse.data.contents,
+      "base64",
+    );
+    // Note: `process.stdout.write` is a Node.js specific API
+    process.stdout.write(fileContents);
+  } finally {
+    rpcClient.close();
+  }
+}

package/src/dbfs/local.ts

@@ -0,0 +1,163 @@
+import * as fsp from "node:fs/promises";
+import {
+  hashDirectoryContents,
+  hashFileContents,
+  type DirectoryEntry,
+} from "@superblocksteam/shared";
+import { unreachable } from "@superblocksteam/util";
+import { listLocalDirectory } from "@superblocksteam/vite-plugin-file-sync/list-dir";
+import type { StdISocketRPCClient } from "../socket/index.js";
+
+export async function doDownloadDirectoryToLocal(
+  rpcClient: StdISocketRPCClient,
+  directoryHash: string,
+  localDirectoryPath: string,
+): Promise<void> {
+  // ensure that the top-level local directory exists
+  await fsp.mkdir(localDirectoryPath, { recursive: true });
+  await doDownloadDirectoryToLocalRec(
+    rpcClient,
+    directoryHash,
+    localDirectoryPath,
+  );
+}
+
+async function doDownloadDirectoryToLocalRec(
+  rpcClient: StdISocketRPCClient,
+  directoryHash: string,
+  localDirectoryPath: string,
+): Promise<void> {
+  const directoryContentsResponse =
+    await rpcClient.call.v1.dbfs.directoryContents.get({ hash: directoryHash });
+  if (!directoryContentsResponse.data) {
+    throw new Error("Failed to get directory contents");
+  }
+
+  for (const entry of directoryContentsResponse.data.contents) {
+    const localFilePath = `${localDirectoryPath}/${entry.name}`;
+    switch (entry.type) {
+      case "-": {
+        const fileContentsResponse =
+          await rpcClient.call.v1.dbfs.fileContents.get({ hash: entry.hash });
+        const fileContents = Buffer.from(
+          fileContentsResponse.data.contents,
+          "base64",
+        );
+        await fsp.writeFile(localFilePath, fileContents);
+        break;
+      }
+      case "l": {
+        await fsp.symlink(entry.target, localFilePath);
+        break;
+      }
+      case "d": {
+        await fsp.mkdir(localFilePath, {
+          recursive: true /* do not reject if exists */,
+        });
+        await doDownloadDirectoryToLocalRec(
+          rpcClient,
+          entry.hash,
+          localFilePath,
+        );
+        break;
+      }
+      default: {
+        unreachable(entry);
+      }
+    }
+  }
+}
+
+/**
+ * Computes the hash of a local directory, including the hashes of all its contents recursively.
+ *
+ * @param localDirectoryPath - The path to the local directory to hash.
+ * @returns A promise that resolves to the hash of the directory and its contents.
+ */
+export async function hashLocalDirectory(localDirectoryPath: string): Promise<{
+  hash: string;
+  contents: DirectoryEntry[];
+}> {
+  const directoryContents: any[] = [];
+  const localDirListing = listLocalDirectory(localDirectoryPath);
+  for await (const localDirEntry of localDirListing) {
+    let entry: any;
+    switch (localDirEntry.type) {
+      case "-": {
+        const hash = await hashFileContents(localDirEntry.contents);
+        entry = {
+          type: "-",
+          executable: localDirEntry.executable,
+          name: localDirEntry.name,
+          hash,
+        };
+        break;
+      }
+      case "d": {
+        const { hash, contents: subdirectoryContents } =
+          await hashLocalDirectory(localDirEntry.localPath);
+        entry = {
+          type: "d",
+          name: localDirEntry.name,
+          hash,
+          contents: subdirectoryContents,
+        };
+        break;
+      }
+      case "l": {
+        entry = {
+          type: "l",
+          target: localDirEntry.target,
+          name: localDirEntry.name,
+        };
+        break;
+      }
+      default:
+        unreachable(localDirEntry);
+    }
+    directoryContents.push(entry);
+  }
+  const hash = await hashDirectoryContents(directoryContents);
+  return { contents: directoryContents, hash };
+}
+
+export async function doUploadLocalDirectory(
+  rpcClient: StdISocketRPCClient,
+  localDirectoryPath: string,
+): Promise<string> {
+  const directoryContents: DirectoryEntry[] = [];
+  const localDirListing = listLocalDirectory(localDirectoryPath);
+  for await (const localDirEntry of localDirListing) {
+    let entry: DirectoryEntry;
+    if (localDirEntry.type === "-") {
+      const putFileResponse = await rpcClient.call.v1.dbfs.fileContents.put({
+        contents: localDirEntry.contents.toString("base64"),
+      });
+      entry = {
+        type: "-",
+        hash: putFileResponse.data.hash,
+        executable: localDirEntry.executable,
+        name: localDirEntry.name,
+      };
+    } else if (localDirEntry.type === "d") {
+      const childHash = await doUploadLocalDirectory(
+        rpcClient,
+        localDirEntry.localPath,
+      );
+      entry = { type: "d", name: localDirEntry.name, hash: childHash };
+    } else if (localDirEntry.type === "l") {
+      entry = {
+        type: "l",
+        target: localDirEntry.target,
+        name: localDirEntry.name,
+      };
+    } else {
+      unreachable(localDirEntry);
+    }
+    directoryContents.push(entry);
+  }
+  const putDirResponse = await rpcClient.call.v1.dbfs.directoryContents.put({
+    contents: directoryContents,
+  });
+  return putDirResponse.data.hash;
+}

package/src/dev-utils/custom-build.mts

@@ -0,0 +1,113 @@
+import path from "node:path";
+import watcher, { type FSWatcher } from "chokidar";
+import { red } from "colorette";
+import fs from "fs-extra";
+import semver from "semver";
+import { build } from "vite";
+import { getLegacyComponentsConfig } from "./custom-config.mjs";
+import type { Plugin, UserConfig, ViteDevServer } from "vite";
+
+// we only need to build if not on React 18
+export async function isCustomComponentsEnabled() {
+  try {
+    const packageJson = await fs.readJson(
+      path.join(process.cwd(), "custom", "package.json"),
+    );
+    const reactVersion =
+      packageJson.dependencies?.["react"] ||
+      packageJson.devDependencies?.["react"];
+    if (!reactVersion) {
+      return false;
+    }
+    return semver.lt(semver.coerce(reactVersion)!, "18.0.0");
+  } catch {
+    return false;
+  }
+}
+
+async function buildCustomComponents(config: UserConfig) {
+  try {
+    console.log("Building custom components...");
+    await build(config);
+    console.log("Custom components build complete");
+    return true;
+  } catch (error) {
+    console.error(red("Custom components build failed:"), "\n", error);
+    return false;
+  }
+}
+
+let __watcher: FSWatcher | undefined;
+
+async function watchCustomComponents(folder: string) {
+  if (__watcher) {
+    return __watcher;
+  }
+  __watcher = watcher.watch(folder, {
+    ignored: [/dist/, /node_modules/],
+    persistent: true,
+    ignoreInitial: true,
+  });
+  return __watcher;
+}
+
+async function buildCustomComponentsProject() {
+  const customComponentsFolder = path.join(process.cwd(), "custom");
+  const config = getLegacyComponentsConfig(customComponentsFolder);
+  await buildCustomComponents(config);
+}
+
+// on file changes, we want to rebuild the custom components library
+// and let our vite server process that the file as changed, which triggers HMR
+// Related reading: https://vite.dev/changes/hotupdate-hook.html
+// TODO: can we run a vite dev server instead of a vite build in order to use hotUpdate to get changes?
+async function watchForChanges(server: ViteDevServer) {
+  const customComponentsFolder = path.join(process.cwd(), "custom");
+  const watcher = await watchCustomComponents(customComponentsFolder);
+
+  const handleChange = async (filePath: string) => {
+    // We need to rebuild the configuration to pick up on any file system changes
+    const config = getLegacyComponentsConfig(customComponentsFolder);
+
+    console.log(`Custom component file changed: ${filePath}`);
+    const url = path.relative(customComponentsFolder, filePath);
+    await buildCustomComponents(config);
+
+    const module = await server.moduleGraph.getModuleByUrl(
+      `/custom/dist/${url}`.replace(/\.tsx?$/, ".js"),
+    );
+    if (module) {
+      server.reloadModule(module);
+    }
+  };
+
+  watcher.on("change", handleChange);
+  watcher.on("add", handleChange);
+  watcher.on("unlink", handleChange);
+}
+
+export const customComponentsPlugin = (): Plugin => {
+  return {
+    enforce: "pre",
+    name: "custom-components",
+    // vite is calling this twice during the initial dev server start due to how remix loads
+    // configs via resolveConfig
+    async buildStart() {
+      const enabled = await isCustomComponentsEnabled();
+      if (!enabled) {
+        return;
+      }
+      await buildCustomComponentsProject();
+    },
+    async configureServer(server) {
+      const enabled = await isCustomComponentsEnabled();
+      if (!enabled) {
+        return;
+      }
+      await watchForChanges(server);
+    },
+    buildEnd() {
+      __watcher?.close();
+    },
+  };
+};