@solana-mobile/dapp-store-cli 0.16.0 → 1.0.0

package/src/upload/CachedStorageDriver.ts

@@ -1,179 +0,0 @@
-import fs from "fs";
-import path from "path";
-import type { MetaplexFile, StorageDriver } from "@metaplex-foundation/js";
-import { createHash } from "crypto";
-import { normalizePublicContentUrl } from "./contentGateway.js";
-
-type URI = string;
-
-type Asset = {
-  path: string;
-  sha256: string;
-  uri: URI;
-};
-
-export type AssetManifestSchema = {
-  schema_version: string;
-  assets: {
-    [path: string]: Asset;
-  };
-};
-
-// TODO(jon): We need to manage the removal / replacement of assets in the manifest
-export class CachedStorageDriver implements StorageDriver {
-  // NOTE: this schema version is independent of the publishing JSON schema. It should be updated
-  // when the AssetManifestSchema or Asset types are updated.
-  static readonly SCHEMA_VERSION = "0.1";
-
-  assetManifest: AssetManifestSchema;
-  assetManifestPath: string;
-  storageDriver: StorageDriver;
-
-  constructor(
-    storageDriver: StorageDriver,
-    { assetManifestPath }: { assetManifestPath: string }
-  ) {
-    this.assetManifestPath = assetManifestPath;
-    this.assetManifest = this.loadAssetManifest(assetManifestPath) ?? {
-      schema_version: CachedStorageDriver.SCHEMA_VERSION,
-      assets: {},
-    };
-    this.storageDriver = storageDriver;
-  }
-
-  async getUploadPrice(bytes: number) {
-    return this.storageDriver.getUploadPrice(bytes);
-  }
-
-  private resolveAssetManifestPath(filename = this.assetManifestPath): string {
-    return path.resolve(process.cwd(), filename);
-  }
-
-  private normalizeAsset(
-    filename: string,
-    asset: unknown
-  ): Asset | undefined {
-    if (!asset || typeof asset !== "object") return;
-
-    const candidate = asset as Partial<Asset>;
-    const pathValue =
-      typeof candidate.path === "string" ? candidate.path : filename;
-
-    if (
-      typeof candidate.sha256 !== "string" ||
-      typeof candidate.uri !== "string"
-    ) {
-      return;
-    }
-
-    return {
-      path: pathValue,
-      sha256: candidate.sha256,
-      uri: candidate.uri,
-    };
-  }
-
-  private normalizeAssetManifest(
-    assetManifest: Partial<AssetManifestSchema> | undefined
-  ): AssetManifestSchema | undefined {
-    if (!assetManifest || typeof assetManifest !== "object") return;
-
-    const assets: Record<string, Asset> = {};
-    const assetEntries =
-      assetManifest.assets && typeof assetManifest.assets === "object"
-        ? Object.entries(assetManifest.assets)
-        : [];
-
-    for (const [filename, asset] of assetEntries) {
-      const normalizedAsset = this.normalizeAsset(filename, asset);
-      if (normalizedAsset) {
-        assets[filename] = normalizedAsset;
-      }
-    }
-
-    return {
-      schema_version:
-        typeof assetManifest.schema_version === "string"
-          ? assetManifest.schema_version
-          : CachedStorageDriver.SCHEMA_VERSION,
-      assets,
-    };
-  }
-
-  private async writeAssetManifest(): Promise<void> {
-    const normalizedAssetManifest = this.normalizeAssetManifest(
-      this.assetManifest
-    );
-
-    if (!normalizedAssetManifest) {
-      throw new Error("Asset manifest is not serializable");
-    }
-
-    this.assetManifest = normalizedAssetManifest;
-    await fs.promises.writeFile(
-      this.resolveAssetManifestPath(),
-      JSON.stringify(this.assetManifest, null, 2),
-      "utf-8"
-    );
-  }
-
-  loadAssetManifest(filename: string): AssetManifestSchema | undefined {
-    try {
-      return this.normalizeAssetManifest(
-        JSON.parse(fs.readFileSync(this.resolveAssetManifestPath(filename), "utf-8"))
-      );
-    } catch (error) {
-      console.warn(`Failed opening ${filename}; initializing with a blank asset manifest`);
-      return;
-    }
-  }
-
-  uploadedAsset(filename: string, { sha256 }: { sha256: string }) {
-    if (this.assetManifest.assets[filename]?.sha256 === sha256) {
-      return this.assetManifest.assets[filename];
-    }
-    return null;
-  }
-
-  async upload(file: MetaplexFile): Promise<string> {
-    // `inline.json` is the NFT-related metadata. This data is not stable so we'll skip the caching step
-    if (file.fileName === "inline.json") {
-      return normalizePublicContentUrl(await this.storageDriver.upload(file));
-    }
-    const hash = createHash("sha256").update(file.buffer).digest("base64");
-
-    const uploadedAsset = this.uploadedAsset(file.fileName, { sha256: hash });
-    if (uploadedAsset) {
-      const normalizedUri = normalizePublicContentUrl(uploadedAsset.uri);
-      if (normalizedUri !== uploadedAsset.uri) {
-        uploadedAsset.uri = normalizedUri;
-        try {
-          await this.writeAssetManifest();
-        } catch (error) {
-          const message = error instanceof Error ? error.message : String(error);
-          console.warn(
-            `Failed to rewrite ${this.assetManifestPath}; continuing with normalized URL: ${message}`
-          );
-        }
-      }
-      console.log(
-        `Asset ${file.fileName} already uploaded at ${normalizedUri}`
-      );
-      return normalizedUri;
-    }
-
-    console.log(`Uploading ${file.fileName}`);
-    const uri = normalizePublicContentUrl(await this.storageDriver.upload(file));
-
-    this.assetManifest.assets[file.fileName] = {
-      path: file.fileName,
-      sha256: hash,
-      uri,
-    };
-
-    await this.writeAssetManifest();
-    console.log(`${file.fileName} uploaded at ${uri}`)
-
-    return uri;
-  }
-}

package/src/upload/TurboStorageDriver.ts

@@ -1,283 +0,0 @@
-import type { Keypair } from "@solana/web3.js";
-import type { MetaplexFile } from "@metaplex-foundation/js";
-import { TurboFactory, lamportToTokenAmount } from "@ardrive/turbo-sdk";
-import bs58 from "bs58";
-import debugModule from "debug";
-import { buildPublicContentUrl, type StorageNetwork } from "./contentGateway.js";
-
-const debug = debugModule("cli:turbo-storage");
-
-interface TurboClient {
-  getUploadCosts(args: {
-    bytes: number[];
-  }): Promise<Array<{ winc: string | number | bigint }>>;
-  getBalance(): Promise<{ winc: string | number | bigint }>;
-  getWincForToken?(args: {
-    tokenAmount: number;
-  }): Promise<{ winc: string | number | bigint }>;
-  topUpWithTokens?(args: { tokenAmount: string | number }): Promise<unknown>;
-  uploadFile(args: {
-    fileStreamFactory: () => Buffer;
-    fileSizeFactory: () => number;
-    dataItemOpts?: { tags?: Array<{ name: string; value: string }> };
-  }): Promise<{ id: string }>;
-}
-
-type TurboServiceConfig = {
-  gatewayUrl?: string;
-  paymentServiceConfig?: { url: string };
-  uploadServiceConfig?: { url: string };
-};
-
-const SOL_IN_LAMPORTS = 1_000_000_000;
-const MIN_TOP_UP_LAMPORTS = 1_000_000;
-const MIN_TOP_UP_SOL = MIN_TOP_UP_LAMPORTS / SOL_IN_LAMPORTS;
-
-const CONSTANTS = {
-  FREE_UPLOAD_LIMIT: 97_280, // 95 KiB
-  UPLOAD_DELAY_MS: 2000,
-  MAX_RETRIES: 5,
-  SOL_IN_LAMPORTS,
-  MIN_TOP_UP_SOL,
-  MIN_TOP_UP_LAMPORTS,
-  BACKOFF: {
-    BASE_MS: 500,
-    MAX_MS: 8000,
-  },
-  SERVICE_URLS: {
-    devnet: {
-      gatewayUrl: "https://api.devnet.solana.com",
-      uploadServiceConfig: { url: "https://upload.ardrive.dev" },
-      paymentServiceConfig: { url: "https://payment.ardrive.dev" },
-    },
-  },
-} as const;
-
-const delay = (ms: number): Promise<void> =>
-  new Promise((resolve) => setTimeout(resolve, ms));
-
-export class TurboStorageDriver {
-  private turbo: TurboClient;
-  private bufferPercentage: number;
-  private network: StorageNetwork;
-
-  private uploadQueue: Array<{
-    file: MetaplexFile;
-    resolve: (url: string) => void;
-    reject: (error: Error) => void;
-  }> = [];
-  private isProcessingQueue = false;
-
-  constructor(
-    keypair: Keypair,
-    network: StorageNetwork = "mainnet",
-    bufferPercentage = 20
-  ) {
-    this.network = network;
-    this.bufferPercentage = bufferPercentage;
-
-    this.turbo = TurboFactory.authenticated({
-      privateKey: bs58.encode(keypair.secretKey),
-      token: "solana",
-      ...getTurboServiceConfig(network),
-    }) as TurboClient;
-  }
-
-  async getUploadPrice(bytes: number): Promise<bigint> {
-    if (bytes <= CONSTANTS.FREE_UPLOAD_LIMIT) return BigInt(0);
-
-    const [cost] = await this.turbo.getUploadCosts({ bytes: [bytes] });
-    const base = BigInt(String(cost.winc));
-    return (base * BigInt(100 + this.bufferPercentage)) / BigInt(100);
-  }
-
-  private async withRetry<T>(
-    operation: () => Promise<T>,
-    isRetriable: (error: string) => boolean = (msg) =>
-      msg.includes("429") || msg.includes("Too Many Requests")
-  ): Promise<T> {
-    for (let retry = 0; retry <= CONSTANTS.MAX_RETRIES; retry++) {
-      try {
-        return await operation();
-      } catch (error) {
-        const errorMessage =
-          error instanceof Error ? error.message : String(error);
-
-        if (retry < CONSTANTS.MAX_RETRIES && isRetriable(errorMessage)) {
-          const delayMs = Math.min(
-            CONSTANTS.BACKOFF.BASE_MS * Math.pow(2, retry),
-            CONSTANTS.BACKOFF.MAX_MS
-          );
-          console.log(
-            `Rate limited, retrying after ${delayMs}ms (attempt ${retry + 1}/${
-              CONSTANTS.MAX_RETRIES
-            })...`
-          );
-          await delay(delayMs);
-          continue;
-        }
-        throw error;
-      }
-    }
-    throw new Error("Max retries exceeded");
-  }
-
-  private formatInsufficientFundsError(errorMessage: string): void {
-    const match = errorMessage.match(/insufficient lamports (\d+), need (\d+)/);
-    if (!match) return;
-
-    const [current, needed] = [BigInt(match[1]), BigInt(match[2])];
-    const [currentSOL, neededSOL] = [
-      Number(current) / 1e9,
-      Number(needed) / 1e9,
-    ];
-
-    console.error(`\nInsufficient SOL balance for top-up:`);
-    console.error(`  Current: ${currentSOL.toFixed(9)} SOL`);
-    console.error(`  Required: ${neededSOL.toFixed(9)} SOL`);
-    console.error(`  Shortfall: ${(neededSOL - currentSOL).toFixed(9)} SOL\n`);
-  }
-
-  private async topUpCredits(wincAmount: bigint): Promise<void> {
-    if (wincAmount === 0n) {
-      debug("No Winston Credits requested; skipping top-up.");
-      return;
-    }
-
-    try {
-      await this.withRetry(async () => {
-        const exchangeRate = await this.turbo.getWincForToken?.({
-          tokenAmount: CONSTANTS.SOL_IN_LAMPORTS,
-        });
-
-        if (!exchangeRate) {
-          throw new Error("Unable to get Winston Credits exchange rate");
-        }
-
-        const wincPerSol = BigInt(String(exchangeRate.winc));
-        if (wincPerSol <= 0n) {
-          throw new Error("Invalid Winston Credits exchange rate");
-        }
-
-        const solInLamports = BigInt(CONSTANTS.SOL_IN_LAMPORTS);
-        const numerator = wincAmount * solInLamports;
-        const lamportsCalculated =
-          (numerator + (wincPerSol - 1n)) / wincPerSol;
-
-        const minLamports = BigInt(CONSTANTS.MIN_TOP_UP_LAMPORTS);
-        const lamportsToUse =
-          lamportsCalculated < minLamports ? minLamports : lamportsCalculated;
-
-        if (lamportsToUse > lamportsCalculated) {
-          debug(
-            `Applying minimum top-up of ${CONSTANTS.MIN_TOP_UP_SOL} SOL (${CONSTANTS.MIN_TOP_UP_LAMPORTS} lamports)`
-          );
-        }
-        const solAmount =
-          Number(lamportsToUse) / CONSTANTS.SOL_IN_LAMPORTS;
-
-        debug(
-          `Buying at least ${wincAmount} Winston Credits (~${solAmount.toFixed(9)} SOL / ${lamportsToUse} lamports)`
-        );
-
-        await this.turbo.topUpWithTokens?.({
-          tokenAmount: String(
-            lamportToTokenAmount(lamportsToUse.toString())
-          ),
-        });
-
-        debug(`Top-up initiated for ${wincAmount} Winston Credits`);
-      });
-    } catch (error) {
-      const errorMessage =
-        error instanceof Error ? error.message : String(error);
-      debug("Top-up failed:", error);
-
-      if (errorMessage.includes("insufficient lamports")) {
-        this.formatInsufficientFundsError(errorMessage);
-      }
-
-      throw new Error(
-        `Failed to top up ${wincAmount} Winston Credits: ${errorMessage}`
-      );
-    }
-  }
-
-  private async checkBalanceAndTopUp(requiredWinc: bigint): Promise<void> {
-    if (requiredWinc === BigInt(0)) return;
-
-    const current = BigInt(String((await this.turbo.getBalance()).winc));
-
-    if (current >= requiredWinc) {
-      debug(
-        `Sufficient balance: ${current} Winston Credits (required: ${requiredWinc})`
-      );
-      return;
-    }
-
-    const deficit = requiredWinc - current;
-    debug(
-      `Current: ${current}, Required: ${requiredWinc}, Topping up: ${deficit}`
-    );
-    await this.topUpCredits(deficit);
-  }
-
-  private async processQueue(): Promise<void> {
-    if (this.isProcessingQueue || !this.uploadQueue.length) return;
-
-    this.isProcessingQueue = true;
-
-    while (this.uploadQueue.length > 0) {
-      const item = this.uploadQueue.shift();
-      if (!item) continue;
-
-      try {
-        debug(
-          `Processing upload for ${item.file.fileName} (${item.file.buffer.length} bytes)`
-        );
-
-        const estimated = await this.getUploadPrice(item.file.buffer.length);
-        await this.checkBalanceAndTopUp(estimated);
-
-        const tags = [...(item.file.tags ?? [])];
-        if (item.file.contentType) {
-          tags.push({ name: "Content-Type", value: item.file.contentType });
-        }
-
-        const uploadResult = await this.turbo.uploadFile({
-          fileStreamFactory: () => item.file.buffer,
-          fileSizeFactory: () => item.file.buffer.byteLength,
-          dataItemOpts: { tags },
-        });
-
-        const url = buildPublicContentUrl(uploadResult.id, this.network);
-        debug(`Upload complete: ${url}`);
-        item.resolve(url);
-
-        if (this.uploadQueue.length > 0) {
-          debug(`Waiting ${CONSTANTS.UPLOAD_DELAY_MS}ms before next upload...`);
-          await delay(CONSTANTS.UPLOAD_DELAY_MS);
-        }
-      } catch (error) {
-        item.reject(error instanceof Error ? error : new Error(String(error)));
-      }
-    }
-
-    this.isProcessingQueue = false;
-  }
-
-  async upload(file: MetaplexFile): Promise<string> {
-    return new Promise((resolve, reject) => {
-      debug(
-        `Queueing upload for ${file.fileName} (${file.buffer.length} bytes)`
-      );
-      this.uploadQueue.push({ file, resolve, reject });
-      this.processQueue().catch(reject);
-    });
-  }
-}
-
-export const getTurboServiceConfig = (
-  network: StorageNetwork
-): TurboServiceConfig =>
-  network === "devnet" ? CONSTANTS.SERVICE_URLS.devnet : {};

package/src/upload/__tests__/CachedStorageDriver.test.ts

@@ -1,246 +0,0 @@
-import fs from "fs";
-import os from "os";
-import path from "path";
-import { createHash } from "crypto";
-import { jest } from "@jest/globals";
-import type { MetaplexFile, StorageDriver } from "@metaplex-foundation/js";
-import { CachedStorageDriver } from "../CachedStorageDriver";
-
-type MockStorageDriver = Pick<StorageDriver, "getUploadPrice" | "upload">;
-
-describe("CachedStorageDriver", () => {
-  const originalCwd = process.cwd();
-  let tempDir: string;
-
-  beforeEach(() => {
-    tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "cached-storage-driver-"));
-    process.chdir(tempDir);
-  });
-
-  afterEach(() => {
-    process.chdir(originalCwd);
-    fs.rmSync(tempDir, { recursive: true, force: true });
-  });
-
-  test("normalizes a cached legacy arweave URL and persists the rewritten manifest", async () => {
-    const file = makeMetaplexFile("icon.png", Buffer.from("cached-asset"));
-    const hash = hashBuffer(file.buffer);
-    const manifestPath = ".asset-manifest.json";
-
-    fs.writeFileSync(
-      path.join(tempDir, manifestPath),
-      JSON.stringify(
-        {
-          assets: {
-            [file.fileName]: {
-              path: file.fileName,
-              sha256: hash,
-              uri: "https://arweave.net/legacy-id",
-            },
-          },
-        },
-        null,
-        2
-      )
-    );
-
-    let uploadCalls = 0;
-    const storageDriver: MockStorageDriver = {
-      getUploadPrice: async (_bytes: number) => {
-        throw new Error("getUploadPrice should not be called in this test");
-      },
-      upload: async (_file: MetaplexFile) => {
-        uploadCalls += 1;
-        throw new Error("upload should not be called in this test");
-      },
-    };
-
-    const driver = new CachedStorageDriver(storageDriver as StorageDriver, {
-      assetManifestPath: manifestPath,
-    });
-
-    await expect(driver.upload(file)).resolves.toBe(
-      "https://dappstorecontent.com/legacy-id"
-    );
-    expect(uploadCalls).toBe(0);
-
-    const rewrittenManifest = JSON.parse(
-      fs.readFileSync(path.join(tempDir, manifestPath), "utf-8")
-    );
-    expect(rewrittenManifest.schema_version).toBe("0.1");
-    expect(rewrittenManifest.assets[file.fileName].uri).toBe(
-      "https://dappstorecontent.com/legacy-id"
-    );
-  });
-
-  test("supports absolute asset manifest paths", async () => {
-    const file = makeMetaplexFile("absolute-icon.png", Buffer.from("cached-asset"));
-    const hash = hashBuffer(file.buffer);
-    const manifestPath = path.join(tempDir, ".asset-manifest.json");
-
-    fs.writeFileSync(
-      manifestPath,
-      JSON.stringify(
-        {
-          assets: {
-            [file.fileName]: {
-              path: file.fileName,
-              sha256: hash,
-              uri: "https://arweave.net/absolute-id",
-            },
-          },
-        },
-        null,
-        2
-      )
-    );
-
-    let uploadCalls = 0;
-    const storageDriver: MockStorageDriver = {
-      getUploadPrice: async (_bytes: number) => {
-        throw new Error("getUploadPrice should not be called in this test");
-      },
-      upload: async (_file: MetaplexFile) => {
-        uploadCalls += 1;
-        throw new Error("upload should not be called in this test");
-      },
-    };
-
-    const driver = new CachedStorageDriver(storageDriver as StorageDriver, {
-      assetManifestPath: manifestPath,
-    });
-
-    await expect(driver.upload(file)).resolves.toBe(
-      "https://dappstorecontent.com/absolute-id"
-    );
-    expect(uploadCalls).toBe(0);
-
-    const rewrittenManifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
-    expect(rewrittenManifest.assets[file.fileName].uri).toBe(
-      "https://dappstorecontent.com/absolute-id"
-    );
-  });
-
-  test("normalizes fresh uploads before persisting them in the manifest", async () => {
-    const file = makeMetaplexFile("banner.png", Buffer.from("fresh-asset"));
-    const manifestPath = ".asset-manifest.json";
-
-    let uploadCalls = 0;
-    const storageDriver: MockStorageDriver = {
-      getUploadPrice: async (_bytes: number) => {
-        throw new Error("getUploadPrice should not be called in this test");
-      },
-      upload: async (_file: MetaplexFile) => {
-        uploadCalls += 1;
-        return "https://arweave.com/fresh-id";
-      },
-    };
-
-    const driver = new CachedStorageDriver(storageDriver as StorageDriver, {
-      assetManifestPath: manifestPath,
-    });
-
-    await expect(driver.upload(file)).resolves.toBe(
-      "https://dappstorecontent.com/fresh-id"
-    );
-    expect(uploadCalls).toBe(1);
-
-    const persistedManifest = JSON.parse(
-      fs.readFileSync(path.join(tempDir, manifestPath), "utf-8")
-    );
-    expect(persistedManifest.schema_version).toBe("0.1");
-    expect(persistedManifest.assets[file.fileName].uri).toBe(
-      "https://dappstorecontent.com/fresh-id"
-    );
-  });
-
-  test("normalizes inline metadata uploads without caching them", async () => {
-    const file = makeMetaplexFile("inline.json", Buffer.from("{}"));
-    let uploadCalls = 0;
-    const storageDriver: MockStorageDriver = {
-      getUploadPrice: async (_bytes: number) => {
-        throw new Error("getUploadPrice should not be called in this test");
-      },
-      upload: async (_file: MetaplexFile) => {
-        uploadCalls += 1;
-        return "https://arweave.net/metadata-id";
-      },
-    };
-
-    const driver = new CachedStorageDriver(storageDriver as StorageDriver, {
-      assetManifestPath: ".asset-manifest.json",
-    });
-
-    await expect(driver.upload(file)).resolves.toBe(
-      "https://dappstorecontent.com/metadata-id"
-    );
-    expect(uploadCalls).toBe(1);
-    expect(fs.existsSync(path.join(tempDir, ".asset-manifest.json"))).toBe(
-      false
-    );
-  });
-
-  test("returns a normalized cached URL even when manifest rewrite persistence fails", async () => {
-    const file = makeMetaplexFile("icon.png", Buffer.from("cached-asset"));
-    const hash = hashBuffer(file.buffer);
-    const manifestPath = ".asset-manifest.json";
-
-    fs.writeFileSync(
-      path.join(tempDir, manifestPath),
-      JSON.stringify(
-        {
-          assets: {
-            [file.fileName]: {
-              path: file.fileName,
-              sha256: hash,
-              uri: "https://arweave.net/rewrite-failure-id",
-            },
-          },
-        },
-        null,
-        2
-      )
-    );
-
-    const writeFileSpy = jest
-      .spyOn(fs.promises, "writeFile")
-      .mockRejectedValueOnce(new Error("disk full"));
-    const warnSpy = jest.spyOn(console, "warn").mockImplementation(() => undefined);
-
-    const storageDriver: MockStorageDriver = {
-      getUploadPrice: async (_bytes: number) => {
-        throw new Error("getUploadPrice should not be called in this test");
-      },
-      upload: async (_file: MetaplexFile) => {
-        throw new Error("upload should not be called in this test");
-      },
-    };
-
-    const driver = new CachedStorageDriver(storageDriver as StorageDriver, {
-      assetManifestPath: manifestPath,
-    });
-
-    try {
-      await expect(driver.upload(file)).resolves.toBe(
-        "https://dappstorecontent.com/rewrite-failure-id"
-      );
-      expect(warnSpy).toHaveBeenCalledWith(
-        expect.stringContaining(
-          "Failed to rewrite .asset-manifest.json; continuing with normalized URL"
-        )
-      );
-    } finally {
-      writeFileSpy.mockRestore();
-      warnSpy.mockRestore();
-    }
-  });
-});
-
-const makeMetaplexFile = (fileName: string, buffer: Buffer): MetaplexFile =>
-  ({
-    fileName,
-    buffer,
-  }) as MetaplexFile;
-
-const hashBuffer = (buffer: Buffer): string =>
-  createHash("sha256").update(buffer).digest("base64");