@effectstream/npm-midnight-indexer 0.3.99 → 0.100.8

package/binary.js

@@ -4,7 +4,8 @@ const axios = require("axios");
 const extract = require("extract-zip");
 const path = require("path");
 
-const CURRENT_BINARY_VERSION = "v2.1.4";
+const CURRENT_BINARY_VERSION = "v4.0.0";
+const FINAL_BINARY_NAME = "indexer-standalone";
 
 /*
 @returns {string} The platform and architecture of the current machine. Example: "linux-amd64"
@@ -39,9 +40,12 @@ function getBinaryUrl() {
   if (!supportedPlatforms.includes(platform)) {
     throw new Error(`Unsupported platform: ${platform}`);
   }
-  return `https://paima-midnight.nyc3.cdn.digitaloceanspaces.com/binaries/indexer-standalone-${platform}-${CURRENT_BINARY_VERSION}.zip`;
+
+  return `https://github.com/effectstream/binaries/releases/download/0.3.120/indexer-standalone-${platform}-${CURRENT_BINARY_VERSION}.zip`;
 }
 
+const ZIP_FILE_PATH = path.join(__dirname, "indexer-standalone.zip");
+
 /*
 @returns {Promise<void>} Downloads and saves the binary for the current platform.
 */
@@ -50,9 +54,7 @@ async function downloadAndSaveBinary() {
   try {
     console.error(`Downloading... ${url}`);
     const response = await axios.get(url, { responseType: "stream" });
-    const writer = fs.createWriteStream(
-      path.join(__dirname, "indexer-standalone.zip"),
-    );
+    const writer = fs.createWriteStream(ZIP_FILE_PATH);
 
     response.data.pipe(writer);
 
@@ -74,21 +76,73 @@ async function unzipBinary() {
   if (!fs.existsSync(dir)) {
     fs.mkdirSync(dir, { recursive: true });
   }
-  await extract(path.join(__dirname, "indexer-standalone.zip"), { dir });
-  fs.unlinkSync(path.join(__dirname, "indexer-standalone.zip"));
+  await extract(ZIP_FILE_PATH, { dir });
+  const dataDir = path.join(dir, "data");
+  if (!fs.existsSync(dataDir)) {
+    fs.mkdirSync(dataDir, { recursive: true });
+  }
 
   const platform = getPlatform();
-  const parts = platform.split("-");
-  if (parts[0] === "linux") {
-    fs.chmodSync(
-      path.join(
-        __dirname,
-        "indexer-standalone",
-        `indexer-standalone-${platform}`,
-      ),
-      0o755,
-    );
+  const possibleBinaryNames = [
+    `indexer-standalone-${platform}-${CURRENT_BINARY_VERSION}`,
+    `indexer-standalone`,
+    `indexer-standalone-${platform}`
+  ];
+
+  let extractedBinaryPath = null;
+  for (const name of possibleBinaryNames) {
+    const p = path.join(dir, name);
+    if (fs.existsSync(p) && fs.statSync(p).isFile()) {
+      extractedBinaryPath = p;
+      break;
+    }
+  }
+
+  const finalBinaryPath = path.join(dir, FINAL_BINARY_NAME);
+
+  // Rename the extracted file to indexer-standalone
+  if (extractedBinaryPath) {
+    if (extractedBinaryPath !== finalBinaryPath) {
+      if (fs.existsSync(finalBinaryPath)) {
+        fs.unlinkSync(finalBinaryPath);
+      }
+      fs.renameSync(extractedBinaryPath, finalBinaryPath);
+    }
+  } else {
+    throw new Error(`Extracted binary not found in: ${dir}. Expected one of: ${possibleBinaryNames.join(", ")}`);
+  }
+
+  // Clean up any other extracted files (e.g., readme.md)
+  const files = fs.readdirSync(dir);
+  for (const file of files) {
+    const filePath = path.join(dir, file);
+    // Explicitly protect config.yaml and other potential config files
+    const isConfigFile =
+      file === "config.yaml" ||
+      file === "config.yml" ||
+      file.endsWith(".yaml") ||
+      file.endsWith(".yml") ||
+      file.endsWith(".toml") ||
+      file.endsWith(".json");
+
+    if (
+      filePath !== finalBinaryPath &&
+      file !== "data" &&
+      !isConfigFile &&
+      fs.existsSync(filePath) &&
+      fs.statSync(filePath).isFile()
+    ) {
+      console.log(`Cleaning up extracted file: ${file}`);
+      fs.unlinkSync(filePath);
+    }
+  }
+
+  if (!fs.existsSync(finalBinaryPath)) {
+    throw new Error(`Expected binary not found: ${finalBinaryPath}`);
   }
+
+  fs.chmodSync(finalBinaryPath, 0o755);
+  fs.unlinkSync(ZIP_FILE_PATH);
 }
 
 async function binary() {
@@ -96,7 +150,68 @@ async function binary() {
   await unzipBinary();
 }
 
+async function cleanBinaries() {
+  const binaryDir = path.join(__dirname, "indexer-standalone");
+  const binaryPath = path.join(binaryDir, FINAL_BINARY_NAME);
+  const zipPath = path.join(__dirname, "indexer-standalone.zip");
+
+  let deletedFiles = [];
+
+  // Delete the binary executable if it exists
+  if (fs.existsSync(binaryPath)) {
+    try {
+      fs.unlinkSync(binaryPath);
+      deletedFiles.push(binaryPath);
+    } catch (error) {
+      console.error(`Error removing binary ${binaryPath}:`, error.message);
+    }
+  }
+
+  // Delete the zip file if it exists
+  if (fs.existsSync(zipPath)) {
+    try {
+      fs.unlinkSync(zipPath);
+      deletedFiles.push(zipPath);
+    } catch (error) {
+      console.error(`Error removing file ${zipPath}:`, error.message);
+    }
+  }
+
+  // Delete any other binary-related files from the directory, preserving config files
+  if (fs.existsSync(binaryDir)) {
+    const files = fs.readdirSync(binaryDir);
+    for (const file of files) {
+      const filePath = path.join(binaryDir, file);
+      const stat = fs.statSync(filePath);
+
+      // Preserve config files and readme
+      const isConfigFile =
+        file === "config.yaml" ||
+        file === "config.yml" ||
+        file.endsWith(".yaml") ||
+        file.endsWith(".yml") ||
+        file.endsWith(".toml") ||
+        file.endsWith(".json") ||
+        file.toLowerCase().startsWith("readme") ||
+        file.toLowerCase().startsWith("license") ||
+        file.toLowerCase().startsWith("changelog");
+
+      if (!isConfigFile && stat.isFile() && fs.existsSync(filePath)) {
+        try {
+          fs.unlinkSync(filePath);
+          deletedFiles.push(filePath);
+        } catch (error) {
+          console.error(`Error removing file ${filePath}:`, error.message);
+        }
+      }
+    }
+  }
+
+  return deletedFiles;
+}
+
 module.exports = {
   binary,
   getPlatform,
+  cleanBinaries,
 };

package/index.js

@@ -1,4 +1,4 @@
-const { binary, getPlatform } = require("./binary");
+const { binary, getPlatform, cleanBinaries } = require("./binary");
 const { runMidnightIndexer } = require("./run_midnight_indexer");
 const { checkIfDockerExists, pullDockerImage, runDockerContainer } = require(
   "./docker",
@@ -8,12 +8,11 @@ const path = require("path");
 const readline = require("readline");
 const os = require("os");
 
+const FINAL_BINARY_NAME = "indexer-standalone";
+
 function checkIfBinaryExists() {
-  const platform = getPlatform();
-  const parts = platform.split("-");
-  const binaryName = `indexer-standalone-${platform}`;
   return fs.existsSync(
-    path.join(__dirname, "indexer-standalone", binaryName),
+    path.join(__dirname, "indexer-standalone", FINAL_BINARY_NAME),
   );
 }
 
@@ -22,21 +21,9 @@ function checkIfBinaryExists() {
  * @returns {boolean} True if binary execution is supported
  */
 function isBinarySupported() {
-  const platform = os.platform();
-  const arch = os.arch();
-
-  // Check if the current platform-arch combination is supported
+  const platformString = getPlatform();
   const supportedPlatforms = require("./package.json").supportedPlatforms;
 
-  let platformString;
-  if (platform === "darwin") {
-    platformString = arch === "x64" ? "macos-amd64" : `macos-${arch}`;
-  } else {
-    platformString = arch === "x64"
-      ? `${platform}-amd64`
-      : `${platform}-${arch}`;
-  }
-
   return supportedPlatforms.includes(platformString);
 }
 
@@ -69,9 +56,12 @@ function showUsage() {
 Usage: node index.js [options] [args...]
 
 Options:
-  --docker    Force use of Docker container
-  --binary    Force use of binary execution (supports Linux and macOS arm64)
-  --help      Show this help message
+  --docker         Force use of Docker container
+  --binary         Force use of binary execution (supports Linux and macOS arm64)
+  --clean-binaries Delete downloaded binaries and download them again
+  --only-clean     Only delete downloaded binaries without downloading them again
+  --clean          Clean the SQLite database (existing flag)
+  --help           Show this help message
 
 Environment Variables:
   APP__INFRA__SECRET             Required: Secret key for the application
@@ -87,10 +77,12 @@ Environment Variables:
 Note: macOS Intel users will automatically use Docker. macOS arm64 supports both binary and Docker execution.
 
 Examples:
-  APP__INFRA__SECRET=mysecret node index.js --docker    # Use Docker
-  APP__INFRA__SECRET=mysecret node index.js --binary    # Use binary (if supported)
-  APP__INFRA__SECRET=mysecret node index.js             # Interactive mode (or auto-Docker on unsupported platforms)
-  node index.js --help                                  # Show this help
+  APP__INFRA__SECRET=mysecret node index.js --docker          # Use Docker
+  APP__INFRA__SECRET=mysecret node index.js --binary          # Use binary (if supported)
+  APP__INFRA__SECRET=mysecret node index.js --clean-binaries   # Delete and redownload binaries
+  node index.js --only-clean                                 # Only delete downloaded binaries
+  APP__INFRA__SECRET=mysecret node index.js --clean           # Clean SQLite database
+  node index.js --help                                       # Show this help
 `);
 }
 
@@ -103,6 +95,8 @@ function parseFlags(args) {
   const flags = {
     useDocker: false,
     useBinary: false,
+    cleanBinaries: false,
+    onlyClean: false,
     showHelp: false,
     remainingArgs: [],
   };
@@ -112,6 +106,10 @@ function parseFlags(args) {
       flags.useDocker = true;
     } else if (args[i] === "--binary") {
       flags.useBinary = true;
+    } else if (args[i] === "--clean-binaries") {
+      flags.cleanBinaries = true;
+    } else if (args[i] === "--only-clean") {
+      flags.onlyClean = true;
     } else if (args[i] === "--help" || args[i] === "-h") {
       flags.showHelp = true;
     } else {
@@ -166,7 +164,7 @@ function setBinaryDefaults(env) {
   return binaryEnv;
 }
 
-async function runWithBinary(env, args) {
+async function runWithBinary(env, args, forceClean = false) {
   console.log("Using binary to run midnight indexer...");
 
   // Check for required environment variable
@@ -176,8 +174,12 @@ async function runWithBinary(env, args) {
     process.exit(1);
   }
 
-  if (!checkIfBinaryExists()) {
-    console.log("Binary not found, downloading...");
+  if (forceClean || !checkIfBinaryExists()) {
+    if (forceClean) {
+      console.log("Cleaning downloaded binaries...");
+      await cleanBinaries();
+    }
+    console.log("Downloading binary...");
     await binary();
   }
 
@@ -197,6 +199,18 @@ async function main(args) {
     process.exit(0);
   }
 
+  // Handle --only-clean flag
+  if (flags.onlyClean) {
+    console.log("Cleaning downloaded binaries...");
+    const deletedFiles = await cleanBinaries();
+    if (deletedFiles.length > 0) {
+      console.log("Deleted:", deletedFiles.join(", "));
+    } else {
+      console.log("No downloaded binaries found to delete.");
+    }
+    process.exit(0);
+  }
+
   // If both flags are provided, show error
   if (flags.useDocker && flags.useBinary) {
     console.error(
@@ -205,6 +219,14 @@ async function main(args) {
     process.exit(1);
   }
 
+  // Validate clean flag usage
+  if (flags.cleanBinaries && flags.useDocker) {
+    console.error(
+      "Error: --clean-binaries flag cannot be used with --docker flag",
+    );
+    process.exit(1);
+  }
+
   // If --docker flag is explicitly provided
   if (flags.useDocker) {
     const dockerAvailable = await checkIfDockerExists();
@@ -242,7 +264,7 @@ async function main(args) {
       );
       process.exit(1);
     }
-    return runWithBinary(env, flags.remainingArgs);
+    return runWithBinary(env, flags.remainingArgs, flags.cleanBinaries);
   }
 
   // No explicit flag provided - determine best execution method
@@ -298,7 +320,7 @@ async function main(args) {
   }
 
   // Default to binary execution (only on supported platforms)
-  return runWithBinary(env, flags.remainingArgs);
+  return runWithBinary(env, flags.remainingArgs, flags.clean);
 }
 
 // Handle unhandled promise rejections
@@ -313,4 +335,8 @@ process.on("SIGINT", () => {
   process.exit(0);
 });
 
+module.exports = {
+  cleanBinaries,
+};
+
 main(process.argv.slice(2));

package/indexer-standalone/config.yaml

@@ -1,37 +1,57 @@
-run_migrations: true
-network_id: &network_id "Undeployed"
-
-chain_indexer_application:
-  network_id: *network_id
-  blocks_buffer: 60
-  save_zswap_state_after: 1000
-  caught_up_max_distance: 60
-  caught_up_leeway: 30
+thread_stack_size: "24MiB"
 
-wallet_indexer_application:
-  network_id: *network_id
-  active_wallets_repeat_delay: "100ms"
+application:
+  network_id: "undeployed"
+  blocks_buffer: 10
+  save_ledger_state_after: 1000
+  caught_up_max_distance: 10
+  caught_up_leeway: 5
+  active_wallets_query_delay: "500ms"
   active_wallets_ttl: "30m"
-  transaction_batch_size: 10
-  # Number of cores by default.
-  # parallelism:
+  transaction_batch_size: 50
+  # 1 by default
+  # concurrency_limit:
+
+# CHECK: linux binary requires this flag in the root.
+run_migrations: true
 
 infra:
+  run_migrations: true
+  storage:
+    cnn_url: "data/indexer.sqlite"
+
+  ledger_db:
+    cache_size: "1kiB"
+    cnn_url: "data/ledger-db.sqlite"
+
   node:
     url: "ws://localhost:9944"
     reconnect_max_delay: "10s" # 10ms, 100ms, 1s, 10s
     reconnect_max_attempts: 30 # Roughly 5m
-
-  storage:
-    cnn_url: "./data/indexer.sqlite"
+    subscription_recovery_timeout: "30s" # Re-subscribe if no block received within this time
 
   api:
-    address: "127.0.0.1"
+    address: "0.0.0.0"
     port: 8088
     request_body_limit: "1MiB"
     max_complexity: 200
     max_depth: 15
-    network_id: *network_id
+    subscription:
+      blocks:
+        batch_size: 20
+      contract_actions:
+        batch_size: 20
+      dust_ledger_events:
+        batch_size: 20
+      shielded_transactions:
+        batch_size: 20
+        progress_update_interval: "30s"
+        keep_wallet_alive_interval: "1m"
+      unshielded_transactions:
+        batch_size: 20
+        progress_update_interval: "30s"
+      zswap_ledger_events:
+        batch_size: 20
 
 telemetry:
   tracing:

package/package.json

@@ -1,10 +1,12 @@
 {
   "name": "@effectstream/npm-midnight-indexer",
-  "version": "0.3.99",
+  "version": "0.100.8",
   "description": "Downloads and runs the Midnight Indexer. It needs a running Midnight Node",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "clean": "node index.js --clean-binaries",
+    "clean-only": "node index.js --only-clean"
   },
   "bin": {
     "npm-midnight-indexer": "index.js"
@@ -22,4 +24,4 @@
     "extract-zip": "^2.0.1",
     "js-yaml": "^4.1.0"
   }
-}
+}

package/run_midnight_indexer.js

@@ -2,7 +2,93 @@ const { spawn } = require("child_process");
 const path = require("path");
 const fs = require("fs");
 const yaml = require("js-yaml");
-const { getPlatform } = require("./binary");
+
+const BINARY_NAME = "indexer-standalone";
+
+/**
+ * Resolves the configuration file path
+ * @param {Object} env - Environment variables
+ * @param {string} workingDir - The working directory where the indexer runs
+ * @returns {string} The resolved configuration file path
+ */
+function resolveConfigPath(env, workingDir) {
+  if (env.CONFIG_FILE) {
+    return path.isAbsolute(env.CONFIG_FILE)
+      ? env.CONFIG_FILE
+      : path.resolve(workingDir, env.CONFIG_FILE);
+  }
+  // Fall back to config.yaml in the current working directory
+  return path.join(workingDir, "config.yaml");
+}
+
+/**
+ * Ensures that a configuration file exists at the given path
+ * @param {string} configPath - The path to the configuration file
+ * @param {Object} env - Environment variables
+ */
+function ensureConfigExists(configPath, env) {
+  if (fs.existsSync(configPath)) {
+    return;
+  }
+
+  console.log(
+    `Config file not found. Generating default config at: ${configPath}`,
+  );
+
+  const networkId = env.LEDGER_NETWORK_ID || "Undeployed";
+  const nodeUrl = env.SUBSTRATE_NODE_WS_URL ||
+    env.APP__INFRA__NODE__URL ||
+    "ws://localhost:9944";
+  const cnnUrl = env.APP__INFRA__STORAGE__CNN_URL || "./data/indexer.sqlite";
+  const apiPort = env.APP__INFRA__API__PORT || 8088;
+
+  const defaultConfig = `
+run_migrations: true
+network_id: &network_id "${networkId}"
+
+chain_indexer_application:
+  network_id: *network_id
+  blocks_buffer: 60
+  save_zswap_state_after: 1000
+  caught_up_max_distance: 60
+  caught_up_leeway: 30
+
+wallet_indexer_application:
+  network_id: *network_id
+  active_wallets_repeat_delay: "100ms"
+  active_wallets_ttl: "30m"
+  transaction_batch_size: 10
+
+infra:
+  node:
+    url: "${nodeUrl}"
+    reconnect_max_delay: "10s"
+    reconnect_max_attempts: 30
+
+  storage:
+    cnn_url: "${cnnUrl}"
+
+  api:
+    address: "0.0.0.0"
+    port: ${apiPort}
+    request_body_limit: "1MiB"
+    max_complexity: 200
+    max_depth: 15
+    network_id: *network_id
+
+telemetry:
+  tracing:
+    enabled: false
+    service_name: "indexer"
+  metrics:
+    enabled: false
+    address: "0.0.0.0"
+    port: 9000
+`;
+
+  fs.writeFileSync(configPath, defaultConfig.trim());
+}
+
 /**
  * Resolves the SQLite database path using the midnight-indexer configuration rules
  * @param {Object} env - Environment variables
@@ -17,17 +103,7 @@ function resolveSqlitePath(env, workingDir) {
     return envCnnUrl;
   }
 
-  // Determine config file path using the configuration resolution rules
-  let configPath;
-  if (env.CONFIG_FILE) {
-    configPath = path.isAbsolute(env.CONFIG_FILE)
-      ? env.CONFIG_FILE
-      : path.resolve(workingDir, env.CONFIG_FILE);
-  } else {
-    // Fall back to config.yaml in the current working directory
-    configPath = path.join(workingDir, "config.yaml");
-  }
-
+  const configPath = resolveConfigPath(env, workingDir);
   console.log(`Looking for config file at: ${configPath}`);
 
   // Check if config file exists
@@ -64,6 +140,24 @@ function resolveSqlitePath(env, workingDir) {
   }
 }
 
+/**
+ * Resolves the SQLite database path with a fallback to the default location
+ * @param {Object} env - Environment variables
+ * @param {string} workingDir - The working directory where the indexer runs
+ * @returns {string} The resolved SQLite database path
+ */
+function resolveSqlitePathWithFallback(env, workingDir) {
+  const sqlitePath = resolveSqlitePath(env, workingDir);
+  if (sqlitePath) {
+    return sqlitePath;
+  }
+
+  // Fallback to default indexer location: data/indexer.sqlite
+  const defaultPath = path.join(workingDir, "data", "indexer.sqlite");
+  console.log(`Using default SQLite path: ${defaultPath}`);
+  return defaultPath;
+}
+
 /**
  * Handles the --clean flag by deleting the SQLite database file
  * @param {Object} env - Environment variables
@@ -72,12 +166,7 @@ function resolveSqlitePath(env, workingDir) {
 function handleCleanFlag(env, workingDir) {
   console.log("Processing --clean flag...");
 
-  const sqlitePath = resolveSqlitePath(env, workingDir);
-
-  if (!sqlitePath) {
-    console.warn("Could not resolve SQLite database path. Skipping cleanup.");
-    return;
-  }
+  const sqlitePath = resolveSqlitePathWithFallback(env, workingDir);
 
   // Handle sqlite:// URLs and extract the file path
   let filePath = sqlitePath;
@@ -110,16 +199,16 @@ function handleCleanFlag(env, workingDir) {
  * @returns {ChildProcess} The spawned child process
  */
 function runMidnightIndexer(env = process.env, args = []) {
-  const platform = getPlatform();
-  const parts = platform.split("-");
-  const binaryName = `indexer-standalone-${platform}`;
   const binaryPath = path.join(
     __dirname,
     "indexer-standalone",
-    binaryName,
+    BINARY_NAME,
   );
   const workingDir = path.join(__dirname, "indexer-standalone");
 
+  const configPath = resolveConfigPath(env, workingDir);
+  ensureConfigExists(configPath, env);
+
   // Check for --clean flag and handle it
   const cleanFlagIndex = args.indexOf("--clean");
   if (cleanFlagIndex !== -1) {
@@ -130,6 +219,11 @@ function runMidnightIndexer(env = process.env, args = []) {
 
   console.log(`Starting midnight-indexer binary at: ${binaryPath}`);
 
+  const dataDir = path.join(workingDir, "data");
+  if (!fs.existsSync(dataDir)) {
+    fs.mkdirSync(dataDir, { recursive: true });
+  }
+
   const childProcess = spawn(binaryPath, args, {
     env: env,
     stdio: "inherit", // Inherit stdin, stdout, stderr from parent process