@effectstream/npm-midnight-indexer 0.104.0 → 0.200.1

package/README.md

@@ -2,9 +2,10 @@
 
 NPM wrapper that runs the [Midnight](https://midnight.network) Indexer either as a Docker container or as a native binary. Boots alongside `@effectstream/npm-midnight-node` and `@effectstream/npm-midnight-proof-server` to give Effectstream a local indexer to consume.
 
-- Pinned Midnight indexer v4.3.3.
+- Pinned Midnight indexer `4.4.0-rc.1`.
+- Native targets are `macos-arm64` and `linux-amd64`; no Linux arm64 asset is published.
 - Docker or binary mode, with platform-aware defaults (macOS arm64 and Linux can use either; Windows is Docker-only).
-- One env var to set: `APP__INFRA__SECRET`. Everything else has a default that works against the local Midnight stack.
+- One env var to set: `APP__INFRA__SECRET`, as a hex-encoded 32-byte value (64 hex characters). Everything else has a default that works against the local Midnight stack.
 - Used by the orchestrator's Midnight step; sits in front of `MidnightFetcher` on the sync side.
 - Maps port 8088 (Docker) or runs on localhost (binary) so the rest of the local stack reaches it the same way.
 
@@ -24,13 +25,13 @@ Pick a mode and pass `APP__INFRA__SECRET`. Without a flag, the wrapper prompts i
 
 ```bash
 # Docker (recommended where available)
-APP__INFRA__SECRET=<secret> bunx npm-midnight-indexer --docker
+APP__INFRA__SECRET=<64-hex-characters> bunx npm-midnight-indexer --docker
 
 # Native binary (Linux, macOS arm64)
-APP__INFRA__SECRET=<secret> bunx npm-midnight-indexer --binary
+APP__INFRA__SECRET=<64-hex-characters> bunx npm-midnight-indexer --binary
 
 # Interactive: prompts for Docker vs binary
-APP__INFRA__SECRET=<secret> bunx npm-midnight-indexer
+APP__INFRA__SECRET=<64-hex-characters> bunx npm-midnight-indexer
 
 # Help
 bunx npm-midnight-indexer --help
@@ -42,7 +43,7 @@ The Docker path pulls `midnightntwrk/indexer-standalone` and maps container port
 
 | Variable | Required | Docker default | Binary default | Purpose |
 | --- | --- | --- | --- | --- |
-| `APP__INFRA__SECRET` | yes | - | - | Indexer secret. |
+| `APP__INFRA__SECRET` | yes | - | - | Hex-encoded 32-byte indexer secret. |
 | `LEDGER_NETWORK_ID` | no | `Undeployed` | `Undeployed` | Ledger network selector. |
 | `SUBSTRATE_NODE_WS_URL` | no | `ws://node:9944` | `ws://localhost:9944` | Substrate node WS. |
 | `FEATURES_WALLET_ENABLED` | no | `true` | `true` | Wallet features. |
@@ -55,7 +56,7 @@ The Docker path pulls `midnightntwrk/indexer-standalone` and maps container port
 
 ### Supported binary platforms
 
-Linux arm64, Linux amd64, macOS arm64.
+Linux amd64 and macOS arm64.
 
 ## Inside Effectstream
 

package/binary.js

@@ -4,7 +4,7 @@ const axios = require("axios");
 const extract = require("extract-zip");
 const path = require("path");
 
-const CURRENT_BINARY_VERSION = "v4.3.3";
+const CURRENT_BINARY_VERSION = "v4.4.0-rc.1";
 const FINAL_BINARY_NAME = "indexer-standalone";
 
 /*

package/docker.js

@@ -1,6 +1,7 @@
 const { spawn, exec } = require('child_process');
 const { promisify } = require('util');
 const execAsync = promisify(exec);
+const IMAGE = 'midnightntwrk/indexer-standalone:4.4.0-rc.1';
 
 /**
  * Checks if Docker is installed and available on the system
@@ -20,12 +21,10 @@ async function checkIfDockerExists() {
  * @returns {Promise<void>}
  */
 async function pullDockerImage() {
-    const imageName = 'midnightntwrk/indexer-standalone';
-    
-    console.log(`Pulling Docker image: ${imageName}`);
+    console.log(`Pulling Docker image: ${IMAGE}`);
     
     return new Promise((resolve, reject) => {
-        const childProcess = spawn('docker', ['pull', imageName], {
+        const childProcess = spawn('docker', ['pull', IMAGE], {
             stdio: 'inherit'
         });
         
@@ -51,7 +50,6 @@ async function pullDockerImage() {
  * @returns {ChildProcess} The spawned Docker process
  */
 function runDockerContainer(env = process.env, args = []) {
-    const imageName = 'midnightntwrk/indexer-standalone';
     const containerName = 'midnight-local-indexer';
     
     // Stop and remove existing container if it exists
@@ -124,7 +122,7 @@ function runDockerContainer(env = process.env, args = []) {
     });
     
     // Add the image name
-    dockerArgs.push(imageName);
+    dockerArgs.push(IMAGE);
     
     // Add any remaining arguments
     if (args.length > 0) {

package/index.js

@@ -1,5 +1,5 @@
 const { binary, getPlatform, cleanBinaries } = require("./binary");
-const { runMidnightIndexer, waitForNodeBlock } = require(
+const { runMidnightIndexer, waitForNodeBlock, isValidIndexerSecret } = require(
   "./run_midnight_indexer",
 );
 const { checkIfDockerExists, pullDockerImage, runDockerContainer } = require(
@@ -66,7 +66,7 @@ Options:
   --help           Show this help message
 
 Environment Variables:
-  APP__INFRA__SECRET             Required: Secret key for the application
+  APP__INFRA__SECRET             Required: 64-character hexadecimal secret
   LEDGER_NETWORK_ID              Optional: Ledger network ID (default: Undeployed)
   SUBSTRATE_NODE_WS_URL          Optional: Substrate node WebSocket URL
                                  (Docker default: ws://node:9944, Binary default: ws://localhost:9944)
@@ -79,11 +79,11 @@ Environment Variables:
 Note: macOS Intel users will automatically use Docker. macOS arm64 supports both binary and Docker execution.
 
 Examples:
-  APP__INFRA__SECRET=mysecret node index.js --docker          # Use Docker
-  APP__INFRA__SECRET=mysecret node index.js --binary          # Use binary (if supported)
-  APP__INFRA__SECRET=mysecret node index.js --clean-binaries   # Delete and redownload binaries
+  APP__INFRA__SECRET=<64_hex_characters> node index.js --docker          # Use Docker
+  APP__INFRA__SECRET=<64_hex_characters> node index.js --binary          # Use binary (if supported)
+  APP__INFRA__SECRET=<64_hex_characters> node index.js --clean-binaries   # Delete and redownload binaries
   node index.js --only-clean                                 # Only delete downloaded binaries
-  APP__INFRA__SECRET=mysecret node index.js --clean           # Clean SQLite database
+  APP__INFRA__SECRET=<64_hex_characters> node index.js --clean           # Clean SQLite database
   node index.js --help                                       # Show this help
 `);
 }
@@ -170,9 +170,9 @@ async function runWithBinary(env, args, forceClean = false) {
   console.log("Using binary to run midnight indexer...");
 
   // Check for required environment variable
-  if (!env.APP__INFRA__SECRET) {
-    console.error("Error: APP__INFRA__SECRET environment variable is required");
-    console.log("Please set APP__INFRA__SECRET=<your_secret> and run again");
+  if (!isValidIndexerSecret(env.APP__INFRA__SECRET)) {
+    console.error("Error: APP__INFRA__SECRET must be exactly 64 hexadecimal characters");
+    console.log("Please set APP__INFRA__SECRET=<64_hex_characters> and run again");
     process.exit(1);
   }
 
@@ -243,12 +243,12 @@ async function main(args) {
     }
 
     // Check for required environment variable
-    if (!env.APP__INFRA__SECRET) {
+    if (!isValidIndexerSecret(env.APP__INFRA__SECRET)) {
       console.error(
-        "Error: APP__INFRA__SECRET environment variable is required",
+        "Error: APP__INFRA__SECRET must be exactly 64 hexadecimal characters",
       );
       console.log(
-        "Please set APP__INFRA__SECRET=<your_secret> when running with --docker",
+        "Please set APP__INFRA__SECRET=<64_hex_characters> when running with --docker",
       );
       process.exit(1);
     }
@@ -296,11 +296,11 @@ async function main(args) {
     );
 
     // Check for required environment variable
-    if (!env.APP__INFRA__SECRET) {
+    if (!isValidIndexerSecret(env.APP__INFRA__SECRET)) {
       console.error(
-        "Error: APP__INFRA__SECRET environment variable is required",
+        "Error: APP__INFRA__SECRET must be exactly 64 hexadecimal characters",
       );
-      console.log("Please set APP__INFRA__SECRET=<your_secret> and run again");
+      console.log("Please set APP__INFRA__SECRET=<64_hex_characters> and run again");
       process.exit(1);
     }
 
@@ -312,12 +312,12 @@ async function main(args) {
     const useDocker = await promptUserForDockerChoice();
     if (useDocker) {
       // Check for required environment variable
-      if (!env.APP__INFRA__SECRET) {
+      if (!isValidIndexerSecret(env.APP__INFRA__SECRET)) {
         console.error(
-          "Error: APP__INFRA__SECRET environment variable is required",
+          "Error: APP__INFRA__SECRET must be exactly 64 hexadecimal characters",
         );
         console.log(
-          "Please set APP__INFRA__SECRET=<your_secret> and run again",
+          "Please set APP__INFRA__SECRET=<64_hex_characters> and run again",
         );
         process.exit(1);
       }
@@ -326,7 +326,7 @@ async function main(args) {
   }
 
   // Default to binary execution (only on supported platforms)
-  return runWithBinary(env, flags.remainingArgs, flags.clean);
+  return runWithBinary(env, flags.remainingArgs, flags.cleanBinaries);
 }
 
 // Handle unhandled promise rejections

package/indexer-standalone/config.yaml

@@ -5,6 +5,8 @@ application:
   blocks_buffer: 10
   caught_up_max_distance: 10
   caught_up_leeway: 5
+  gc_bound: "200ms"
+  ledger_state_retention: 1000
   active_wallets_query_delay: "500ms"
   active_wallets_ttl: "30m"
   transaction_batch_size: 50
@@ -54,6 +56,8 @@ infra:
         batch_size: 20
       contract_actions:
         batch_size: 20
+      contract_events:
+        batch_size: 20
       dust_generations:
         batch_size: 20
       dust_ledger_events:
@@ -71,6 +75,9 @@ infra:
         progress_update_interval: "30s"
       zswap_ledger_events:
         batch_size: 20
+      progress_cache:
+        max_capacity: 10000
+        time_to_live: "5s"
     quota:
       max_concurrent_per_connection: 20
       max_session_subscriptions_per_minute: 10

package/package.json

@@ -1,10 +1,10 @@
 {
   "name": "@effectstream/npm-midnight-indexer",
-  "version": "0.104.0",
+  "version": "0.200.1",
   "description": "Downloads and runs the Midnight Indexer. It needs a running Midnight Node",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "bun test run_midnight_indexer.test.js",
     "clean": "node index.js --clean-binaries",
     "clean-only": "node index.js --only-clean"
   },
@@ -23,7 +23,6 @@
     "url": "https://github.com/effectstream/effectstream/issues"
   },
   "supportedPlatforms": [
-    "linux-arm64",
     "linux-amd64",
     "macos-arm64"
   ],

package/run_midnight_indexer.js

@@ -6,10 +6,14 @@ const axios = require("axios");
 
 const BINARY_NAME = "indexer-standalone";
 
+function isValidIndexerSecret(secret) {
+  return typeof secret === "string" && /^[0-9a-fA-F]{64}$/.test(secret);
+}
+
 /**
  * Waits until the Midnight node has produced at least `minBlock`.
  *
- * The v4.3.3 indexer bundles an spo-indexer that, on a fresh DB, reads block #1
+ * The v4.4.0-rc.1 indexer bundles an spo-indexer that, on a fresh DB, reads block #1
  * to anchor the first epoch and exits(1) — killing the whole indexer — if that
  * block does not exist yet. Gating startup on block #1 avoids that startup race.
  * 
@@ -90,47 +94,81 @@ function ensureConfigExists(configPath, env) {
   const apiPort = env.APP__INFRA__API__PORT || 8088;
 
   const defaultConfig = `
-run_migrations: true
-network_id: &network_id "${networkId}"
-
-chain_indexer_application:
-  network_id: *network_id
-  blocks_buffer: 60
-  save_zswap_state_after: 1000
-  caught_up_max_distance: 60
-  caught_up_leeway: 30
-
-wallet_indexer_application:
-  network_id: *network_id
-  active_wallets_repeat_delay: "100ms"
+thread_stack_size: "24MiB"
+
+application:
+  network_id: "${networkId.toLowerCase()}"
+  blocks_buffer: 10
+  caught_up_max_distance: 10
+  caught_up_leeway: 5
+  gc_bound: "200ms"
+  ledger_state_retention: 1000
+  active_wallets_query_delay: "500ms"
   active_wallets_ttl: "30m"
-  transaction_batch_size: 10
+  transaction_batch_size: 50
+
+spo:
+  interval: 5000
+  stake_refresh:
+    period_secs: 900
+    page_size: 100
+    max_rps: 2
 
 infra:
+  run_migrations: true
+  storage:
+    cnn_url: "${cnnUrl}"
+  ledger_db:
+    cache_size: "1kiB"
+    cnn_url: "${env.APP__INFRA__LEDGER_DB__CNN_URL || "./data/ledger-db.sqlite"}"
   node:
     url: "${nodeUrl}"
     reconnect_max_delay: "10s"
     reconnect_max_attempts: 30
-
-  storage:
-    cnn_url: "${cnnUrl}"
-
+    subscription_recovery_timeout: "30s"
+  spo_node:
+    url: "${env.APP__INFRA__SPO_NODE__URL || nodeUrl}"
+    reconnect_max_delay: "10s"
+    reconnect_max_attempts: 30
+    blockfrost_id: "${env.APP__INFRA__SPO_NODE__BLOCKFROST_ID || "dummy-not-using-spo"}"
   api:
-    address: "0.0.0.0"
+    address: "${env.APP__INFRA__API__ADDRESS || "0.0.0.0"}"
     port: ${apiPort}
     request_body_limit: "1MiB"
     max_complexity: 200
     max_depth: 15
-    network_id: *network_id
+    subscription:
+      blocks: { batch_size: 20 }
+      contract_actions: { batch_size: 20 }
+      contract_events: { batch_size: 20 }
+      dust_generations: { batch_size: 20 }
+      dust_ledger_events: { batch_size: 20 }
+      dust_nullifier_transactions: { batch_size: 20 }
+      shielded_nullifier_transactions: { batch_size: 20 }
+      shielded_transactions:
+        batch_size: 20
+        progress_update_interval: "30s"
+        keep_wallet_alive_interval: "1m"
+      unshielded_transactions:
+        batch_size: 20
+        progress_update_interval: "30s"
+      zswap_ledger_events: { batch_size: 20 }
+      progress_cache:
+        max_capacity: 10000
+        time_to_live: "5s"
+    quota:
+      max_concurrent_per_connection: 20
+      max_session_subscriptions_per_minute: 10
 
 telemetry:
   tracing:
     enabled: false
     service_name: "indexer"
+    otlp_exporter_endpoint: "http://localhost:4317"
   metrics:
     enabled: false
     address: "0.0.0.0"
-    port: 9000
+    port: ${env.APP__TELEMETRY__METRICS__PORT || 9000}
 `;
 
   fs.writeFileSync(configPath, defaultConfig.trim());
@@ -298,4 +336,9 @@ function runMidnightIndexer(env = process.env, args = []) {
   return childProcess;
 }
 
-module.exports = { runMidnightIndexer, waitForNodeBlock };
+module.exports = {
+  ensureConfigExists,
+  isValidIndexerSecret,
+  runMidnightIndexer,
+  waitForNodeBlock,
+};

package/run_midnight_indexer.test.js

@@ -0,0 +1,59 @@
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+
+const { describe, expect, test } = require("bun:test");
+const yaml = require("js-yaml");
+
+const {
+  ensureConfigExists,
+  isValidIndexerSecret,
+} = require("./run_midnight_indexer");
+
+function expectRc1Schema(config) {
+  expect(config.application.gc_bound).toBe("200ms");
+  expect(config.application.ledger_state_retention).toBe(1000);
+  expect(config.infra.api.subscription.contract_events.batch_size).toBe(20);
+  expect(config.infra.api.subscription.progress_cache.max_capacity).toBe(10000);
+  expect(config.infra.api.subscription.progress_cache.time_to_live).toBe("5s");
+}
+
+describe("indexer 4.4.0-rc.1 configuration", () => {
+  test("accepts exactly a hex-encoded 32-byte application secret", () => {
+    expect(isValidIndexerSecret("ab".repeat(32))).toBe(true);
+    expect(isValidIndexerSecret("mysecret")).toBe(false);
+    expect(isValidIndexerSecret("ab".repeat(31))).toBe(false);
+    expect(isValidIndexerSecret("zz".repeat(32))).toBe(false);
+  });
+
+  test("the packaged config contains all required rc.1 subscription fields", () => {
+    const config = yaml.load(
+      fs.readFileSync(
+        path.join(__dirname, "indexer-standalone", "config.yaml"),
+        "utf8",
+      ),
+    );
+    expectRc1Schema(config);
+  });
+
+  test("a generated config uses the rc.1 schema and caller endpoints", () => {
+    const dir = fs.mkdtempSync(path.join(os.tmpdir(), "midnight-indexer-config-"));
+    const configPath = path.join(dir, "config.yaml");
+    try {
+      ensureConfigExists(configPath, {
+        LEDGER_NETWORK_ID: "Undeployed",
+        SUBSTRATE_NODE_WS_URL: "ws://127.0.0.1:15501",
+        APP__INFRA__STORAGE__CNN_URL: "/tmp/indexer.sqlite",
+        APP__INFRA__LEDGER_DB__CNN_URL: "/tmp/ledger.sqlite",
+        APP__INFRA__API__PORT: "15502",
+      });
+      const config = yaml.load(fs.readFileSync(configPath, "utf8"));
+      expectRc1Schema(config);
+      expect(config.application.network_id).toBe("undeployed");
+      expect(config.infra.node.url).toBe("ws://127.0.0.1:15501");
+      expect(config.infra.api.port).toBe(15502);
+    } finally {
+      fs.rmSync(dir, { force: true, recursive: true });
+    }
+  });
+});