@blinklabs/dingo 0.17.0 → 0.18.0

package/PLUGIN_DEVELOPMENT.md

@@ -0,0 +1,299 @@
+# Plugin Development Guide
+
+This guide explains how to develop plugins for Dingo's storage system.
+
+## Overview
+
+Dingo supports pluggable storage backends through a registration-based plugin system. Plugins can extend the system with new blob storage (blocks, transactions) and metadata storage (indexes, state) implementations.
+
+## Plugin Types
+
+### Blob Storage Plugins
+Store blockchain data (blocks, transactions, etc.). Examples:
+- `badger` - Local BadgerDB key-value store
+- `gcs` - Google Cloud Storage
+- `s3` - AWS S3
+
+### Metadata Storage Plugins
+Store metadata and indexes. Examples:
+- `sqlite` - SQLite relational database
+
+## Plugin Interface
+
+All plugins must implement the `plugin.Plugin` interface:
+
+```go
+type Plugin interface {
+    Start() error
+    Stop() error
+}
+```
+
+## Plugin Registration
+
+Plugins register themselves during package initialization using the `plugin.Register()` function:
+
+```go
+func init() {
+    plugin.Register(plugin.PluginEntry{
+        Type:               plugin.PluginTypeBlob, // or PluginTypeMetadata
+        Name:               "myplugin",
+        Description:        "My custom storage plugin",
+        NewFromOptionsFunc: NewFromCmdlineOptions,
+        Options: []plugin.PluginOption{
+            // Plugin-specific options
+        },
+    })
+}
+```
+
+## Plugin Options
+
+Plugins define configuration options using the `PluginOption` struct:
+
+```go
+plugin.PluginOption{
+    Name:         "data-dir",           // Option name
+    Type:         plugin.PluginOptionTypeString, // Data type
+    Description:  "Data directory path", // Help text
+    DefaultValue: "/tmp/data",         // Default value
+    Dest:         &cmdlineOptions.dataDir, // Destination variable
+}
+```
+
+Supported option types:
+- `PluginOptionTypeString`
+- `PluginOptionTypeBool`
+- `PluginOptionTypeInt`
+- `PluginOptionTypeUint`
+
+## Environment Variables
+
+Plugins automatically support environment variables with the pattern:
+`DINGO_DATABASE_{TYPE}_{PLUGIN}_{OPTION}`
+
+Examples:
+- `DINGO_DATABASE_BLOB_BADGER_DATA_DIR=/data`
+- `DINGO_DATABASE_METADATA_SQLITE_DATA_DIR=/metadata.db`
+
+## YAML Configuration
+
+Plugins can be configured in `dingo.yaml`:
+
+```yaml
+database:
+  blob:
+    plugin: "myplugin"
+    myplugin:
+      option1: "value1"
+      option2: 42
+  metadata:
+    plugin: "sqlite"
+    sqlite:
+      data-dir: "/data/metadata.db"
+```
+
+## Configuration Precedence
+
+1. Command-line flags (highest priority)
+2. Environment variables
+3. YAML configuration
+4. Default values (lowest priority)
+
+## Command Line Options
+
+Plugins support command-line flags with the pattern:
+`--{type}-{plugin}-{option}`
+
+Examples:
+- `--blob-badger-data-dir /data`
+- `--metadata-sqlite-data-dir /metadata.db`
+
+## Plugin Development Steps
+
+### 1. Create Plugin Structure
+
+```text
+database/plugin/{type}/{name}/
+├── plugin.go      # Registration and options
+├── options.go     # Option functions
+├── database.go    # Core implementation
+└── options_test.go # Unit tests
+```
+
+### 2. Implement Core Plugin
+
+Create the main plugin struct that implements `plugin.Plugin`:
+
+```go
+type MyPlugin struct {
+    // Fields
+}
+
+func (p *MyPlugin) Start() error {
+    // Initialize resources
+    return nil
+}
+
+func (p *MyPlugin) Stop() error {
+    // Clean up resources
+    return nil
+}
+```
+
+### 3. Define Options
+
+Create option functions following the pattern:
+
+```go
+func WithOptionName(value Type) OptionFunc {
+    return func(p *MyPlugin) {
+        p.field = value
+    }
+}
+```
+
+### 4. Implement Constructors
+
+Provide both options-based and legacy constructors:
+
+```go
+func NewWithOptions(opts ...OptionFunc) (*MyPlugin, error) {
+    p := &MyPlugin{}
+    for _, opt := range opts {
+        opt(p)
+    }
+    return p, nil
+}
+
+func New(legacyParam1, legacyParam2) (*MyPlugin, error) {
+    // For backward compatibility
+    return NewWithOptions(
+        WithOption1(legacyParam1),
+        WithOption2(legacyParam2),
+    )
+}
+```
+
+### 5. Register Plugin
+
+In `plugin.go`, register during initialization:
+
+```go
+var cmdlineOptions struct {
+    option1 string
+    option2 int
+}
+
+func init() {
+    plugin.Register(plugin.PluginEntry{
+        Type: plugin.PluginTypeBlob,
+        Name: "myplugin",
+        Description: "My custom plugin",
+        NewFromOptionsFunc: NewFromCmdlineOptions,
+        Options: []plugin.PluginOption{
+            {
+                Name: "option1",
+                Type: plugin.PluginOptionTypeString,
+                Description: "First option",
+                DefaultValue: "default",
+                Dest: &cmdlineOptions.option1,
+            },
+            // More options...
+        },
+    })
+}
+
+func NewFromCmdlineOptions() plugin.Plugin {
+    p, err := NewWithOptions(
+        WithOption1(cmdlineOptions.option1),
+        WithOption2(cmdlineOptions.option2),
+    )
+    if err != nil {
+        panic(err)
+    }
+    return p
+}
+```
+
+### 6. Add Tests
+
+Create comprehensive tests:
+
+```go
+func TestOptions(t *testing.T) {
+    // Test option functions
+}
+
+func TestLifecycle(t *testing.T) {
+    p, err := NewWithOptions(WithOption1("test"))
+    // Test Start/Stop
+}
+```
+
+### 7. Update Imports
+
+Add your plugin to the import list in the appropriate store file:
+- `database/plugin/blob/blob.go` for blob plugins
+- `database/plugin/metadata/metadata.go` for metadata plugins
+
+## Example: Complete Plugin
+
+See the existing plugins for complete examples:
+- `database/plugin/blob/badger/` - BadgerDB implementation
+- `database/plugin/metadata/sqlite/` - SQLite implementation
+- `database/plugin/blob/gcs/` - Google Cloud Storage implementation
+- `database/plugin/blob/aws/` - AWS S3 implementation
+
+## Best Practices
+
+1. **Error Handling**: Always return descriptive errors
+2. **Resource Management**: Properly implement Start/Stop for resource lifecycle
+3. **Thread Safety**: Ensure plugins are safe for concurrent use
+4. **Configuration Validation**: Validate configuration during construction
+5. **Backward Compatibility**: Maintain compatibility with existing deployments
+6. **Documentation**: Document all options and their effects
+7. **Testing**: Provide comprehensive unit and integration tests
+
+## Testing Your Plugin
+
+### Unit Tests
+Test individual components and option functions.
+
+### Integration Tests
+Test the complete plugin lifecycle and interaction with the plugin system.
+
+### CLI Testing
+Use the CLI to test plugin listing and selection:
+
+```bash
+./dingo --blob list
+./dingo --metadata list
+```
+
+### Configuration Testing
+Test environment variables and YAML configuration:
+
+```bash
+DINGO_DATABASE_BLOB_MYPLUGIN_OPTION1=value ./dingo --blob myplugin
+```
+
+## Programmatic Option Overrides (for tests)
+
+When writing tests or programmatically constructing database instances you can override plugin options
+without importing plugin implementation packages directly by using the plugin registry helper:
+
+```go
+// Set data-dir for the blob plugin to a per-test temp directory
+plugin.SetPluginOption(plugin.PluginTypeBlob, "badger", "data-dir", t.TempDir())
+
+// Set data-dir for the metadata plugin
+plugin.SetPluginOption(plugin.PluginTypeMetadata, "sqlite", "data-dir", t.TempDir())
+```
+
+The helper sets the plugin option's destination variable in the registry before plugin instantiation.
+If the requested option is not defined by the targeted plugin the call is non-fatal and returns nil,
+allowing tests to run regardless of which plugin implementation is selected.
+
+Using `t.TempDir()` guarantees each test uses its own on-disk path and prevents concurrent tests from
+colliding on shared directories (for example the default `.dingo` Badger directory).

package/README.md

@@ -13,6 +13,8 @@
 
 ⚠️ This is a work in progress and is currently under heavy development
 
+**Note:** On Windows systems, named pipes are used instead of Unix sockets for node-to-client communication.
+
 <div align="center">
   <img src="./.github/dingo-20241210.png" alt="dingo screenshot" width="640">
 </div>
@@ -62,6 +64,78 @@ This behavior can be changed via the following environment variables:
     (default: empty)
 - `TLS_KEY_FILE_PATH` - SSL certificate key to use (default: empty)
 
+## Database Plugins
+
+Dingo supports pluggable storage backends for both blob storage (blocks, transactions) and metadata storage. This allows you to choose the best storage solution for your use case.
+
+### Available Plugins
+
+**Blob Storage Plugins:**
+- `badger` - BadgerDB local key-value store (default)
+- `gcs` - Google Cloud Storage blob store
+- `s3` - AWS S3 blob store
+
+**Metadata Storage Plugins:**
+- `sqlite` - SQLite relational database (default)
+
+### Plugin Selection
+
+Plugins can be selected via command-line flags, environment variables, or configuration file:
+
+```bash
+# Command line
+./dingo --blob gcs --metadata sqlite
+
+# Environment variables
+DINGO_DATABASE_BLOB_PLUGIN=gcs
+DINGO_DATABASE_METADATA_PLUGIN=sqlite
+
+# Configuration file (dingo.yaml)
+database:
+  blob:
+    plugin: "gcs"
+  metadata:
+    plugin: "sqlite"
+```
+
+### Plugin Configuration
+
+Each plugin supports specific configuration options. See `dingo.yaml.example` for detailed configuration examples.
+
+**BadgerDB Options:**
+- `data-dir` - Directory for database files
+- `block-cache-size` - Block cache size in bytes
+- `index-cache-size` - Index cache size in bytes
+- `gc` - Enable garbage collection
+
+**Google Cloud Storage Options:**
+- `bucket` - GCS bucket name
+- `project-id` - Google Cloud project ID
+- `prefix` - Path prefix within bucket
+- `credentials-file` - Path to service account credentials file (optional - uses Application Default Credentials if not provided)
+
+**AWS S3 Options:**
+- `bucket` - S3 bucket name
+- `region` - AWS region
+- `prefix` - Path prefix within bucket
+- `access-key-id` - AWS access key ID (optional - uses default credential chain if not provided)
+- `secret-access-key` - AWS secret access key (optional - uses default credential chain if not provided)
+
+**SQLite Options:**
+- `data-dir` - Path to SQLite database file
+
+### Listing Available Plugins
+
+You can see all available plugins and their descriptions:
+
+```bash
+./dingo list
+```
+
+## Plugin Development
+
+For information on developing custom storage plugins, see [PLUGIN_DEVELOPMENT.md](PLUGIN_DEVELOPMENT.md).
+
 ### Example
 
 Running on mainnet (:sweat_smile:):

package/benchmark_results.md

@@ -0,0 +1,130 @@
+# Dingo Ledger & Database Benchmark Results
+
+## Latest Results
+
+### Test Environment
+- **Date**: November 26, 2025
+- **Go Version**: 1.24.1
+- **OS**: Linux
+- **Architecture**: aarch64
+- **CPU Cores**: 128
+- **Data Source**: Real Cardano preview testnet data (40k+ blocks, slots 0-863,996)
+
+### Benchmark Results
+
+All benchmarks run with `-benchmem` flag showing memory allocations and operation counts.
+
+| Benchmark | Operations/sec | Time/op | Memory/op | Allocs/op |
+|-----------|----------------|---------|-----------|-----------|
+| Pool Lookup By Key Hash No Data | 36231 | 33604ns | 4KB | 79 |
+| Pool Registration Lookups No Data | 24210 | 46595ns | 10KB | 93 |
+| Account Lookup By Stake Key Real Data | 39950 | 34026ns | 4KB | 75 |
+| Utxo Lookup By Address No Data | 109825 | 16460ns | 2KB | 19 |
+| Storage Backends/memory | 32593 | 34818ns | 13KB | 70 |
+| Transaction Validation | 230958193 | 5.238ns | 0B | 0 |
+| Real Block Reading | 22 | 53427289ns | 2183KB | 74472 |
+| Block Retrieval By Index Real Data | 303981 | 3868ns | 472B | 11 |
+| Index Building Time | 14078 | 87080ns | 17KB | 119 |
+| Chain Sync From Genesis | 74 | 15096067ns | 100.0blocks_processed | 2247880 |
+| Block Retrieval By Index No Data | 315110 | 3652ns | 472B | 11 |
+| Transaction Create | 77013 | 16824ns | 2KB | 18 |
+| Utxo Lookup By Address Real Data | 82125 | 16014ns | 2KB | 19 |
+| Era Transition Performance | 459890545 | 2.178ns | 0B | 0 |
+| Protocol Parameters Lookup By Epoch Real Data | 44181 | 32903ns | 5KB | 62 |
+| Pool Registration Lookups Real Data | 22616 | 48729ns | 10KB | 93 |
+| Stake Registration Lookups Real Data | 38761 | 33724ns | 5KB | 69 |
+| Era Transition Performance Real Data | 4340 | 267246ns | 83KB | 490 |
+| Real Block Processing | 13651 | 84993ns | 17KB | 119 |
+| Utxo Lookup By Ref No Data | 10461 | 119506ns | 8KB | 131 |
+| Storage Backends/disk | 33004 | 32561ns | 13KB | 70 |
+| Test Load/memory | 1912 | 650898ns | 260KB | 1400 |
+| Stake Registration Lookups No Data | 33963 | 33887ns | 5KB | 69 |
+| Utxo Lookup By Ref Real Data | 10000 | 122404ns | 8KB | 131 |
+| Protocol Parameters Lookup By Epoch No Data | 37996 | 32298ns | 5KB | 62 |
+| Datum Lookup By Hash No Data | 32635 | 34072ns | 4KB | 69 |
+| Block Processing Throughput | 3904 | 290370ns | 3444blocks/sec | 22462 |
+| Real Data Queries | 68570 | 17752ns | 5KB | 43 |
+| Block Nonce Lookup Real Data | 34851 | 37815ns | 4KB | 73 |
+| Test Load/disk | 2068 | 536686ns | 260KB | 1400 |
+| Pool Lookup By Key Hash Real Data | 40214 | 33631ns | 4KB | 79 |
+| D Rep Lookup By Key Hash No Data | 35266 | 36094ns | 4KB | 77 |
+| Transaction History Queries No Data | 34167 | 33889ns | 4KB | 78 |
+| Datum Lookup By Hash Real Data | 44810 | 33686ns | 4KB | 69 |
+| Block Nonce Lookup No Data | 32011 | 36515ns | 4KB | 73 |
+| Account Lookup By Stake Key No Data | 34898 | 35142ns | 4KB | 75 |
+| Transaction History Queries Real Data | 37575 | 34812ns | 4KB | 78 |
+| Concurrent Queries | 15408 | 68781ns | 14581queries/sec | 3943 |
+| D Rep Lookup By Key Hash Real Data | 39360 | 34907ns | 4KB | 77 |
+| Block Memory Usage | 29292 | 44369ns | 14KB | 49 |
+
+## Performance Changes
+
+Changes since **November 26, 2025**:
+
+### Summary
+- **Faster benchmarks**: 19
+- **Slower benchmarks**: 20
+- **New benchmarks**: 0
+- **Removed benchmarks**: 0
+
+### Top Improvements
+- Utxo Lookup By Address No Data (+44%)
+- Transaction Validation (+0%)
+- Test Load/memory (+19%)
+- Test Load/disk (+31%)
+- Storage Backends/memory (+3%)
+
+### Performance Regressions
+- Pool Lookup By Key Hash No Data (-0%)
+- Index Building Time (-0%)
+- Account Lookup By Stake Key No Data (-0%)
+- Transaction History Queries No Data (-1%)
+- Stake Registration Lookups No Data (-1%)
+
+
+## Historical Results
+
+### November 26, 2025
+
+| Benchmark | Operations/sec | Time/op | Memory/op | Allocs/op |
+|-----------|----------------|---------|-----------|-----------|
+| Pool Lookup By Key Hash No Data | 36328 | 33556ns | 4KB | 79 |
+| Pool Registration Lookups No Data | 24898 | 51298ns | 10KB | 93 |
+| Account Lookup By Stake Key Real Data | 39936 | 33871ns | 4KB | 75 |
+| Utxo Lookup By Address No Data | 75966 | 16953ns | 2KB | 19 |
+| Storage Backends/memory | 31369 | 33394ns | 13KB | 70 |
+| Transaction Validation | 230637091 | 5.213ns | 0B | 0 |
+| Real Block Reading | 20 | 55046083ns | 2183KB | 74472 |
+| Block Retrieval By Index Real Data | 312492 | 4102ns | 472B | 11 |
+| Index Building Time | 14168 | 86378ns | 17KB | 119 |
+| Chain Sync From Genesis | 100 | 14189212ns | 100.0blocks_processed | 2247966 |
+| Block Retrieval By Index No Data | 277410 | 4025ns | 472B | 11 |
+| Transaction Create | 92761 | 16456ns | 2KB | 18 |
+| Utxo Lookup By Address Real Data | 86467 | 15705ns | 2KB | 19 |
+| Era Transition Performance | 499031763 | 2.139ns | 0B | 0 |
+| Protocol Parameters Lookup By Epoch Real Data | 44150 | 30066ns | 5KB | 62 |
+| Pool Registration Lookups Real Data | 23724 | 48201ns | 10KB | 93 |
+| Stake Registration Lookups Real Data | 40082 | 32844ns | 5KB | 69 |
+| Era Transition Performance Real Data | 3525 | 305364ns | 83KB | 490 |
+| Real Block Processing | 13509 | 87846ns | 17KB | 119 |
+| Utxo Lookup By Ref No Data | 10724 | 111792ns | 8KB | 131 |
+| Storage Backends/disk | 28960 | 37643ns | 13KB | 70 |
+| Test Load/memory | 1605 | 669272ns | 260KB | 1400 |
+| Stake Registration Lookups No Data | 34359 | 36417ns | 5KB | 69 |
+| Utxo Lookup By Ref Real Data | 10000 | 111756ns | 8KB | 131 |
+| Protocol Parameters Lookup By Epoch No Data | 37886 | 32905ns | 5KB | 62 |
+| Datum Lookup By Hash No Data | 40302 | 29076ns | 4KB | 69 |
+| Block Processing Throughput | 4632 | 287053ns | 3484blocks/sec | 22466 |
+| Real Data Queries | 62532 | 19011ns | 5KB | 43 |
+| Block Nonce Lookup Real Data | 35782 | 38514ns | 4KB | 73 |
+| Test Load/disk | 1574 | 711525ns | 260KB | 1400 |
+| Pool Lookup By Key Hash Real Data | 39812 | 33599ns | 4KB | 79 |
+| D Rep Lookup By Key Hash No Data | 35139 | 35088ns | 4KB | 77 |
+| Transaction History Queries No Data | 34530 | 36887ns | 4KB | 78 |
+| Datum Lookup By Hash Real Data | 36972 | 31369ns | 4KB | 69 |
+| Block Nonce Lookup No Data | 32977 | 36460ns | 4KB | 73 |
+| Account Lookup By Stake Key No Data | 35020 | 32890ns | 4KB | 75 |
+| Transaction History Queries Real Data | 38739 | 35904ns | 4KB | 78 |
+| D Rep Lookup By Key Hash Real Data | 38977 | 36859ns | 4KB | 77 |
+| Concurrent Queries | 19104 | 59527ns | 168280queries/sec | 3851 |
+| Block Memory Usage | 26552 | 46212ns | 14KB | 49 |

package/devnet.sh

@@ -1,5 +1,19 @@
 #!/usr/bin/env bash
 
+# Copyright 2025 Blink Labs Software
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 export CARDANO_NETWORK=devnet
 export CARDANO_CONFIG=./config/cardano/devnet/config.json
 export CARDANO_DATABASE_PATH=.devnet

package/dingo.yaml.example

@@ -9,8 +9,51 @@ bindAddr: "0.0.0.0"
 # Can be overridden with the config environment variable
 cardanoConfig: "./config/cardano/preview/config.json"
 
-# A directory which contains the ledger database files
-databasePath: ".dingo"
+# Database configuration
+database:
+  # Blob storage plugin configuration
+  blob:
+    # Plugin to use for blob storage (badger, gcs, s3)
+    plugin: "badger"
+    # Configuration options for each plugin
+    badger:
+      # Data directory for BadgerDB storage
+      data-dir: ".dingo/badger"
+      # Block cache size in bytes (default: 1610612736 ~1.5GB)
+      block-cache-size: 1610612736
+      # Index cache size in bytes (default: 536870912 ~512MB)
+      index-cache-size: 536870912
+      # Enable garbage collection (default: true)
+      gc: true
+    gcs:
+      # Google Cloud Storage bucket name
+      bucket: ""
+      # Google Cloud project ID
+      project-id: ""
+      # Path prefix within the bucket
+      prefix: ""
+      # Path to service account credentials file (optional - uses Application Default Credentials if not set)
+      credentials-file: ""
+    s3:
+      # AWS S3 bucket name
+      bucket: ""
+      # AWS region
+      region: ""
+      # Path prefix within the bucket
+      prefix: ""
+      # AWS access key ID (optional - uses default credential chain if not set)
+      access-key-id: ""
+      # AWS secret access key (optional - uses default credential chain if not set)
+      secret-access-key: ""
+
+  # Metadata storage plugin configuration
+  metadata:
+    # Plugin to use for metadata storage (sqlite)
+    plugin: "sqlite"
+    # Configuration options for each plugin
+    sqlite:
+      # Data directory for SQLite database file
+      data-dir: ".dingo/metadata.db"
 
 # Path to the UNIX domain socket file used by the server
 socketPath: "dingo.socket"
@@ -52,10 +95,6 @@ utxorpcPort: 9090
 # This is experimental and may break — use with caution
 intersectTip: false
 
-# Maximum cache size in bytes used by BadgerDB for block/index cache
-# Default: 1073741824 (1 GB)
-badgerCacheSize: 1073741824
-
 # Maximum total size (in bytes) of all transactions allowed in the mempool.
 # Transactions exceeding this limit will be rejected.
 # Default: 1048576 (1 MB)

package/generate_benchmarks.sh

@@ -0,0 +1,423 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 Blink Labs Software
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Script to generate benchmark results for Dingo ledger and database with historical tracking
+# Usage: ./generate_benchmarks.sh [output_file] [--write]
+#   --write: Write results to file (default: display only)
+
+WRITE_TO_FILE=false
+OUTPUT_FILE="benchmark_results.md"
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --write)
+            WRITE_TO_FILE=true
+            shift
+            ;;
+        -*)
+            echo "Unknown option: $1"
+            echo "Usage: $0 [output_file] [--write]"
+            exit 1
+            ;;
+        *)
+            # First non-option argument is the output file
+            if [[ -z "$OUTPUT_FILE_SET" ]]; then
+                OUTPUT_FILE="$1"
+                OUTPUT_FILE_SET=true
+            else
+                echo "Too many arguments. Usage: $0 [output_file] [--write]"
+                exit 1
+            fi
+            shift
+            ;;
+    esac
+done
+
+DATE=$(date +"%B %d, %Y")
+
+# Initialize environment information for benchmark report
+GO_VERSION=$(go version | awk '{print $3}' | sed 's/go//')
+OS=$(uname -s)
+ARCH=$(uname -m)
+CPU_CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "unknown")
+
+echo "Running all Dingo benchmarks..."
+echo "==============================="
+
+# Run benchmarks with progress output first
+echo "Executing benchmarks (this may take a few minutes)..."
+
+# Enable pipefail to catch go test failures in the pipeline
+set -o pipefail
+
+# Run go test once, capture output while showing progress
+BENCHMARK_OUTPUT=$(go test -bench=. -benchmem ./... -run=^$ 2>&1)
+GO_TEST_EXIT_CODE=$?
+
+# Show progress by parsing benchmark names from output
+echo "$BENCHMARK_OUTPUT" | grep "^Benchmark" | sed 's/Benchmark//' | sed 's/-[0-9]*$//' | while read -r name rest; do
+    echo "Running: $name-128"
+done
+
+# Check if go test succeeded
+if [[ $GO_TEST_EXIT_CODE -ne 0 ]]; then
+    echo "Benchmark run failed!"
+    exit 1
+fi
+
+# Count benchmarks
+BENCHMARK_COUNT=$(echo "$BENCHMARK_OUTPUT" | grep "^Benchmark" | wc -l)
+
+echo "Found $BENCHMARK_COUNT benchmarks across all packages"
+echo ""
+
+# Function to parse benchmark line
+parse_benchmark() {
+    local line="$1"
+    local name
+    name=$(echo "$line" | awk '{print $1}' | sed 's/Benchmark//' | sed 's/-[0-9]*$//')
+    local ops_sec
+    ops_sec=$(echo "$line" | awk '{print $2}' | sed 's/,//g')
+    local time_val
+    time_val=$(echo "$line" | awk '{print $3}')
+    local time_unit
+    time_unit=$(echo "$line" | awk '{print $4}')
+    local mem_val
+    mem_val=$(echo "$line" | awk '{print $5}')
+    local mem_unit
+    mem_unit=$(echo "$line" | awk '{print $6}')
+    local allocs_op
+    allocs_op=$(echo "$line" | awk '{print $7}')
+
+    # Format time
+    if [[ "$time_unit" == "ns/op" ]]; then
+        time_op="${time_val}ns"
+    elif [[ "$time_unit" == "μs/op" ]] || [[ "$time_unit" == "µs/op" ]]; then
+        time_op="${time_val}μs"
+    elif [[ "$time_unit" == "ms/op" ]]; then
+        time_op="${time_val}ms"
+    elif [[ "$time_unit" == "s/op" ]]; then
+        time_op="${time_val}s"
+    else
+        time_op="${time_val}${time_unit}"
+    fi
+
+    # Format memory
+    if [[ "$mem_unit" == "B/op" ]]; then
+        if [[ $mem_val -gt 1000 ]]; then
+            mem_kb=$((mem_val / 1000))
+            mem_op="${mem_kb}KB"
+        else
+            mem_op="${mem_val}B"
+        fi
+    else
+        mem_op="${mem_val}${mem_unit}"
+    fi
+
+    # Format benchmark name nicely
+    formatted_name=$(echo "$name" | sed 's/\([A-Z]\)/ \1/g' | sed 's/^ //' | sed 's/NoData$/ (No Data)/' | sed 's/RealData$/ (Real Data)/')
+
+    echo "$formatted_name|$ops_sec|$time_op|$mem_op|$allocs_op"
+}
+
+# Parse current results into associative array
+declare -A current_results
+while IFS= read -r line; do
+    if [[ "$line" =~ ^Benchmark ]]; then
+        parsed=$(parse_benchmark "$line")
+        name=$(echo "$parsed" | cut -d'|' -f1)
+        data=$(echo "$parsed" | cut -d'|' -f2-)
+        current_results["$name"]="$data"
+    fi
+done <<< "$BENCHMARK_OUTPUT"
+
+# Display current results summary
+echo "Current Benchmark Summary"
+echo "-------------------------"
+echo "Fastest benchmarks (>100k ops/sec):"
+echo "$BENCHMARK_OUTPUT" | grep "^Benchmark" | sort -k2 -nr | head -3 | while read -r line; do
+    name=$(echo "$line" | awk '{print $1}' | sed 's/Benchmark//' | sed 's/-128$//')
+    ops=$(echo "$line" | awk '{print $2}' | sed 's/,//g')
+    echo "  - $name: ${ops} ops/sec"
+done
+
+echo ""
+echo "Slowest benchmarks (<1k ops/sec):"
+echo "$BENCHMARK_OUTPUT" | grep "^Benchmark" | awk '$2 < 1000' | while read -r line; do
+    name=$(echo "$line" | awk '{print $1}' | sed 's/Benchmark//' | sed 's/-128$//')
+    ops=$(echo "$line" | awk '{print $2}' | sed 's/,//g')
+    echo "  - $name: ${ops} ops/sec"
+done
+
+echo ""
+echo "Memory usage:"
+echo "$BENCHMARK_OUTPUT" | grep "^Benchmark" | sort -k5 -nr | head -3 | while read -r line; do
+    name=$(echo "$line" | awk '{print $1}' | sed 's/Benchmark//' | sed 's/-128$//')
+    mem=$(echo "$line" | awk '{print $5}')
+    echo "  - $name: ${mem}B per op"
+done
+
+# Read previous results if file exists and we're comparing
+declare -A previous_results
+previous_date=""
+MAJOR_CHANGES=false
+
+if [[ -f "$OUTPUT_FILE" && "$WRITE_TO_FILE" == "true" ]]; then
+    echo ""
+    echo "Comparing with previous results..."
+    # Extract previous date
+    previous_date=$(grep "\*\*Date\*\*:" "$OUTPUT_FILE" | head -1 | sed 's/.*\*\*Date\*\*: //' || echo "")
+
+    # Parse previous benchmark table
+    in_table=false
+    while IFS= read -r line; do
+        # Stop parsing at performance changes or historical results sections
+        if [[ "$line" == "## Performance Changes" || "$line" == "## Historical Results" ]]; then
+            break
+        fi
+        if [[ "$line" == "| Benchmark | Operations/sec | Time/op | Memory/op | Allocs/op |" ]]; then
+            in_table=true
+            continue
+        fi
+        if [[ "$in_table" == true && "$line" =~ ^\|.*\|.*\|.*\|.*\|.*\|$ && "$line" != "|-----------|*" ]]; then
+            # Parse table row
+            benchmark=$(echo "$line" | sed 's/^| //' | cut -d'|' -f1 | sed 's/ *$//')
+            ops_sec=$(echo "$line" | sed 's/^| //' | cut -d'|' -f2 | sed 's/ //g' | sed 's/,//g')
+            time_op=$(echo "$line" | sed 's/^| //' | cut -d'|' -f3 | sed 's/ //g')
+            mem_op=$(echo "$line" | sed 's/^| //' | cut -d'|' -f4 | sed 's/ //g')
+            allocs_op=$(echo "$line" | sed 's/^| //' | cut -d'|' -f5 | sed 's/ //g')
+            if [[ -n "$benchmark" && -n "$ops_sec" ]]; then
+                previous_results["$benchmark"]="$ops_sec|$time_op|$mem_op|$allocs_op"
+            fi
+        fi
+        if [[ "$in_table" == true && "$line" == "" ]]; then
+            in_table=false
+        fi
+    done < "$OUTPUT_FILE"
+fi
+
+# Generate performance comparison if we have previous results
+if [[ -n "$previous_date" && "$WRITE_TO_FILE" == "true" ]]; then
+    # Track changes
+    declare -a faster_benchmarks
+    declare -a slower_benchmarks
+    declare -a new_benchmarks
+    declare -a removed_benchmarks
+
+    # Compare results
+    for benchmark in "${!current_results[@]}"; do
+        if [[ -n "${previous_results[$benchmark]}" ]]; then
+            # Benchmark exists in both
+            current_data="${current_results[$benchmark]}"
+            previous_data="${previous_results[$benchmark]}"
+
+            current_ops=$(echo "$current_data" | cut -d'|' -f1)
+            previous_ops=$(echo "$previous_data" | cut -d'|' -f1)
+
+            if [[ "$current_ops" =~ ^[0-9]+$ && "$previous_ops" =~ ^[0-9]+$ && $previous_ops -gt 0 ]]; then
+                change=$(( (current_ops - previous_ops) * 100 / previous_ops ))
+                if [[ $change -gt 10 ]]; then
+                    faster_benchmarks+=("$benchmark (+${change}%)")
+                elif [[ $change -lt -10 ]]; then
+                    change_abs=$(( (previous_ops - current_ops) * 100 / previous_ops ))
+                    slower_benchmarks+=("$benchmark (-${change_abs}%)")
+                    MAJOR_CHANGES=true
+                fi
+            fi
+        else
+            new_benchmarks+=("$benchmark")
+        fi
+    done
+
+    # Check for removed benchmarks
+    for benchmark in "${!previous_results[@]}"; do
+        if [[ -z "${current_results[$benchmark]}" ]]; then
+            removed_benchmarks+=("$benchmark")
+        fi
+    done
+
+    echo ""
+    echo "Performance Changes Summary:"
+    echo "  Faster: ${#faster_benchmarks[@]} | Slower: ${#slower_benchmarks[@]} | New: ${#new_benchmarks[@]} | Removed: ${#removed_benchmarks[@]}"
+
+    # Report changes if any improvements, regressions, or new benchmarks detected
+    if [[ ${#faster_benchmarks[@]} -gt 0 || ${#slower_benchmarks[@]} -gt 0 || ${#new_benchmarks[@]} -gt 0 ]]; then
+        MAJOR_CHANGES=true
+    fi
+fi
+
+# Decide whether to write to file
+if [[ "$WRITE_TO_FILE" == "true" ]]; then
+    echo ""
+    if [[ "$MAJOR_CHANGES" == "true" ]]; then
+        echo "Writing results to file (major changes detected)..."
+    elif [[ -z "$previous_date" ]]; then
+        echo "Writing results to file (first benchmark run)..."
+    else
+        echo "Writing results to file (--write flag used)..."
+    fi
+
+        # Generate performance comparison for file
+        generate_comparison() {
+            echo "## Performance Changes"
+            echo ""
+            if [[ -z "$previous_date" ]]; then
+                echo "No previous results found. This is the first benchmark run."
+                echo ""
+                return
+            fi
+
+            echo "Changes since **$previous_date**:"
+            echo ""
+
+            # Track changes
+            declare -a faster_benchmarks
+            declare -a slower_benchmarks
+            declare -a new_benchmarks
+            declare -a removed_benchmarks
+
+            # Compare results
+            for benchmark in "${!current_results[@]}"; do
+                if [[ -n "${previous_results[$benchmark]}" ]]; then
+                    # Benchmark exists in both
+                    current_data="${current_results[$benchmark]}"
+                    previous_data="${previous_results[$benchmark]}"
+
+                    current_ops=$(echo "$current_data" | cut -d'|' -f1)
+                    previous_ops=$(echo "$previous_data" | cut -d'|' -f1)
+
+                    if [[ "$current_ops" =~ ^[0-9]+$ && "$previous_ops" =~ ^[0-9]+$ && $previous_ops -gt 0 ]]; then
+                        if [[ $current_ops -gt $previous_ops ]]; then
+                            change=$(( (current_ops - previous_ops) * 100 / previous_ops ))
+                            faster_benchmarks+=("$benchmark (+${change}%)")
+                        elif [[ $current_ops -lt $previous_ops ]]; then
+                            change=$(( (previous_ops - current_ops) * 100 / previous_ops ))
+                            slower_benchmarks+=("$benchmark (-${change}%)")
+                        fi
+                    fi
+                else
+                    new_benchmarks+=("$benchmark")
+                fi
+            done
+
+            # Check for removed benchmarks
+            for benchmark in "${!previous_results[@]}"; do
+                if [[ -z "${current_results[$benchmark]}" ]]; then
+                    removed_benchmarks+=("$benchmark")
+                fi
+            done
+
+            echo "### Summary"
+            echo "- **Faster benchmarks**: ${#faster_benchmarks[@]}"
+            echo "- **Slower benchmarks**: ${#slower_benchmarks[@]}"
+            echo "- **New benchmarks**: ${#new_benchmarks[@]}"
+            echo "- **Removed benchmarks**: ${#removed_benchmarks[@]}"
+            echo ""
+
+            if [[ ${#faster_benchmarks[@]} -gt 0 ]]; then
+                echo "### Top Improvements"
+                printf '%s\n' "${faster_benchmarks[@]}" | sort -t'(' -k2 -nr | head -5 | sed 's/^/- /'
+                echo ""
+            fi
+
+            if [[ ${#slower_benchmarks[@]} -gt 0 ]]; then
+                echo "### Performance Regressions"
+                printf '%s\n' "${slower_benchmarks[@]}" | sort -t'(' -k2 -nr | head -5 | sed 's/^/- /'
+                echo ""
+            fi
+
+            if [[ ${#new_benchmarks[@]} -gt 0 ]]; then
+                echo "### New Benchmarks Added"
+                printf '%s\n' "${new_benchmarks[@]}" | sed 's/^/- /'
+                echo ""
+            fi
+
+            if [[ ${#removed_benchmarks[@]} -gt 0 ]]; then
+                echo "### Benchmarks Removed"
+                printf '%s\n' "${removed_benchmarks[@]}" | sed 's/^/- /'
+                echo ""
+            fi
+        }
+
+        # Create the markdown file
+        cat > "$OUTPUT_FILE.tmp" << EOF
+# Dingo Ledger & Database Benchmark Results
+
+## Latest Results
+
+### Test Environment
+- **Date**: $DATE
+- **Go Version**: $GO_VERSION
+- **OS**: $OS
+- **Architecture**: $ARCH
+- **CPU Cores**: $CPU_CORES
+- **Data Source**: Real Cardano preview testnet data (40k+ blocks, slots 0-863,996)
+
+### Benchmark Results
+
+All benchmarks run with \`-benchmem\` flag showing memory allocations and operation counts.
+
+| Benchmark | Operations/sec | Time/op | Memory/op | Allocs/op |
+|-----------|----------------|---------|-----------|-----------|
+EOF
+
+        # Add current results to table
+        for benchmark in "${!current_results[@]}"; do
+            data="${current_results[$benchmark]}"
+            ops_sec=$(echo "$data" | cut -d'|' -f1)
+            time_op=$(echo "$data" | cut -d'|' -f2)
+            mem_op=$(echo "$data" | cut -d'|' -f3)
+            allocs_op=$(echo "$data" | cut -d'|' -f4)
+            echo "| $benchmark | $ops_sec | $time_op | $mem_op | $allocs_op |" >> "$OUTPUT_FILE.tmp"
+        done
+
+        # Add comparison section
+        generate_comparison >> "$OUTPUT_FILE.tmp"
+
+        # Add historical section if previous results exist
+        if [[ -n "$previous_date" ]]; then
+            echo "" >> "$OUTPUT_FILE.tmp"
+            echo "## Historical Results" >> "$OUTPUT_FILE.tmp"
+            echo "" >> "$OUTPUT_FILE.tmp"
+            echo "### $previous_date" >> "$OUTPUT_FILE.tmp"
+            echo "" >> "$OUTPUT_FILE.tmp"
+            echo "| Benchmark | Operations/sec | Time/op | Memory/op | Allocs/op |" >> "$OUTPUT_FILE.tmp"
+            echo "|-----------|----------------|---------|-----------|-----------|" >> "$OUTPUT_FILE.tmp"
+
+            # Add previous results
+            for benchmark in "${!previous_results[@]}"; do
+                data="${previous_results[$benchmark]}"
+                ops_sec=$(echo "$data" | cut -d'|' -f1)
+                time_op=$(echo "$data" | cut -d'|' -f2)
+                mem_op=$(echo "$data" | cut -d'|' -f3)
+                allocs_op=$(echo "$data" | cut -d'|' -f4)
+                echo "| $benchmark | $ops_sec | $time_op | $mem_op | $allocs_op |" >> "$OUTPUT_FILE.tmp"
+            done
+        fi
+
+        # Move temp file to final location
+        mv "$OUTPUT_FILE.tmp" "$OUTPUT_FILE"
+
+        echo "Benchmark results saved to $OUTPUT_FILE"
+else
+    echo ""
+    echo "To save these results to file, run: ./generate_benchmarks.sh --write"
+    echo "Results are only saved when major performance changes are detected."
+fi
+
+echo ""
+echo "Benchmark run complete!"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blinklabs/dingo",
-  "version": "0.17.0",
+  "version": "0.18.0",
   "description": "Dingo is a Cardano blockchain data node",
   "main": "index.js",
   "bin": {