@blinklabs/dingo 0.6.0

package/database/plugin/blob/badger/database.go

@@ -0,0 +1,152 @@
+// Copyright 2025 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package badger
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/blinklabs-io/dingo/database/plugin"
+	badger "github.com/dgraph-io/badger/v4"
+)
+
+// Register plugin
+func init() {
+	plugin.Register(
+		plugin.PluginEntry{
+			Type: plugin.PluginTypeMetadata,
+			Name: "sqlite",
+		},
+	)
+}
+
+// BlobStoreBadger stores all data in badger. Data may not be persisted
+type BlobStoreBadger struct {
+	dataDir   string
+	db        *badger.DB
+	logger    *slog.Logger
+	gcEnabled bool
+}
+
+// New creates a new database
+func New(
+	dataDir string,
+	logger *slog.Logger,
+) (*BlobStoreBadger, error) {
+	var blobDb *badger.DB
+	var err error
+	db := &BlobStoreBadger{
+		dataDir: dataDir,
+		logger:  logger,
+	}
+	if dataDir == "" {
+		// No dataDir, use in-memory config
+		badgerOpts := badger.DefaultOptions("").
+			WithLogger(NewBadgerLogger(logger)).
+			// The default INFO logging is a bit verbose
+			WithLoggingLevel(badger.WARNING).
+			WithInMemory(true)
+		blobDb, err = badger.Open(badgerOpts)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		// Make sure that we can read data dir, and create if it doesn't exist
+		if _, err := os.Stat(dataDir); err != nil {
+			if !errors.Is(err, fs.ErrNotExist) {
+				return nil, fmt.Errorf("failed to read data dir: %w", err)
+			}
+			// Create data directory
+			if err := os.MkdirAll(dataDir, fs.ModePerm); err != nil {
+				return nil, fmt.Errorf("failed to create data dir: %w", err)
+			}
+		}
+		blobDir := filepath.Join(
+			dataDir,
+			"blob",
+		)
+		// Run GC periodically
+		db.gcEnabled = true
+		badgerOpts := badger.DefaultOptions(blobDir).
+			WithLogger(NewBadgerLogger(logger)).
+			// The default INFO logging is a bit verbose
+			WithLoggingLevel(badger.WARNING)
+		blobDb, err = badger.Open(badgerOpts)
+		if err != nil {
+			return nil, err
+		}
+	}
+	db.db = blobDb
+	if err := db.init(); err != nil {
+		return db, err
+	}
+	return db, nil
+}
+
+func (d *BlobStoreBadger) init() error {
+	if d.logger == nil {
+		// Create logger to throw away logs
+		// We do this so we don't have to add guards around every log operation
+		d.logger = slog.New(slog.NewJSONHandler(io.Discard, nil))
+	}
+	// Configure metrics
+	d.registerBlobMetrics()
+	// Configure GC
+	if d.gcEnabled {
+		go d.blobGc(time.NewTicker(5 * time.Minute))
+	}
+	return nil
+}
+
+func (d *BlobStoreBadger) blobGc(t *time.Ticker) {
+	for range t.C {
+	again:
+		err := d.DB().RunValueLogGC(0.5)
+		if err != nil {
+			// Log any actual errors
+			if !errors.Is(err, badger.ErrNoRewrite) {
+				d.logger.Warn(
+					fmt.Sprintf("blob DB: GC failure: %s", err),
+					"component", "database",
+				)
+			}
+		} else {
+			// Run it again if it just ran successfully
+			goto again
+		}
+	}
+}
+
+// Close gets the database handle from our BlobStore and closes it
+func (d *BlobStoreBadger) Close() error {
+	db := d.DB()
+	return db.Close()
+}
+
+// DB returns the database handle
+func (d *BlobStoreBadger) DB() *badger.DB {
+	return d.db
+}
+
+// NewTransaction creates a new badger transaction
+func (d *BlobStoreBadger) NewTransaction(update bool) *badger.Txn {
+	return d.DB().NewTransaction(update)
+}

package/database/plugin/blob/badger/logger.go

@@ -0,0 +1,63 @@
+// Copyright 2025 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package badger
+
+import (
+	"fmt"
+	"io"
+	"log/slog"
+)
+
+// BadgerLogger is a wrapper type to give our logger the expected interface
+type BadgerLogger struct {
+	logger *slog.Logger
+}
+
+func NewBadgerLogger(logger *slog.Logger) *BadgerLogger {
+	if logger == nil {
+		// Create logger to throw away logs
+		// We do this so we don't have to add guards around every log operation
+		logger = slog.New(slog.NewJSONHandler(io.Discard, nil))
+	}
+	return &BadgerLogger{logger: logger}
+}
+
+func (b *BadgerLogger) Infof(msg string, args ...any) {
+	b.logger.Info(
+		fmt.Sprintf(msg, args...),
+		"component", "database",
+	)
+}
+
+func (b *BadgerLogger) Warningf(msg string, args ...any) {
+	b.logger.Warn(
+		fmt.Sprintf(msg, args...),
+		"component", "database",
+	)
+}
+
+func (b *BadgerLogger) Debugf(msg string, args ...any) {
+	b.logger.Debug(
+		fmt.Sprintf(msg, args...),
+		"component", "database",
+	)
+}
+
+func (b *BadgerLogger) Errorf(msg string, args ...any) {
+	b.logger.Error(
+		fmt.Sprintf(msg, args...),
+		"component", "database",
+	)
+}

package/database/plugin/blob/badger/metrics.go

@@ -0,0 +1,98 @@
+// Copyright 2024 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package badger
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+)
+
+const (
+	badgerMetricNamePrefix = "database_blob_"
+)
+
+func (d *BlobStoreBadger) registerBlobMetrics() {
+	// Badger exposes metrics via expvar, so we need to set up some translation
+	collector := collectors.NewExpvarCollector(
+		map[string]*prometheus.Desc{
+			// This list of metrics is derived from the metrics defined here:
+			// https://github.com/dgraph-io/badger/blob/v4.2.0/y/metrics.go#L78-L107
+			"badger_read_num_vlog": prometheus.NewDesc(
+				badgerMetricNamePrefix+"read_num_vlog", "", nil, nil,
+			),
+			"badger_read_bytes_vlog": prometheus.NewDesc(
+				badgerMetricNamePrefix+"read_bytes_vlog", "", nil, nil,
+			),
+			"badger_write_num_vlog": prometheus.NewDesc(
+				badgerMetricNamePrefix+"write_num_vlog", "", nil, nil,
+			),
+			"badger_write_bytes_vlog": prometheus.NewDesc(
+				badgerMetricNamePrefix+"write_bytes_vlog", "", nil, nil,
+			),
+			"badger_read_bytes_lsm": prometheus.NewDesc(
+				badgerMetricNamePrefix+"read_bytes_lsm", "", nil, nil,
+			),
+			"badger_write_bytes_l0": prometheus.NewDesc(
+				badgerMetricNamePrefix+"write_bytes_l0", "", nil, nil,
+			),
+			"badger_write_bytes_compaction": prometheus.NewDesc(
+				badgerMetricNamePrefix+"write_bytes_compaction", "", nil, nil,
+			),
+			"badger_get_num_lsm": prometheus.NewDesc(
+				badgerMetricNamePrefix+"get_num_lsm", "", nil, nil,
+			),
+			"badger_hit_num_lsm_bloom_filter": prometheus.NewDesc(
+				badgerMetricNamePrefix+"hit_num_lsm_bloom_filter", "", nil, nil,
+			),
+			"badger_get_num_memtable": prometheus.NewDesc(
+				badgerMetricNamePrefix+"get_num_memtable", "", nil, nil,
+			),
+			"badger_get_num_user": prometheus.NewDesc(
+				badgerMetricNamePrefix+"get_num_user", "", nil, nil,
+			),
+			"badger_put_num_user": prometheus.NewDesc(
+				badgerMetricNamePrefix+"put_num_user", "", nil, nil,
+			),
+			"badger_write_bytes_user": prometheus.NewDesc(
+				badgerMetricNamePrefix+"write_bytes_user", "", nil, nil,
+			),
+			"badger_get_with_result_num_user": prometheus.NewDesc(
+				badgerMetricNamePrefix+"get_with_result_num_user", "", nil, nil,
+			),
+			"badger_iterator_num_user": prometheus.NewDesc(
+				badgerMetricNamePrefix+"iterator_num_user", "", nil, nil,
+			),
+			"badger_size_bytes_lsm": prometheus.NewDesc(
+				badgerMetricNamePrefix+"size_bytes_lsm", "", nil, nil,
+			),
+			"badger_size_bytes_vlog": prometheus.NewDesc(
+				badgerMetricNamePrefix+"size_bytes_vlog", "", nil, nil,
+			),
+			"badger_write_pending_num_memtable": prometheus.NewDesc(
+				badgerMetricNamePrefix+"write_pending_num_memtable",
+				"",
+				nil,
+				nil,
+			),
+			"badger_compaction_current_num_lsm": prometheus.NewDesc(
+				badgerMetricNamePrefix+"compaction_current_num_lsm",
+				"",
+				nil,
+				nil,
+			),
+		},
+	)
+	prometheus.MustRegister(collector)
+}

package/database/plugin/blob/blob.go

@@ -0,0 +1,19 @@
+// Copyright 2025 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blob
+
+import (
+	_ "github.com/blinklabs-io/dingo/database/plugin/blob/badger"
+)

package/database/plugin/blob/store.go

@@ -0,0 +1,40 @@
+// Copyright 2025 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blob
+
+import (
+	"log/slog"
+
+	badgerPlugin "github.com/blinklabs-io/dingo/database/plugin/blob/badger"
+	badger "github.com/dgraph-io/badger/v4"
+)
+
+type BlobStore interface {
+	// matches badger.DB
+	Close() error
+	NewTransaction(bool) *badger.Txn
+
+	// Our specific functions
+	GetCommitTimestamp() (int64, error)
+	SetCommitTimestamp(*badger.Txn, int64) error
+}
+
+// For now, this always returns a badger plugin
+func New(
+	pluginName, dataDir string,
+	logger *slog.Logger,
+) (BlobStore, error) {
+	return badgerPlugin.New(dataDir, logger)
+}

package/database/plugin/log.go

@@ -0,0 +1,27 @@
+// Copyright 2025 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package plugin
+
+// Logger provides a logging interface for plugins.
+type Logger interface {
+	Info(string, ...any)
+	Warn(string, ...any)
+	Debug(string, ...any)
+	Error(string, ...any)
+
+	// Deprecated
+	// Fatal(string, ...any) in favor of Error
+	// With slog Fatal is replaced with Error and os.Exit(1)
+}

package/database/plugin/metadata/metadata.go

@@ -0,0 +1,19 @@
+// Copyright 2025 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metadata
+
+import (
+	_ "github.com/blinklabs-io/dingo/database/plugin/metadata/sqlite"
+)

package/database/plugin/metadata/sqlite/account.go

@@ -0,0 +1,224 @@
+// Copyright 2025 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlite
+
+import (
+	"github.com/blinklabs-io/dingo/database/plugin/metadata/sqlite/models"
+	lcommon "github.com/blinklabs-io/gouroboros/ledger/common"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+)
+
+// GetAccount gets an account
+func (d *MetadataStoreSqlite) GetAccount(
+	stakeKey []byte,
+	txn *gorm.DB,
+) (models.Account, error) {
+	ret := models.Account{}
+	tmpAccount := models.Account{}
+	if txn != nil {
+		if result := txn.First(&tmpAccount, "staking_key = ?", stakeKey); result.Error != nil {
+			return ret, result.Error
+		}
+	} else {
+		if result := d.DB().First(&tmpAccount, "staking_key = ?", stakeKey); result.Error != nil {
+			return ret, result.Error
+		}
+	}
+	ret = tmpAccount
+	return ret, nil
+}
+
+// SetAccount saves an account
+func (d *MetadataStoreSqlite) SetAccount(
+	stakeKey, pkh, drep []byte,
+	slot uint64,
+	active bool,
+	txn *gorm.DB,
+) error {
+	tmpItem := models.Account{
+		StakingKey: stakeKey,
+		AddedSlot:  slot,
+		Pool:       pkh,
+		Drep:       drep,
+	}
+	if txn != nil {
+		if result := txn.Clauses(clause.OnConflict{UpdateAll: true}).Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	} else {
+		if result := d.DB().Clauses(clause.OnConflict{UpdateAll: true}).Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	}
+	return nil
+}
+
+// SetDeregistration saves a deregistration certificate
+func (d *MetadataStoreSqlite) SetDeregistration(
+	cert *lcommon.DeregistrationCertificate,
+	slot uint64,
+	txn *gorm.DB,
+) error {
+	stakeKey := cert.StakeCredential.Credential.Bytes()
+	tmpAccount, err := d.GetAccount(stakeKey, txn)
+	if err != nil {
+		return err
+	}
+	tmpItem := models.Deregistration{
+		StakingKey: stakeKey,
+		AddedSlot:  slot,
+	}
+	tmpAccount.Active = false
+	if txn != nil {
+		if accountErr := txn.Save(&tmpAccount); accountErr.Error != nil {
+			return accountErr.Error
+		}
+		if result := txn.Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	} else {
+		if accountErr := d.DB().Save(&tmpAccount); accountErr.Error != nil {
+			return accountErr.Error
+		}
+		if result := d.DB().Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	}
+	return nil
+}
+
+// SetStakeRegistration saves a stake registration certificate and account
+func (d *MetadataStoreSqlite) SetStakeRegistration(
+	cert *lcommon.StakeRegistrationCertificate,
+	slot, deposit uint64,
+	txn *gorm.DB,
+) error {
+	stakeKey := cert.StakeRegistration.Credential.Bytes()
+	tmpItem := models.StakeRegistration{
+		StakingKey:    stakeKey,
+		AddedSlot:     slot,
+		DepositAmount: deposit,
+	}
+	if err := d.SetAccount(stakeKey, nil, nil, slot, true, txn); err != nil {
+		return err
+	}
+	if txn != nil {
+		if result := txn.Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	} else {
+		if result := d.DB().Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	}
+	return nil
+}
+
+// SetRegistration saves a registration certificate and account
+func (d *MetadataStoreSqlite) SetRegistration(
+	cert *lcommon.RegistrationCertificate,
+	slot, deposit uint64,
+	txn *gorm.DB,
+) error {
+	stakeKey := cert.StakeCredential.Credential.Bytes()
+	tmpItem := models.Registration{
+		StakingKey:    stakeKey,
+		AddedSlot:     slot,
+		DepositAmount: deposit,
+	}
+	if err := d.SetAccount(stakeKey, nil, nil, slot, true, txn); err != nil {
+		return err
+	}
+	if txn != nil {
+		if result := txn.Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	} else {
+		if result := d.DB().Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	}
+	return nil
+}
+
+// SetStakeDelegation saves a stake delegation certificate
+func (d *MetadataStoreSqlite) SetStakeDelegation(
+	cert *lcommon.StakeDelegationCertificate,
+	slot uint64,
+	txn *gorm.DB,
+) error {
+	stakeKey := cert.StakeCredential.Credential.Bytes()
+	tmpAccount, err := d.GetAccount(stakeKey, txn)
+	if err != nil {
+		return err
+	}
+	tmpItem := models.StakeDelegation{
+		StakingKey:  stakeKey,
+		PoolKeyHash: cert.PoolKeyHash[:],
+		AddedSlot:   slot,
+	}
+	tmpAccount.Pool = tmpItem.PoolKeyHash
+	if txn != nil {
+		if accountErr := txn.Save(&tmpAccount); accountErr.Error != nil {
+			return accountErr.Error
+		}
+		if result := txn.Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	} else {
+		if accountErr := d.DB().Save(&tmpAccount); accountErr.Error != nil {
+			return accountErr.Error
+		}
+		if result := d.DB().Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	}
+	return nil
+}
+
+// SetStakeDeregistration saves a stake deregistration certificate
+func (d *MetadataStoreSqlite) SetStakeDeregistration(
+	cert *lcommon.StakeDeregistrationCertificate,
+	slot uint64,
+	txn *gorm.DB,
+) error {
+	stakeKey := cert.StakeDeregistration.Credential.Bytes()
+	tmpAccount, err := d.GetAccount(stakeKey, txn)
+	if err != nil {
+		return err
+	}
+	tmpItem := models.StakeDeregistration{
+		StakingKey: stakeKey,
+		AddedSlot:  slot,
+	}
+	tmpAccount.Active = false
+	if txn != nil {
+		if accountErr := txn.Save(&tmpAccount); accountErr.Error != nil {
+			return accountErr.Error
+		}
+		if result := txn.Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	} else {
+		if accountErr := d.DB().Save(&tmpAccount); accountErr.Error != nil {
+			return accountErr.Error
+		}
+		if result := d.DB().Create(&tmpItem); result.Error != nil {
+			return result.Error
+		}
+	}
+	return nil
+}