@blinklabs/dingo 0.7.0 → 0.8.1

package/database/plugin/blob/badger/database.go

@@ -1,152 +0,0 @@
-// Copyright 2025 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package badger
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"io/fs"
-	"log/slog"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/blinklabs-io/dingo/database/plugin"
-	badger "github.com/dgraph-io/badger/v4"
-)
-
-// Register plugin
-func init() {
-	plugin.Register(
-		plugin.PluginEntry{
-			Type: plugin.PluginTypeMetadata,
-			Name: "sqlite",
-		},
-	)
-}
-
-// BlobStoreBadger stores all data in badger. Data may not be persisted
-type BlobStoreBadger struct {
-	dataDir   string
-	db        *badger.DB
-	logger    *slog.Logger
-	gcEnabled bool
-}
-
-// New creates a new database
-func New(
-	dataDir string,
-	logger *slog.Logger,
-) (*BlobStoreBadger, error) {
-	var blobDb *badger.DB
-	var err error
-	db := &BlobStoreBadger{
-		dataDir: dataDir,
-		logger:  logger,
-	}
-	if dataDir == "" {
-		// No dataDir, use in-memory config
-		badgerOpts := badger.DefaultOptions("").
-			WithLogger(NewBadgerLogger(logger)).
-			// The default INFO logging is a bit verbose
-			WithLoggingLevel(badger.WARNING).
-			WithInMemory(true)
-		blobDb, err = badger.Open(badgerOpts)
-		if err != nil {
-			return nil, err
-		}
-	} else {
-		// Make sure that we can read data dir, and create if it doesn't exist
-		if _, err := os.Stat(dataDir); err != nil {
-			if !errors.Is(err, fs.ErrNotExist) {
-				return nil, fmt.Errorf("failed to read data dir: %w", err)
-			}
-			// Create data directory
-			if err := os.MkdirAll(dataDir, fs.ModePerm); err != nil {
-				return nil, fmt.Errorf("failed to create data dir: %w", err)
-			}
-		}
-		blobDir := filepath.Join(
-			dataDir,
-			"blob",
-		)
-		// Run GC periodically
-		db.gcEnabled = true
-		badgerOpts := badger.DefaultOptions(blobDir).
-			WithLogger(NewBadgerLogger(logger)).
-			// The default INFO logging is a bit verbose
-			WithLoggingLevel(badger.WARNING)
-		blobDb, err = badger.Open(badgerOpts)
-		if err != nil {
-			return nil, err
-		}
-	}
-	db.db = blobDb
-	if err := db.init(); err != nil {
-		return db, err
-	}
-	return db, nil
-}
-
-func (d *BlobStoreBadger) init() error {
-	if d.logger == nil {
-		// Create logger to throw away logs
-		// We do this so we don't have to add guards around every log operation
-		d.logger = slog.New(slog.NewJSONHandler(io.Discard, nil))
-	}
-	// Configure metrics
-	d.registerBlobMetrics()
-	// Configure GC
-	if d.gcEnabled {
-		go d.blobGc(time.NewTicker(5 * time.Minute))
-	}
-	return nil
-}
-
-func (d *BlobStoreBadger) blobGc(t *time.Ticker) {
-	for range t.C {
-	again:
-		err := d.DB().RunValueLogGC(0.5)
-		if err != nil {
-			// Log any actual errors
-			if !errors.Is(err, badger.ErrNoRewrite) {
-				d.logger.Warn(
-					fmt.Sprintf("blob DB: GC failure: %s", err),
-					"component", "database",
-				)
-			}
-		} else {
-			// Run it again if it just ran successfully
-			goto again
-		}
-	}
-}
-
-// Close gets the database handle from our BlobStore and closes it
-func (d *BlobStoreBadger) Close() error {
-	db := d.DB()
-	return db.Close()
-}
-
-// DB returns the database handle
-func (d *BlobStoreBadger) DB() *badger.DB {
-	return d.db
-}
-
-// NewTransaction creates a new badger transaction
-func (d *BlobStoreBadger) NewTransaction(update bool) *badger.Txn {
-	return d.DB().NewTransaction(update)
-}

package/database/plugin/blob/badger/logger.go

@@ -1,63 +0,0 @@
-// Copyright 2025 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package badger
-
-import (
-	"fmt"
-	"io"
-	"log/slog"
-)
-
-// BadgerLogger is a wrapper type to give our logger the expected interface
-type BadgerLogger struct {
-	logger *slog.Logger
-}
-
-func NewBadgerLogger(logger *slog.Logger) *BadgerLogger {
-	if logger == nil {
-		// Create logger to throw away logs
-		// We do this so we don't have to add guards around every log operation
-		logger = slog.New(slog.NewJSONHandler(io.Discard, nil))
-	}
-	return &BadgerLogger{logger: logger}
-}
-
-func (b *BadgerLogger) Infof(msg string, args ...any) {
-	b.logger.Info(
-		fmt.Sprintf(msg, args...),
-		"component", "database",
-	)
-}
-
-func (b *BadgerLogger) Warningf(msg string, args ...any) {
-	b.logger.Warn(
-		fmt.Sprintf(msg, args...),
-		"component", "database",
-	)
-}
-
-func (b *BadgerLogger) Debugf(msg string, args ...any) {
-	b.logger.Debug(
-		fmt.Sprintf(msg, args...),
-		"component", "database",
-	)
-}
-
-func (b *BadgerLogger) Errorf(msg string, args ...any) {
-	b.logger.Error(
-		fmt.Sprintf(msg, args...),
-		"component", "database",
-	)
-}

package/database/plugin/blob/badger/metrics.go

@@ -1,98 +0,0 @@
-// Copyright 2024 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package badger
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
-)
-
-const (
-	badgerMetricNamePrefix = "database_blob_"
-)
-
-func (d *BlobStoreBadger) registerBlobMetrics() {
-	// Badger exposes metrics via expvar, so we need to set up some translation
-	collector := collectors.NewExpvarCollector(
-		map[string]*prometheus.Desc{
-			// This list of metrics is derived from the metrics defined here:
-			// https://github.com/dgraph-io/badger/blob/v4.2.0/y/metrics.go#L78-L107
-			"badger_read_num_vlog": prometheus.NewDesc(
-				badgerMetricNamePrefix+"read_num_vlog", "", nil, nil,
-			),
-			"badger_read_bytes_vlog": prometheus.NewDesc(
-				badgerMetricNamePrefix+"read_bytes_vlog", "", nil, nil,
-			),
-			"badger_write_num_vlog": prometheus.NewDesc(
-				badgerMetricNamePrefix+"write_num_vlog", "", nil, nil,
-			),
-			"badger_write_bytes_vlog": prometheus.NewDesc(
-				badgerMetricNamePrefix+"write_bytes_vlog", "", nil, nil,
-			),
-			"badger_read_bytes_lsm": prometheus.NewDesc(
-				badgerMetricNamePrefix+"read_bytes_lsm", "", nil, nil,
-			),
-			"badger_write_bytes_l0": prometheus.NewDesc(
-				badgerMetricNamePrefix+"write_bytes_l0", "", nil, nil,
-			),
-			"badger_write_bytes_compaction": prometheus.NewDesc(
-				badgerMetricNamePrefix+"write_bytes_compaction", "", nil, nil,
-			),
-			"badger_get_num_lsm": prometheus.NewDesc(
-				badgerMetricNamePrefix+"get_num_lsm", "", nil, nil,
-			),
-			"badger_hit_num_lsm_bloom_filter": prometheus.NewDesc(
-				badgerMetricNamePrefix+"hit_num_lsm_bloom_filter", "", nil, nil,
-			),
-			"badger_get_num_memtable": prometheus.NewDesc(
-				badgerMetricNamePrefix+"get_num_memtable", "", nil, nil,
-			),
-			"badger_get_num_user": prometheus.NewDesc(
-				badgerMetricNamePrefix+"get_num_user", "", nil, nil,
-			),
-			"badger_put_num_user": prometheus.NewDesc(
-				badgerMetricNamePrefix+"put_num_user", "", nil, nil,
-			),
-			"badger_write_bytes_user": prometheus.NewDesc(
-				badgerMetricNamePrefix+"write_bytes_user", "", nil, nil,
-			),
-			"badger_get_with_result_num_user": prometheus.NewDesc(
-				badgerMetricNamePrefix+"get_with_result_num_user", "", nil, nil,
-			),
-			"badger_iterator_num_user": prometheus.NewDesc(
-				badgerMetricNamePrefix+"iterator_num_user", "", nil, nil,
-			),
-			"badger_size_bytes_lsm": prometheus.NewDesc(
-				badgerMetricNamePrefix+"size_bytes_lsm", "", nil, nil,
-			),
-			"badger_size_bytes_vlog": prometheus.NewDesc(
-				badgerMetricNamePrefix+"size_bytes_vlog", "", nil, nil,
-			),
-			"badger_write_pending_num_memtable": prometheus.NewDesc(
-				badgerMetricNamePrefix+"write_pending_num_memtable",
-				"",
-				nil,
-				nil,
-			),
-			"badger_compaction_current_num_lsm": prometheus.NewDesc(
-				badgerMetricNamePrefix+"compaction_current_num_lsm",
-				"",
-				nil,
-				nil,
-			),
-		},
-	)
-	prometheus.MustRegister(collector)
-}

package/database/plugin/blob/blob.go

@@ -1,19 +0,0 @@
-// Copyright 2025 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package blob
-
-import (
-	_ "github.com/blinklabs-io/dingo/database/plugin/blob/badger"
-)

package/database/plugin/blob/store.go

@@ -1,40 +0,0 @@
-// Copyright 2025 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package blob
-
-import (
-	"log/slog"
-
-	badgerPlugin "github.com/blinklabs-io/dingo/database/plugin/blob/badger"
-	badger "github.com/dgraph-io/badger/v4"
-)
-
-type BlobStore interface {
-	// matches badger.DB
-	Close() error
-	NewTransaction(bool) *badger.Txn
-
-	// Our specific functions
-	GetCommitTimestamp() (int64, error)
-	SetCommitTimestamp(*badger.Txn, int64) error
-}
-
-// For now, this always returns a badger plugin
-func New(
-	pluginName, dataDir string,
-	logger *slog.Logger,
-) (BlobStore, error) {
-	return badgerPlugin.New(dataDir, logger)
-}

package/database/plugin/log.go

@@ -1,27 +0,0 @@
-// Copyright 2025 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package plugin
-
-// Logger provides a logging interface for plugins.
-type Logger interface {
-	Info(string, ...any)
-	Warn(string, ...any)
-	Debug(string, ...any)
-	Error(string, ...any)
-
-	// Deprecated
-	// Fatal(string, ...any) in favor of Error
-	// With slog Fatal is replaced with Error and os.Exit(1)
-}

package/database/plugin/metadata/metadata.go

@@ -1,19 +0,0 @@
-// Copyright 2025 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package metadata
-
-import (
-	_ "github.com/blinklabs-io/dingo/database/plugin/metadata/sqlite"
-)