@blinklabs/dingo 0.6.0 → 0.8.0

package/database/immutable/immutable.go

@@ -1,350 +0,0 @@
-// Copyright 2024 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package immutable
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"slices"
-	"strings"
-
-	ocommon "github.com/blinklabs-io/gouroboros/protocol/common"
-)
-
-type ImmutableDb struct {
-	dataDir string
-}
-
-type Block struct {
-	Type  uint
-	Slot  uint64
-	Hash  []byte
-	IsEbb bool
-	Cbor  []byte
-}
-
-// New returns a new ImmutableDb using the specified data directory or an error
-func New(dataDir string) (*ImmutableDb, error) {
-	if _, err := os.Stat(dataDir); err != nil {
-		return nil, err
-	}
-	i := &ImmutableDb{
-		dataDir: dataDir,
-	}
-	return i, nil
-}
-
-func (i *ImmutableDb) getChunkNames() ([]string, error) {
-	ret := []string{}
-	files, err := os.ReadDir(i.dataDir)
-	if err != nil {
-		return nil, err
-	}
-	for _, entry := range files {
-		entryName := entry.Name()
-		entryExt := filepath.Ext(entryName)
-		if entryExt != chunkFileExtension {
-			continue
-		}
-		chunkName := strings.TrimSuffix(entryName, entryExt)
-		ret = append(ret, chunkName)
-	}
-	slices.Sort(ret)
-	return ret, nil
-}
-
-func (i *ImmutableDb) getChunkNamesFromPoint(
-	point ocommon.Point,
-) ([]string, error) {
-	chunkNames, err := i.getChunkNames()
-	if err != nil {
-		return nil, err
-	}
-	// Return all chunks for the origin
-	if point.Slot == 0 {
-		return chunkNames, nil
-	}
-	lowerBound := 0
-	upperBound := len(chunkNames)
-	for lowerBound <= upperBound {
-		// Get chunk in the middle of the current bounds
-		middlePoint := (lowerBound + upperBound) / 2
-		middleChunkName := chunkNames[middlePoint]
-		middleSecondary, err := i.getChunkSecondaryIndex(middleChunkName)
-		if err != nil {
-			return nil, err
-		}
-		next, err := middleSecondary.Next()
-		if err != nil {
-			return nil, err
-		}
-		if next == nil {
-			break
-		}
-		startSlot := next.BlockOrEbb
-		var endSlot uint64
-		for {
-			next, err := middleSecondary.Next()
-			if err != nil {
-				return nil, err
-			}
-			if next == nil {
-				break
-			}
-			endSlot = next.BlockOrEbb
-		}
-		if point.Slot < startSlot {
-			// The slot we're looking for is less than the first slot in the chunk, so
-			// we can eliminate all later chunks
-			upperBound = middlePoint - 1
-		} else if point.Slot > endSlot {
-			// The slot we're looking for is greater than the last slot in the chunk, so
-			// we can eliminate all earlier chunks
-			lowerBound = middlePoint + 1
-		} else {
-			// We found the chunk that (probably) has the requested point
-			break
-		}
-	}
-	return chunkNames[lowerBound:], nil
-}
-
-func (i *ImmutableDb) getChunkPrimaryIndex(
-	chunkName string,
-) (*primaryIndex, error) {
-	primaryFilePath := filepath.Join(
-		i.dataDir,
-		chunkName+primaryFileExtension,
-	)
-	primary := newPrimaryIndex()
-	if err := primary.Open(primaryFilePath); err != nil {
-		return nil, fmt.Errorf(
-			"failed to read primary index: %s: %w",
-			primaryFilePath,
-			err,
-		)
-	}
-	return primary, nil
-}
-
-func (i *ImmutableDb) getChunkSecondaryIndex(
-	chunkName string,
-) (*secondaryIndex, error) {
-	primary, err := i.getChunkPrimaryIndex(chunkName)
-	if err != nil {
-		return nil, err
-	}
-	secondaryFilePath := filepath.Join(
-		i.dataDir,
-		chunkName+secondaryFileExtension,
-	)
-	secondary := newSecondaryIndex()
-	if err := secondary.Open(secondaryFilePath, primary); err != nil {
-		return nil, fmt.Errorf(
-			"failed to read secondary index: %s: %w",
-			secondaryFilePath,
-			err,
-		)
-	}
-	return secondary, nil
-}
-
-func (i *ImmutableDb) getChunk(chunkName string) (*chunk, error) {
-	// Open secondary index
-	secondary, err := i.getChunkSecondaryIndex(chunkName)
-	if err != nil {
-		return nil, err
-	}
-	// Open chunk
-	chunkFilePath := filepath.Join(
-		i.dataDir,
-		chunkName+chunkFileExtension,
-	)
-	chunk := newChunk()
-	if err := chunk.Open(chunkFilePath, secondary); err != nil {
-		return nil, fmt.Errorf(
-			"failed to read chunk: %s: %w",
-			chunkFilePath,
-			err,
-		)
-	}
-	return chunk, nil
-}
-
-func (i *ImmutableDb) GetTip() (*ocommon.Point, error) {
-	var ret *ocommon.Point
-	chunkNames, err := i.getChunkNames()
-	if err != nil {
-		return nil, err
-	}
-	if len(chunkNames) == 0 {
-		return nil, nil
-	}
-	secondary, err := i.getChunkSecondaryIndex(chunkNames[len(chunkNames)-1])
-	if err != nil {
-		return nil, err
-	}
-	var tmpPoint ocommon.Point
-	for {
-		next, err := secondary.Next()
-		if err != nil {
-			return nil, err
-		}
-		if next == nil {
-			break
-		}
-		tmpPoint = ocommon.NewPoint(
-			next.BlockOrEbb,
-			next.HeaderHash[:],
-		)
-		ret = &tmpPoint
-	}
-	return ret, nil
-}
-
-func (i *ImmutableDb) GetBlock(point ocommon.Point) (*Block, error) {
-	var err error
-	chunkNames, err := i.getChunkNamesFromPoint(point)
-	if err != nil {
-		return nil, err
-	}
-	chunk, err := i.getChunk(chunkNames[0])
-	if err != nil {
-		return nil, err
-	}
-	var tmpBlock *Block
-	for {
-		tmpBlock, err = chunk.Next()
-		if err != nil {
-			return nil, err
-		}
-		if tmpBlock == nil {
-			break
-		}
-		if tmpBlock.Slot != point.Slot {
-			continue
-		}
-		if string(tmpBlock.Hash) != string(point.Hash) {
-			continue
-		}
-		return tmpBlock, nil
-	}
-	return nil, nil
-}
-
-func (i *ImmutableDb) TruncateChunksFromPoint(point ocommon.Point) error {
-	chunkNames, err := i.getChunkNamesFromPoint(point)
-	if err != nil {
-		return err
-	}
-	for _, chunkName := range chunkNames {
-		chunkPathPrefix := filepath.Join(
-			i.dataDir,
-			chunkName,
-		)
-		if err := os.Remove(chunkPathPrefix + chunkFileExtension); err != nil {
-			return err
-		}
-		if err := os.Remove(chunkPathPrefix + secondaryFileExtension); err != nil {
-			return err
-		}
-		if err := os.Remove(chunkPathPrefix + primaryFileExtension); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (i *ImmutableDb) BlocksFromPoint(
-	point ocommon.Point,
-) (*BlockIterator, error) {
-	chunkNames, err := i.getChunkNamesFromPoint(point)
-	if err != nil {
-		return nil, err
-	}
-	ret := &BlockIterator{
-		db:         i,
-		chunkNames: chunkNames[:],
-		startPoint: point,
-	}
-	return ret, nil
-}
-
-type BlockIterator struct {
-	db              *ImmutableDb
-	startPoint      ocommon.Point
-	foundStartPoint bool
-	chunkNames      []string
-	chunkIdx        int
-	chunk           *chunk
-}
-
-func (b *BlockIterator) Next() (*Block, error) {
-	var err error
-	var tmpChunk *chunk
-	if b.chunk == nil {
-		if b.chunkIdx == 0 && len(b.chunkNames) > 0 {
-			// Open initial chunk
-			tmpChunk, err = b.db.getChunk(b.chunkNames[b.chunkIdx])
-			if err != nil {
-				return nil, err
-			}
-			b.chunk = tmpChunk
-		} else {
-			return nil, nil
-		}
-	}
-	var tmpBlock *Block
-	for {
-		tmpBlock, err = b.chunk.Next()
-		if err != nil {
-			return nil, err
-		}
-		if tmpBlock == nil {
-			// We've reached the end of the current chunk
-			if err := b.chunk.Close(); err != nil {
-				return nil, err
-			}
-			b.chunk = nil
-			b.chunkIdx++
-			if b.chunkIdx >= len(b.chunkNames) {
-				return nil, nil
-			}
-			tmpChunk, err = b.db.getChunk(b.chunkNames[b.chunkIdx])
-			if err != nil {
-				return nil, err
-			}
-			b.chunk = tmpChunk
-			continue
-		}
-		if !b.foundStartPoint {
-			if tmpBlock.Slot < b.startPoint.Slot {
-				continue
-			}
-			b.foundStartPoint = true
-		}
-		return tmpBlock, nil
-	}
-}
-
-func (b *BlockIterator) Close() error {
-	if b.chunk != nil {
-		if err := b.chunk.Close(); err != nil {
-			return err
-		}
-	}
-	return nil
-}

package/database/immutable/immutable_test.go

@@ -1,59 +0,0 @@
-// Copyright 2024 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package immutable_test
-
-import (
-	"encoding/hex"
-	"testing"
-
-	"github.com/blinklabs-io/dingo/database/immutable"
-)
-
-const (
-	testDataDir = "./testdata"
-)
-
-func TestGetTip(t *testing.T) {
-	// These expected values correspond to the last block in our test data
-	var expectedSlot uint64 = 38426380
-	expectedHash := "7ada6ed78f6caa499370da6548b143c59320f5e5283e5e80e202a994ba7bfebf"
-	imm, err := immutable.New(testDataDir)
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	tip, err := imm.GetTip()
-	if err != nil {
-		t.Fatalf("unexpected error: %s", err)
-	}
-	if tip == nil {
-		t.Fatalf("did not get expected tip value, got nil instead")
-	}
-	if tip.Slot != expectedSlot {
-		t.Fatalf(
-			"did not get expected slot value: expected %d, got %d",
-			expectedSlot,
-			tip.Slot,
-		)
-	}
-	if hex.EncodeToString(tip.Hash) != expectedHash {
-		t.Fatalf(
-			"did not get expected hash value: expected %s, got %x",
-			expectedHash,
-			tip.Hash,
-		)
-	}
-}
-
-// TODO: add tests for getting a specific block and getting a range of blocks that traverses multiple chunks (#386)

package/database/immutable/primary.go

@@ -1,106 +0,0 @@
-// Copyright 2024 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package immutable
-
-import (
-	"encoding/binary"
-	"errors"
-	"io"
-	"os"
-)
-
-const (
-	primaryFileExtension = ".primary"
-)
-
-type primaryIndex struct {
-	file            *os.File
-	slot            int
-	lastOffset      uint32
-	version         uint8
-	seenFirstOffset bool
-}
-
-type primaryIndexEntry struct {
-	RelativeSlot    int
-	SecondaryOffset uint32
-	empty           bool
-}
-
-func newPrimaryIndex() *primaryIndex {
-	return &primaryIndex{}
-}
-
-func (e primaryIndexEntry) Empty() bool {
-	return e.empty
-}
-
-func (p *primaryIndex) Open(path string) error {
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	p.file = f
-	// Read version
-	if err := binary.Read(f, binary.BigEndian, &p.version); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (p *primaryIndex) Close() error {
-	return p.file.Close()
-}
-
-func (p *primaryIndex) Next() (*primaryIndexEntry, error) {
-	var tmpOffset uint32
-	if err := binary.Read(p.file, binary.BigEndian, &tmpOffset); err != nil {
-		if errors.Is(err, io.EOF) {
-			// We've reached the end of the file
-			return nil, nil
-		}
-		return nil, err
-	}
-	empty := true
-	if tmpOffset > p.lastOffset || !p.seenFirstOffset {
-		empty = false
-		p.lastOffset = tmpOffset
-	}
-	tmpEntry := primaryIndexEntry{
-		RelativeSlot:    p.slot,
-		SecondaryOffset: tmpOffset,
-		empty:           empty,
-	}
-	p.slot++
-	p.seenFirstOffset = true
-	return &tmpEntry, nil
-}
-
-func (p *primaryIndex) NextOccupied() (*primaryIndexEntry, error) {
-	for {
-		next, err := p.Next()
-		if err != nil {
-			return nil, err
-		}
-		if next == nil {
-			break
-		}
-		if next.Empty() {
-			continue
-		}
-		return next, nil
-	}
-	return nil, nil
-}

package/database/immutable/secondary.go

@@ -1,103 +0,0 @@
-// Copyright 2024 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package immutable
-
-import (
-	"encoding/binary"
-	"fmt"
-	"os"
-)
-
-const (
-	secondaryFileExtension = ".secondary"
-)
-
-type secondaryIndex struct {
-	file     *os.File
-	fileSize int64
-	primary  *primaryIndex
-}
-
-type secondaryIndexEntryInner struct {
-	BlockOffset  uint64
-	HeaderOffset uint16
-	HeaderSize   uint16
-	Checksum     uint32
-	HeaderHash   [32]byte
-	BlockOrEbb   uint64
-}
-
-type secondaryIndexEntry struct {
-	secondaryIndexEntryInner
-	IsEbb bool
-}
-
-func newSecondaryIndex() *secondaryIndex {
-	return &secondaryIndex{}
-}
-
-func (s *secondaryIndex) Open(path string, primary *primaryIndex) error {
-	f, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-	s.file = f
-	s.primary = primary
-	if stat, err := f.Stat(); err != nil {
-		return err
-	} else {
-		s.fileSize = stat.Size()
-	}
-	return nil
-}
-
-func (s *secondaryIndex) Close() error {
-	if err := s.primary.Close(); err != nil {
-		return err
-	}
-	return s.file.Close()
-}
-
-func (s *secondaryIndex) Next() (*secondaryIndexEntry, error) {
-	nextOccupied, err := s.primary.NextOccupied()
-	if err != nil {
-		return nil, err
-	}
-	if nextOccupied == nil {
-		return nil, nil
-	}
-	// Look for final offset
-	if int64(nextOccupied.SecondaryOffset) == s.fileSize {
-		return nil, nil
-	}
-	// Seek to offset
-	if _, err := s.file.Seek(int64(nextOccupied.SecondaryOffset), 0); err != nil {
-		return nil, fmt.Errorf("failed while seeking: %w", err)
-	}
-	// Read entry
-	var tmpEntryInner secondaryIndexEntryInner
-	if err := binary.Read(s.file, binary.BigEndian, &tmpEntryInner); err != nil {
-		return nil, fmt.Errorf("failed while reading: %w", err)
-	}
-	ret := &secondaryIndexEntry{
-		secondaryIndexEntryInner: tmpEntryInner,
-	}
-	// Check for EBB
-	// A block with its secondary index in the first slot with a small BlockOrEbb value is *probably* an EBB
-	if nextOccupied.RelativeSlot == 0 && ret.BlockOrEbb < 200 {
-		ret.IsEbb = true
-	}
-	return ret, nil
-}

package/database/immutable/testdata/README.md

@@ -1,4 +0,0 @@
-# blockdb
-
-The files in this directory were pulled from the `immutable` directory of a Cardano node
-on the `preview` network

package/database/plugin/blob/badger/commit_timestamp.go

@@ -1,50 +0,0 @@
-// Copyright 2025 Blink Labs Software
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package badger
-
-import (
-	"math/big"
-
-	badger "github.com/dgraph-io/badger/v4"
-)
-
-const (
-	commitTimestampBlobKey = "metadata_commit_timestamp"
-)
-
-func (b *BlobStoreBadger) GetCommitTimestamp() (int64, error) {
-	txn := b.NewTransaction(false)
-	item, err := txn.Get([]byte(commitTimestampBlobKey))
-	if err != nil {
-		return 0, err
-	}
-	val, err := item.ValueCopy(nil)
-	if err != nil {
-		return 0, err
-	}
-	return new(big.Int).SetBytes(val).Int64(), nil
-}
-
-func (b *BlobStoreBadger) SetCommitTimestamp(
-	txn *badger.Txn,
-	timestamp int64,
-) error {
-	// Update badger
-	tmpTimestamp := new(big.Int).SetInt64(timestamp)
-	if err := txn.Set([]byte(commitTimestampBlobKey), tmpTimestamp.Bytes()); err != nil {
-		return err
-	}
-	return nil
-}