@blinklabs/dingo 0.6.0

package/database/immutable/immutable.go

@@ -0,0 +1,350 @@
+// Copyright 2024 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package immutable
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	ocommon "github.com/blinklabs-io/gouroboros/protocol/common"
+)
+
+type ImmutableDb struct {
+	dataDir string
+}
+
+type Block struct {
+	Type  uint
+	Slot  uint64
+	Hash  []byte
+	IsEbb bool
+	Cbor  []byte
+}
+
+// New returns a new ImmutableDb using the specified data directory or an error
+func New(dataDir string) (*ImmutableDb, error) {
+	if _, err := os.Stat(dataDir); err != nil {
+		return nil, err
+	}
+	i := &ImmutableDb{
+		dataDir: dataDir,
+	}
+	return i, nil
+}
+
+func (i *ImmutableDb) getChunkNames() ([]string, error) {
+	ret := []string{}
+	files, err := os.ReadDir(i.dataDir)
+	if err != nil {
+		return nil, err
+	}
+	for _, entry := range files {
+		entryName := entry.Name()
+		entryExt := filepath.Ext(entryName)
+		if entryExt != chunkFileExtension {
+			continue
+		}
+		chunkName := strings.TrimSuffix(entryName, entryExt)
+		ret = append(ret, chunkName)
+	}
+	slices.Sort(ret)
+	return ret, nil
+}
+
+func (i *ImmutableDb) getChunkNamesFromPoint(
+	point ocommon.Point,
+) ([]string, error) {
+	chunkNames, err := i.getChunkNames()
+	if err != nil {
+		return nil, err
+	}
+	// Return all chunks for the origin
+	if point.Slot == 0 {
+		return chunkNames, nil
+	}
+	lowerBound := 0
+	upperBound := len(chunkNames)
+	for lowerBound <= upperBound {
+		// Get chunk in the middle of the current bounds
+		middlePoint := (lowerBound + upperBound) / 2
+		middleChunkName := chunkNames[middlePoint]
+		middleSecondary, err := i.getChunkSecondaryIndex(middleChunkName)
+		if err != nil {
+			return nil, err
+		}
+		next, err := middleSecondary.Next()
+		if err != nil {
+			return nil, err
+		}
+		if next == nil {
+			break
+		}
+		startSlot := next.BlockOrEbb
+		var endSlot uint64
+		for {
+			next, err := middleSecondary.Next()
+			if err != nil {
+				return nil, err
+			}
+			if next == nil {
+				break
+			}
+			endSlot = next.BlockOrEbb
+		}
+		if point.Slot < startSlot {
+			// The slot we're looking for is less than the first slot in the chunk, so
+			// we can eliminate all later chunks
+			upperBound = middlePoint - 1
+		} else if point.Slot > endSlot {
+			// The slot we're looking for is greater than the last slot in the chunk, so
+			// we can eliminate all earlier chunks
+			lowerBound = middlePoint + 1
+		} else {
+			// We found the chunk that (probably) has the requested point
+			break
+		}
+	}
+	return chunkNames[lowerBound:], nil
+}
+
+func (i *ImmutableDb) getChunkPrimaryIndex(
+	chunkName string,
+) (*primaryIndex, error) {
+	primaryFilePath := filepath.Join(
+		i.dataDir,
+		chunkName+primaryFileExtension,
+	)
+	primary := newPrimaryIndex()
+	if err := primary.Open(primaryFilePath); err != nil {
+		return nil, fmt.Errorf(
+			"failed to read primary index: %s: %w",
+			primaryFilePath,
+			err,
+		)
+	}
+	return primary, nil
+}
+
+func (i *ImmutableDb) getChunkSecondaryIndex(
+	chunkName string,
+) (*secondaryIndex, error) {
+	primary, err := i.getChunkPrimaryIndex(chunkName)
+	if err != nil {
+		return nil, err
+	}
+	secondaryFilePath := filepath.Join(
+		i.dataDir,
+		chunkName+secondaryFileExtension,
+	)
+	secondary := newSecondaryIndex()
+	if err := secondary.Open(secondaryFilePath, primary); err != nil {
+		return nil, fmt.Errorf(
+			"failed to read secondary index: %s: %w",
+			secondaryFilePath,
+			err,
+		)
+	}
+	return secondary, nil
+}
+
+func (i *ImmutableDb) getChunk(chunkName string) (*chunk, error) {
+	// Open secondary index
+	secondary, err := i.getChunkSecondaryIndex(chunkName)
+	if err != nil {
+		return nil, err
+	}
+	// Open chunk
+	chunkFilePath := filepath.Join(
+		i.dataDir,
+		chunkName+chunkFileExtension,
+	)
+	chunk := newChunk()
+	if err := chunk.Open(chunkFilePath, secondary); err != nil {
+		return nil, fmt.Errorf(
+			"failed to read chunk: %s: %w",
+			chunkFilePath,
+			err,
+		)
+	}
+	return chunk, nil
+}
+
+func (i *ImmutableDb) GetTip() (*ocommon.Point, error) {
+	var ret *ocommon.Point
+	chunkNames, err := i.getChunkNames()
+	if err != nil {
+		return nil, err
+	}
+	if len(chunkNames) == 0 {
+		return nil, nil
+	}
+	secondary, err := i.getChunkSecondaryIndex(chunkNames[len(chunkNames)-1])
+	if err != nil {
+		return nil, err
+	}
+	var tmpPoint ocommon.Point
+	for {
+		next, err := secondary.Next()
+		if err != nil {
+			return nil, err
+		}
+		if next == nil {
+			break
+		}
+		tmpPoint = ocommon.NewPoint(
+			next.BlockOrEbb,
+			next.HeaderHash[:],
+		)
+		ret = &tmpPoint
+	}
+	return ret, nil
+}
+
+func (i *ImmutableDb) GetBlock(point ocommon.Point) (*Block, error) {
+	var err error
+	chunkNames, err := i.getChunkNamesFromPoint(point)
+	if err != nil {
+		return nil, err
+	}
+	chunk, err := i.getChunk(chunkNames[0])
+	if err != nil {
+		return nil, err
+	}
+	var tmpBlock *Block
+	for {
+		tmpBlock, err = chunk.Next()
+		if err != nil {
+			return nil, err
+		}
+		if tmpBlock == nil {
+			break
+		}
+		if tmpBlock.Slot != point.Slot {
+			continue
+		}
+		if string(tmpBlock.Hash) != string(point.Hash) {
+			continue
+		}
+		return tmpBlock, nil
+	}
+	return nil, nil
+}
+
+func (i *ImmutableDb) TruncateChunksFromPoint(point ocommon.Point) error {
+	chunkNames, err := i.getChunkNamesFromPoint(point)
+	if err != nil {
+		return err
+	}
+	for _, chunkName := range chunkNames {
+		chunkPathPrefix := filepath.Join(
+			i.dataDir,
+			chunkName,
+		)
+		if err := os.Remove(chunkPathPrefix + chunkFileExtension); err != nil {
+			return err
+		}
+		if err := os.Remove(chunkPathPrefix + secondaryFileExtension); err != nil {
+			return err
+		}
+		if err := os.Remove(chunkPathPrefix + primaryFileExtension); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (i *ImmutableDb) BlocksFromPoint(
+	point ocommon.Point,
+) (*BlockIterator, error) {
+	chunkNames, err := i.getChunkNamesFromPoint(point)
+	if err != nil {
+		return nil, err
+	}
+	ret := &BlockIterator{
+		db:         i,
+		chunkNames: chunkNames[:],
+		startPoint: point,
+	}
+	return ret, nil
+}
+
+type BlockIterator struct {
+	db              *ImmutableDb
+	startPoint      ocommon.Point
+	foundStartPoint bool
+	chunkNames      []string
+	chunkIdx        int
+	chunk           *chunk
+}
+
+func (b *BlockIterator) Next() (*Block, error) {
+	var err error
+	var tmpChunk *chunk
+	if b.chunk == nil {
+		if b.chunkIdx == 0 && len(b.chunkNames) > 0 {
+			// Open initial chunk
+			tmpChunk, err = b.db.getChunk(b.chunkNames[b.chunkIdx])
+			if err != nil {
+				return nil, err
+			}
+			b.chunk = tmpChunk
+		} else {
+			return nil, nil
+		}
+	}
+	var tmpBlock *Block
+	for {
+		tmpBlock, err = b.chunk.Next()
+		if err != nil {
+			return nil, err
+		}
+		if tmpBlock == nil {
+			// We've reached the end of the current chunk
+			if err := b.chunk.Close(); err != nil {
+				return nil, err
+			}
+			b.chunk = nil
+			b.chunkIdx++
+			if b.chunkIdx >= len(b.chunkNames) {
+				return nil, nil
+			}
+			tmpChunk, err = b.db.getChunk(b.chunkNames[b.chunkIdx])
+			if err != nil {
+				return nil, err
+			}
+			b.chunk = tmpChunk
+			continue
+		}
+		if !b.foundStartPoint {
+			if tmpBlock.Slot < b.startPoint.Slot {
+				continue
+			}
+			b.foundStartPoint = true
+		}
+		return tmpBlock, nil
+	}
+}
+
+func (b *BlockIterator) Close() error {
+	if b.chunk != nil {
+		if err := b.chunk.Close(); err != nil {
+			return err
+		}
+	}
+	return nil
+}

package/database/immutable/immutable_test.go

@@ -0,0 +1,59 @@
+// Copyright 2024 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package immutable_test
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/blinklabs-io/dingo/database/immutable"
+)
+
+const (
+	testDataDir = "./testdata"
+)
+
+func TestGetTip(t *testing.T) {
+	// These expected values correspond to the last block in our test data
+	var expectedSlot uint64 = 38426380
+	expectedHash := "7ada6ed78f6caa499370da6548b143c59320f5e5283e5e80e202a994ba7bfebf"
+	imm, err := immutable.New(testDataDir)
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	tip, err := imm.GetTip()
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err)
+	}
+	if tip == nil {
+		t.Fatalf("did not get expected tip value, got nil instead")
+	}
+	if tip.Slot != expectedSlot {
+		t.Fatalf(
+			"did not get expected slot value: expected %d, got %d",
+			expectedSlot,
+			tip.Slot,
+		)
+	}
+	if hex.EncodeToString(tip.Hash) != expectedHash {
+		t.Fatalf(
+			"did not get expected hash value: expected %s, got %x",
+			expectedHash,
+			tip.Hash,
+		)
+	}
+}
+
+// TODO: add tests for getting a specific block and getting a range of blocks that traverses multiple chunks (#386)

package/database/immutable/primary.go

@@ -0,0 +1,106 @@
+// Copyright 2024 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package immutable
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+	"os"
+)
+
+const (
+	primaryFileExtension = ".primary"
+)
+
+type primaryIndex struct {
+	file            *os.File
+	slot            int
+	lastOffset      uint32
+	version         uint8
+	seenFirstOffset bool
+}
+
+type primaryIndexEntry struct {
+	RelativeSlot    int
+	SecondaryOffset uint32
+	empty           bool
+}
+
+func newPrimaryIndex() *primaryIndex {
+	return &primaryIndex{}
+}
+
+func (e primaryIndexEntry) Empty() bool {
+	return e.empty
+}
+
+func (p *primaryIndex) Open(path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	p.file = f
+	// Read version
+	if err := binary.Read(f, binary.BigEndian, &p.version); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (p *primaryIndex) Close() error {
+	return p.file.Close()
+}
+
+func (p *primaryIndex) Next() (*primaryIndexEntry, error) {
+	var tmpOffset uint32
+	if err := binary.Read(p.file, binary.BigEndian, &tmpOffset); err != nil {
+		if errors.Is(err, io.EOF) {
+			// We've reached the end of the file
+			return nil, nil
+		}
+		return nil, err
+	}
+	empty := true
+	if tmpOffset > p.lastOffset || !p.seenFirstOffset {
+		empty = false
+		p.lastOffset = tmpOffset
+	}
+	tmpEntry := primaryIndexEntry{
+		RelativeSlot:    p.slot,
+		SecondaryOffset: tmpOffset,
+		empty:           empty,
+	}
+	p.slot++
+	p.seenFirstOffset = true
+	return &tmpEntry, nil
+}
+
+func (p *primaryIndex) NextOccupied() (*primaryIndexEntry, error) {
+	for {
+		next, err := p.Next()
+		if err != nil {
+			return nil, err
+		}
+		if next == nil {
+			break
+		}
+		if next.Empty() {
+			continue
+		}
+		return next, nil
+	}
+	return nil, nil
+}

package/database/immutable/secondary.go

@@ -0,0 +1,103 @@
+// Copyright 2024 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package immutable
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+)
+
+const (
+	secondaryFileExtension = ".secondary"
+)
+
+type secondaryIndex struct {
+	file     *os.File
+	fileSize int64
+	primary  *primaryIndex
+}
+
+type secondaryIndexEntryInner struct {
+	BlockOffset  uint64
+	HeaderOffset uint16
+	HeaderSize   uint16
+	Checksum     uint32
+	HeaderHash   [32]byte
+	BlockOrEbb   uint64
+}
+
+type secondaryIndexEntry struct {
+	secondaryIndexEntryInner
+	IsEbb bool
+}
+
+func newSecondaryIndex() *secondaryIndex {
+	return &secondaryIndex{}
+}
+
+func (s *secondaryIndex) Open(path string, primary *primaryIndex) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	s.file = f
+	s.primary = primary
+	if stat, err := f.Stat(); err != nil {
+		return err
+	} else {
+		s.fileSize = stat.Size()
+	}
+	return nil
+}
+
+func (s *secondaryIndex) Close() error {
+	if err := s.primary.Close(); err != nil {
+		return err
+	}
+	return s.file.Close()
+}
+
+func (s *secondaryIndex) Next() (*secondaryIndexEntry, error) {
+	nextOccupied, err := s.primary.NextOccupied()
+	if err != nil {
+		return nil, err
+	}
+	if nextOccupied == nil {
+		return nil, nil
+	}
+	// Look for final offset
+	if int64(nextOccupied.SecondaryOffset) == s.fileSize {
+		return nil, nil
+	}
+	// Seek to offset
+	if _, err := s.file.Seek(int64(nextOccupied.SecondaryOffset), 0); err != nil {
+		return nil, fmt.Errorf("failed while seeking: %w", err)
+	}
+	// Read entry
+	var tmpEntryInner secondaryIndexEntryInner
+	if err := binary.Read(s.file, binary.BigEndian, &tmpEntryInner); err != nil {
+		return nil, fmt.Errorf("failed while reading: %w", err)
+	}
+	ret := &secondaryIndexEntry{
+		secondaryIndexEntryInner: tmpEntryInner,
+	}
+	// Check for EBB
+	// A block with its secondary index in the first slot with a small BlockOrEbb value is *probably* an EBB
+	if nextOccupied.RelativeSlot == 0 && ret.BlockOrEbb < 200 {
+		ret.IsEbb = true
+	}
+	return ret, nil
+}

package/database/immutable/testdata/README.md

@@ -0,0 +1,4 @@
+# blockdb
+
+The files in this directory were pulled from the `immutable` directory of a Cardano node
+on the `preview` network

package/database/plugin/blob/badger/commit_timestamp.go

@@ -0,0 +1,50 @@
+// Copyright 2025 Blink Labs Software
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package badger
+
+import (
+	"math/big"
+
+	badger "github.com/dgraph-io/badger/v4"
+)
+
+const (
+	commitTimestampBlobKey = "metadata_commit_timestamp"
+)
+
+func (b *BlobStoreBadger) GetCommitTimestamp() (int64, error) {
+	txn := b.NewTransaction(false)
+	item, err := txn.Get([]byte(commitTimestampBlobKey))
+	if err != nil {
+		return 0, err
+	}
+	val, err := item.ValueCopy(nil)
+	if err != nil {
+		return 0, err
+	}
+	return new(big.Int).SetBytes(val).Int64(), nil
+}
+
+func (b *BlobStoreBadger) SetCommitTimestamp(
+	txn *badger.Txn,
+	timestamp int64,
+) error {
+	// Update badger
+	tmpTimestamp := new(big.Int).SetInt64(timestamp)
+	if err := txn.Set([]byte(commitTimestampBlobKey), tmpTimestamp.Bytes()); err != nil {
+		return err
+	}
+	return nil
+}