feedx 0.12.6 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab1b8d12efde331dcd1ca96c686c29e1ef16ab895f049121078b9f0503d23276
-  data.tar.gz: 3d51319d3ea39cbceeb480252e06c37238fce0cda68aa6b65cf1f59c0f0af38b
+  metadata.gz: c1978c7def650e2a2630e36fc0bfc59b9eb66c5ead0f7dda9cc83e773a85c1d8
+  data.tar.gz: 548208072baf9e6f84a7541df5e8a22a5960982007bce4385c7fa538e0290e94
 SHA512:
-  metadata.gz: d08c34b59158c69d500b7cffe8019550dafa87b760e4b0cb3b25afc367801a3f3473486987dcb62e5c1bc62c05d3653bf26692fcbdcbb552d2af1073f4eced50
-  data.tar.gz: cfa3f24458cead0c12c1f03b559ded816095e1e9cf29a1918c142cd255f782fa3c51d7c45ca5833ab25b5a745e0e68f4b1b1062c7760c56ff1b57da122767135
+  metadata.gz: a5b0be41feeafabbffcbbda6ccc1b12df4d24b8d9a9f03f4b8d04ce2fec7b644b99f15db98e67a79294dca1c3b56c072a67ab20b8cbffaed25ec502e74de9ca2
+  data.tar.gz: d3434d54accaf281ec5e06e25753de74a052d1386a63badf97ff8bfa8a1426162528d8256cbd288969ef1c348d24cf9358c0827b4c7b1af500c71e5349900b63

data/.github/workflows/test.yml

@@ -8,53 +8,6 @@ on:
       - main
 jobs:
   go:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go-version: [1.15.x, 1.16.x]
-    steps:
-      - name: Fetch staticcheck
-        uses: engineerd/configurator@v0.0.6
-        with:
-          name: staticcheck
-          fromGitHubReleases: true
-          repo: dominikh/go-tools
-          version: latest
-          urlTemplate: "https://github.com/dominikh/go-tools/releases/download/2020.2.3/staticcheck_linux_amd64.tar.gz"
-          pathInArchive: staticcheck/staticcheck
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Cache dependencies
-        uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-      - name: Setup Go
-        uses: actions/setup-go@v2
-        with:
-          go-version: ${{ matrix.go-version }}
-      - name: Run tests
-        run: make test
-      - name: Run staticcheck
-        run: make staticcheck
+    uses: bsm/misc/.github/workflows/test-go.yml@main
   ruby:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        ruby-version: ["2.6", "2.7", "3.0"]
-    steps:
-      - name: Install libarrow
-        run: |
-          wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
-          sudo apt install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
-          sudo apt update
-          sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
-      - uses: actions/checkout@v2
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby-version }}
-          bundler-cache: true
-      - run: bundle exec rake
+    uses: bsm/misc/.github/workflows/test-ruby.yml@main

data/.golangci.yml

@@ -0,0 +1,13 @@
+version: "2"
+linters:
+  settings:
+    errcheck:
+      exclude-functions:
+        - (*github.com/bsm/feedx.Writer).Discard
+        - (*github.com/bsm/feedx.Reader).Close
+        - (github.com/bsm/feedx.Consumer).Close
+        - (*github.com/bsm/feedx.IncrementalProducer).Close
+        - (*github.com/bsm/feedx.Producer).Close
+        - (*github.com/bsm/bfs.InMem).Close
+        - (*github.com/bsm/bfs.Object).Close
+        - (github.com/bsm/bfs.Writer).Discard

data/.rubocop.yml

@@ -1,17 +1,11 @@
-inherit_gem:
-  rubocop-bsm:
-    - default.yml
+plugins:
+  - rubocop-rake
+  - rubocop-rspec
+
+inherit_from:
+  - https://raw.githubusercontent.com/bsm/misc/refs/heads/main/rubocop/default.yml
+  - https://raw.githubusercontent.com/bsm/misc/refs/heads/main/rubocop/rspec.yml
+
 inherit_mode:
   merge:
     - Exclude
-
-AllCops:
-  TargetRubyVersion: 2.6
-Naming/MethodParameterName:
-  MinNameLength: 2
-Naming/MemoizedInstanceVariableName:
-  Enabled: false
-Naming/FileName:
-  Exclude: [lib/sortable-by.rb]
-Metrics/ParameterLists:
-  Max: 8

data/.tool-versions

@@ -0,0 +1 @@
+ruby 3.4.7

data/Gemfile

@@ -1,2 +1,10 @@
 source 'https://rubygems.org'
+
 gemspec
+gem 'bundler'
+gem 'pbio'
+gem 'rake'
+gem 'rspec'
+gem 'rubocop'
+gem 'rubocop-rake'
+gem 'rubocop-rspec'

data/Gemfile.lock

@@ -1,83 +1,67 @@
 PATH
   remote: .
   specs:
-    feedx (0.12.6)
+    feedx (0.14.0)
       bfs (>= 0.8.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.2)
-    bfs (0.8.4)
-    bigdecimal (3.0.2)
-    diff-lcs (1.4.4)
-    extpp (0.0.9)
-    gio2 (3.4.4)
-      gobject-introspection (= 3.4.4)
-    glib2 (3.4.4)
-      native-package-installer (>= 1.0.3)
-      pkg-config (>= 1.3.5)
-    gobject-introspection (3.4.4)
-      glib2 (= 3.4.4)
-    google-protobuf (3.17.3)
-    native-package-installer (1.1.1)
-    parallel (1.20.1)
-    parser (3.0.1.1)
+    ast (2.4.3)
+    bfs (0.9.1)
+    diff-lcs (1.6.2)
+    google-protobuf (3.25.8)
+    json (2.16.0)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    parallel (1.27.0)
+    parser (3.3.10.0)
       ast (~> 2.4.1)
-    pbio (0.3.0)
-      google-protobuf
-    pkg-config (1.4.6)
-    rainbow (3.0.0)
-    rake (13.0.3)
-    red-arrow (3.0.0)
-      bigdecimal (>= 2.0.3)
-      extpp (>= 0.0.7)
-      gio2 (>= 3.3.6)
-      native-package-installer
-      pkg-config
-    red-parquet (3.0.0)
-      red-arrow (= 3.0.0)
-    regexp_parser (2.1.1)
-    rexml (3.2.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+      racc
+    pbio (0.4.0)
+      google-protobuf (> 3, < 4)
+    prism (1.6.0)
+    racc (1.8.1)
+    rainbow (3.1.1)
+    rake (13.3.1)
+    regexp_parser (2.11.3)
+    rspec (3.13.2)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.6)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.7)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.17.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.6)
+    rubocop (1.81.7)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.7.0, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.47.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.7.0)
-      parser (>= 3.0.1.1)
-    rubocop-bsm (0.6.0)
-      rubocop (~> 1.0)
-      rubocop-performance
-      rubocop-rake
-      rubocop-rspec
-    rubocop-performance (1.11.3)
-      rubocop (>= 1.7.0, < 2.0)
-      rubocop-ast (>= 0.4.0)
-    rubocop-rake (0.5.1)
-      rubocop
-    rubocop-rspec (2.4.0)
-      rubocop (~> 1.0)
-      rubocop-ast (>= 1.1.0)
-    ruby-progressbar (1.11.0)
-    unicode-display_width (2.0.0)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.48.0)
+      parser (>= 3.3.7.2)
+      prism (~> 1.4)
+    rubocop-rake (0.7.1)
+      lint_roller (~> 1.1)
+      rubocop (>= 1.72.1)
+    rubocop-rspec (3.7.0)
+      lint_roller (~> 1.1)
+      rubocop (~> 1.72, >= 1.72.1)
+    ruby-progressbar (1.13.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.1.0)
 
 PLATFORMS
   ruby
@@ -87,9 +71,10 @@ DEPENDENCIES
   feedx!
   pbio
   rake
-  red-parquet (>= 3.0, < 4.0)
   rspec
-  rubocop-bsm
+  rubocop
+  rubocop-rake
+  rubocop-rspec
 
 BUNDLED WITH
-   2.1.4
+   2.3.16

data/Makefile

@@ -1,9 +1,9 @@
 default: test
 
-.common.makefile:
-	curl -fsSL -o $@ https://gitlab.com/bsm/misc/raw/master/make/go/common.makefile
+.minimal.makefile:
+	curl -fsSL -o $@ https://gitlab.com/bsm/misc/raw/master/make/go/minimal.makefile
 
-include .common.makefile
+include .minimal.makefile
 
 proto: internal/testdata/testdata.pb.go
 

data/README.md

@@ -1,6 +1,8 @@
 # Feedx
 
-[![Test](https://github.com/bsm/feedx/actions/workflows/test.yml/badge.svg)](https://github.com/bsm/feedx/actions/workflows/test.yml)
+[![Go reference](https://pkg.go.dev/badge/github.com/bsm/feedx.svg)](https://pkg.go.dev/github.com/bsm/feedx)
+[![GitHub release](https://img.shields.io/github/tag/bsm/feedx.svg?label=release)](https://github.com/bsm/feedx/releases)
+[![Go Report Card](https://goreportcard.com/badge/github.com/bsm/feedx)](https://goreportcard.com/report/github.com/bsm/feedx)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 Feed-based data exchange between services.

data/compression.go

@@ -5,6 +5,8 @@ import (
 	"compress/gzip"
 	"io"
 	"path"
+
+	"github.com/klauspost/compress/zstd"
 )
 
 // Compression represents the data compression.
@@ -23,6 +25,8 @@ func DetectCompression(name string) Compression {
 			return GZipCompression
 		} else if ext == ".flate" {
 			return FlateCompression
+		} else if ext == ".zst" {
+			return ZstdCompression
 		}
 	}
 	return NoCompression
@@ -79,3 +83,28 @@ func (flateCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
 func (flateCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
 	return flate.NewWriter(w, flate.BestSpeed)
 }
+
+// --------------------------------------------------------------------
+
+// ZstdCompression supports zstd compression format.
+var ZstdCompression = zstdCompression{}
+
+type zstdCompression struct{}
+
+func (zstdCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
+	zr, err := zstd.NewReader(r)
+	if err != nil {
+		return nil, err
+	}
+	return zstdDecoder{Decoder: zr}, nil
+}
+
+func (zstdCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
+	return zstd.NewWriter(w)
+}
+
+type zstdDecoder struct {
+	*zstd.Decoder
+}
+
+func (zstdDecoder) Close() error { return nil }

data/compression_test.go

@@ -2,77 +2,89 @@ package feedx_test
 
 import (
 	"bytes"
+	"testing"
 
 	"github.com/bsm/feedx"
-	. "github.com/bsm/ginkgo"
-	. "github.com/bsm/gomega"
 )
 
-var _ = Describe("Compression", func() {
-	var data = bytes.Repeat([]byte("wxyz"), 1024)
-
-	runSharedTest := func(subject feedx.Compression) {
-		buf := new(bytes.Buffer)
-
-		w, err := subject.NewWriter(buf)
-		Expect(err).NotTo(HaveOccurred())
-		defer w.Close()
-
-		Expect(w.Write(data)).To(Equal(4096))
-		Expect(w.Write(data)).To(Equal(4096))
-		Expect(w.Close()).To(Succeed())
-
-		r, err := subject.NewReader(buf)
-		Expect(err).NotTo(HaveOccurred())
-		defer r.Close()
-
-		p := make([]byte, 20)
-		Expect(r.Read(p)).To(Equal(20))
-		Expect(string(p)).To(Equal("wxyzwxyzwxyzwxyzwxyz"))
-		Expect(r.Close()).To(Succeed())
+func TestDetectCompression(t *testing.T) {
+	examples := []struct {
+		Input string
+		Exp   feedx.Compression
+	}{
+		{Input: "/path/to/file.json", Exp: feedx.NoCompression},
+		{Input: "/path/to/file.json.gz", Exp: feedx.GZipCompression},
+		{Input: "/path/to/file.jsonz", Exp: feedx.GZipCompression},
+		{Input: "/path/to/file.pb", Exp: feedx.NoCompression},
+		{Input: "/path/to/file.pb.gz", Exp: feedx.GZipCompression},
+		{Input: "/path/to/file.pbz", Exp: feedx.GZipCompression},
+		{Input: "/path/to/file.flate", Exp: feedx.FlateCompression},
+		{Input: "/path/to/file.whatever.flate", Exp: feedx.FlateCompression},
+		{Input: "/path/to/file.zst", Exp: feedx.ZstdCompression},
+		{Input: "", Exp: feedx.NoCompression},
+		{Input: "/path/to/file", Exp: feedx.NoCompression},
+		{Input: "/path/to/file.txt", Exp: feedx.NoCompression},
 	}
+	for _, x := range examples {
+		if got := feedx.DetectCompression(x.Input); got != x.Exp {
+			t.Errorf("expected %s for %q, but got %s", x.Exp, x.Input, got)
+		}
+	}
+}
 
-	It("detects the format", func() {
-		Expect(feedx.DetectCompression("/path/to/file.json")).To(Equal(feedx.NoCompression))
-		Expect(feedx.DetectCompression("/path/to/file.json.gz")).To(Equal(feedx.GZipCompression))
-		Expect(feedx.DetectCompression("/path/to/file.jsonz")).To(Equal(feedx.GZipCompression))
-
-		Expect(feedx.DetectCompression("/path/to/file.pb")).To(Equal(feedx.NoCompression))
-		Expect(feedx.DetectCompression("/path/to/file.pb.gz")).To(Equal(feedx.GZipCompression))
-		Expect(feedx.DetectCompression("/path/to/file.pbz")).To(Equal(feedx.GZipCompression))
-
-		Expect(feedx.DetectCompression("/path/to/file.flate")).To(Equal(feedx.FlateCompression))
-		Expect(feedx.DetectCompression("/path/to/file.whatever.flate")).To(Equal(feedx.FlateCompression))
+func TestCompression(t *testing.T) {
+	data := bytes.Repeat([]byte("wxyz"), 1024)
 
-		Expect(feedx.DetectCompression("")).To(Equal(feedx.NoCompression))
-		Expect(feedx.DetectCompression("/path/to/file")).To(Equal(feedx.NoCompression))
-		Expect(feedx.DetectCompression("/path/to/file.txt")).To(Equal(feedx.NoCompression))
+	t.Run("no compression", func(t *testing.T) {
+		testCompression(t, feedx.NoCompression, data)
 	})
-
-	Describe("NoCompression", func() {
-		var subject = feedx.NoCompression
-		var _ feedx.Compression = subject
-
-		It("writes/reads", func() {
-			runSharedTest(subject)
-		})
+	t.Run("gzip", func(t *testing.T) {
+		testCompression(t, feedx.GZipCompression, data)
+	})
+	t.Run("flate", func(t *testing.T) {
+		testCompression(t, feedx.FlateCompression, data)
 	})
+	t.Run("zstd", func(t *testing.T) {
+		testCompression(t, feedx.ZstdCompression, data)
+	})
+}
 
-	Describe("GZipCompression", func() {
-		var subject = feedx.GZipCompression
-		var _ feedx.Compression = subject
+func testCompression(t *testing.T, c feedx.Compression, data []byte) {
+	t.Helper()
+	buf := new(bytes.Buffer)
 
-		It("writes/reads", func() {
-			runSharedTest(subject)
-		})
-	})
+	w, err := c.NewWriter(buf)
+	if err != nil {
+		t.Fatal("expected no error, got", err)
+	}
+	defer func() { _ = w.Close() }()
 
-	Describe("FlateCompression", func() {
-		var subject = feedx.FlateCompression
-		var _ feedx.Compression = subject
+	if _, err := w.Write(data); err != nil {
+		t.Fatal("expected no error, got", err)
+	}
+	if _, err := w.Write(data); err != nil {
+		t.Fatal("expected no error, got", err)
+	}
+	if err := w.Close(); err != nil {
+		t.Fatal("expected no error, got", err)
+	}
 
-		It("writes/reads", func() {
-			runSharedTest(subject)
-		})
-	})
-})
+	r, err := c.NewReader(buf)
+	if err != nil {
+		t.Fatal("expected no error, got", err)
+	}
+	defer func() { _ = r.Close() }()
+
+	p := make([]byte, 20)
+	if n, err := r.Read(p); err != nil {
+		t.Fatal("expected no error, got", err)
+	} else if n != 20 {
+		t.Errorf("expected to read 20 bytes, got %d", n)
+	} else if exp, got := "wxyzwxyzwxyzwxyzwxyz", string(p); exp != got {
+		t.Errorf("expected %q, got %q", exp, got)
+	}
+
+	if err := r.Close(); err != nil {
+		t.Fatal("expected no error, got", err)
+	}
+}