feedx 0.12.2 → 0.12.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 898f80c359182952ab192f95fb05dc1e6ae806f0d661d71244813b779595d0f8
-  data.tar.gz: 43778c6084131e8ffa9ba59b7836fe198f7595e5dc07af1c4e11a9b03ac49419
+  metadata.gz: 6a8328c3af95657d103cf62048d13cb5a14cc50466351f2178082612c347a8cf
+  data.tar.gz: f2fdb20a577c88f165f2cbac3dadf217889716b67b9ae20e4941ed9ef396e864
 SHA512:
-  metadata.gz: fa8f2e4e26c140bb4c2732655ac73c39ef20055f215ac39c8c62bd77e7cf27d7d8140287ed6c90a4c4da6bf461d491123f76784ce5d770189a45da7fea897965
-  data.tar.gz: a35d5296c5aec452b450e95985535c98b6b99f1a872624b3ad3974cbb0f56b0e6d0ae8395edd849c97242ae0a77ee77537404cb3dce22882a3a310a6e0f04a04
+  metadata.gz: 6cf78b2fb0917c7afb33014e6b3e30d045ffca874f4c8463092f25a7b22b268ab9f2cbd03a1dfda9df7a9314fe213d11c332ff965eb7b81436768cb94d80bf83
+  data.tar.gz: 9131ba95738e600ab4e731546b6e876cae77120782e148a6eac6e3ea4b90e398975f15f2af00b325e86f3fc73a79c39c4a6be25e80bc7298881ff636d61d150b

data/.editorconfig

@@ -7,3 +7,6 @@ end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
+
+[{*.go,Makefile}]
+indent_style = tab

data/.github/workflows/lint.yml

@@ -0,0 +1,18 @@
+name: Lint
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  golangci:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Run lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest

data/.github/workflows/test.yml

@@ -0,0 +1,48 @@
+name: Test
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  go:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go-version: [1.16.x, 1.17.x]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Run tests
+        run: make test
+  ruby:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ["2.7", "3.0", "3.1"]
+    steps:
+      - name: Install libarrow
+        run: |
+          wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt update
+          sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+      - run: bundle exec rake

data/.gitignore

@@ -1,3 +1,4 @@
 .rubocop-*
 pkg/
 *~
+*.makefile

data/.golangci.yml

@@ -0,0 +1,4 @@
+linters-settings:
+  errcheck:
+    exclude-functions:
+      - (*github.com/bsm/feedx.Writer).Discard

data/.rubocop.yml

@@ -1,8 +1,17 @@
-require: rubocop-performance
-inherit_from:
-  - https://gitlab.com/bsm/misc/raw/master/rubocop/default.yml
+inherit_gem:
+  rubocop-bsm:
+    - default.yml
+inherit_mode:
+  merge:
+    - Exclude
 
 AllCops:
-  TargetRubyVersion: "2.4"
+  TargetRubyVersion: 2.7
+Naming/MethodParameterName:
+  MinNameLength: 2
+Naming/MemoizedInstanceVariableName:
+  Enabled: false
+Naming/FileName:
+  Exclude: [lib/sortable-by.rb]
 Metrics/ParameterLists:
-  Max: 10
+  Max: 8

data/Gemfile

@@ -1,4 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-
-gem 'google-protobuf', '>= 3.7.0-rc2'

data/Gemfile.lock

@@ -1,70 +1,82 @@
 PATH
   remote: .
   specs:
-    feedx (0.12.2)
-      bfs (>= 0.5.0)
+    feedx (0.12.7)
+      bfs (>= 0.8.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.1)
-    bfs (0.7.2)
-    diff-lcs (1.4.4)
-    extpp (0.0.8)
-    gio2 (3.4.3)
-      gobject-introspection (= 3.4.3)
-    glib2 (3.4.3)
+    ast (2.4.2)
+    bfs (0.9.0)
+    bigdecimal (3.1.1)
+    diff-lcs (1.5.0)
+    extpp (0.1.0)
+    gio2 (3.4.9)
+      gobject-introspection (= 3.4.9)
+    glib2 (3.4.9)
       native-package-installer (>= 1.0.3)
       pkg-config (>= 1.3.5)
-    gobject-introspection (3.4.3)
-      glib2 (= 3.4.3)
-    google-protobuf (3.12.2)
-    native-package-installer (1.0.9)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    gobject-introspection (3.4.9)
+      glib2 (= 3.4.9)
+    google-protobuf (3.19.2)
+    native-package-installer (1.1.1)
+    parallel (1.21.0)
+    parser (3.1.0.0)
       ast (~> 2.4.1)
-    pbio (0.2.1)
+    pbio (0.3.0)
       google-protobuf
-    pkg-config (1.4.1)
+    pkg-config (1.4.6)
     rainbow (3.0.0)
-    rake (13.0.1)
-    red-arrow (0.17.1)
+    rake (13.0.6)
+    red-arrow (3.0.0)
+      bigdecimal (>= 2.0.3)
       extpp (>= 0.0.7)
       gio2 (>= 3.3.6)
       native-package-installer
       pkg-config
-    red-parquet (0.17.1)
-      red-arrow (= 0.17.1)
-    regexp_parser (1.7.1)
-    rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    red-parquet (3.0.0)
+      red-arrow (= 3.0.0)
+    regexp_parser (2.2.0)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rubocop (0.86.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.3)
+    rubocop (1.24.1)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.0.3, < 1.0)
+      rubocop-ast (>= 1.15.1, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.1.0)
-      parser (>= 2.7.0.1)
-    rubocop-performance (1.6.1)
-      rubocop (>= 0.71.0)
-    ruby-progressbar (1.10.1)
-    unicode-display_width (1.7.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.15.1)
+      parser (>= 3.0.1.1)
+    rubocop-bsm (0.6.0)
+      rubocop (~> 1.0)
+      rubocop-performance
+      rubocop-rake
+      rubocop-rspec
+    rubocop-performance (1.13.1)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rake (0.6.0)
+      rubocop (~> 1.0)
+    rubocop-rspec (2.7.0)
+      rubocop (~> 1.19)
+    ruby-progressbar (1.11.0)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
   ruby
@@ -72,13 +84,11 @@ PLATFORMS
 DEPENDENCIES
   bundler
   feedx!
-  google-protobuf (>= 3.7.0.pre.rc2)
   pbio
   rake
-  red-parquet
+  red-parquet (>= 3.0, < 4.0)
   rspec
-  rubocop
-  rubocop-performance
+  rubocop-bsm
 
 BUNDLED WITH
-   2.1.4
+   2.2.27

data/Makefile

@@ -1,12 +1,12 @@
-default: vet test
+default: test
 
-test:
-	go test ./...
+.common.makefile:
+	curl -fsSL -o $@ https://gitlab.com/bsm/misc/raw/master/make/go/common.makefile
 
-vet:
-	go vet ./...
+include .common.makefile
 
 proto: internal/testdata/testdata.pb.go
 
 %.pb.go: %.proto
-	protoc -I=. --gogo_out=paths=source_relative:. $<
+	# may need to `go install google.golang.org/protobuf/cmd/protoc-gen-go`
+	protoc -I=. --go_out=paths=source_relative:. $<

data/README.md

@@ -1,6 +1,6 @@
 # Feedx
 
-[![Build Status](https://travis-ci.org/bsm/feedx.png?branch=master)](https://travis-ci.org/bsm/feedx)
+[![Test](https://github.com/bsm/feedx/actions/workflows/test.yml/badge.svg)](https://github.com/bsm/feedx/actions/workflows/test.yml)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 Feed-based data exchange between services.

data/compression.go

@@ -1,6 +1,7 @@
 package feedx
 
 import (
+	"compress/flate"
 	"compress/gzip"
 	"io"
 	"path"
@@ -20,6 +21,8 @@ func DetectCompression(name string) Compression {
 		ext := path.Ext(path.Base(name))
 		if ext != "" && ext[0] == '.' && ext[len(ext)-1] == 'z' {
 			return GZipCompression
+		} else if ext == ".flate" {
+			return FlateCompression
 		}
 	}
 	return NoCompression
@@ -61,3 +64,18 @@ func (gzipCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
 func (gzipCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
 	return gzip.NewWriter(w), nil
 }
+
+// --------------------------------------------------------------------
+
+// FlateCompression supports flate compression format.
+var FlateCompression = flateCompression{}
+
+type flateCompression struct{}
+
+func (flateCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
+	return flate.NewReader(r), nil
+}
+
+func (flateCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
+	return flate.NewWriter(w, flate.BestSpeed)
+}

data/compression_test.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 
 	"github.com/bsm/feedx"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Compression", func() {
@@ -32,7 +32,7 @@ var _ = Describe("Compression", func() {
 		Expect(r.Close()).To(Succeed())
 	}
 
-	It("should detect the format", func() {
+	It("detects the format", func() {
 		Expect(feedx.DetectCompression("/path/to/file.json")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file.json.gz")).To(Equal(feedx.GZipCompression))
 		Expect(feedx.DetectCompression("/path/to/file.jsonz")).To(Equal(feedx.GZipCompression))
@@ -41,6 +41,9 @@ var _ = Describe("Compression", func() {
 		Expect(feedx.DetectCompression("/path/to/file.pb.gz")).To(Equal(feedx.GZipCompression))
 		Expect(feedx.DetectCompression("/path/to/file.pbz")).To(Equal(feedx.GZipCompression))
 
+		Expect(feedx.DetectCompression("/path/to/file.flate")).To(Equal(feedx.FlateCompression))
+		Expect(feedx.DetectCompression("/path/to/file.whatever.flate")).To(Equal(feedx.FlateCompression))
+
 		Expect(feedx.DetectCompression("")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file.txt")).To(Equal(feedx.NoCompression))
@@ -50,7 +53,7 @@ var _ = Describe("Compression", func() {
 		var subject = feedx.NoCompression
 		var _ feedx.Compression = subject
 
-		It("should write/read", func() {
+		It("writes/reads", func() {
 			runSharedTest(subject)
 		})
 	})
@@ -59,7 +62,16 @@ var _ = Describe("Compression", func() {
 		var subject = feedx.GZipCompression
 		var _ feedx.Compression = subject
 
-		It("should write/read", func() {
+		It("writes/reads", func() {
+			runSharedTest(subject)
+		})
+	})
+
+	Describe("FlateCompression", func() {
+		var subject = feedx.FlateCompression
+		var _ feedx.Compression = subject
+
+		It("writes/reads", func() {
 			runSharedTest(subject)
 		})
 	})

data/consumer.go

@@ -50,6 +50,8 @@ type Consumer interface {
 	Data() interface{}
 	// LastSync returns time of last sync attempt.
 	LastSync() time.Time
+	// LastConsumed returns time of last feed consumption.
+	LastConsumed() time.Time
 	// LastModified returns time at which the remote feed was last modified.
 	LastModified() time.Time
 	// NumRead returns the number of values consumed during the last sync.
@@ -114,7 +116,7 @@ type consumer struct {
 	cfn  ConsumeFunc
 	data atomic.Value
 
-	numRead, lastMod, lastSync int64
+	numRead, lastMod, lastSync, lastConsumed int64
 }
 
 // Data implements Consumer interface.
@@ -132,6 +134,11 @@ func (c *consumer) LastSync() time.Time {
 	return timestamp(atomic.LoadInt64(&c.lastSync)).Time()
 }
 
+// LastConsumed implements Consumer interface.
+func (c *consumer) LastConsumed() time.Time {
+	return timestamp(atomic.LoadInt64(&c.lastConsumed)).Time()
+}
+
 // LastModified implements Consumer interface.
 func (c *consumer) LastModified() time.Time {
 	return timestamp(atomic.LoadInt64(&c.lastMod)).Time()
@@ -147,8 +154,9 @@ func (c *consumer) Close() error {
 }
 
 func (c *consumer) sync(force bool) (*ConsumerSync, error) {
+	syncTime := timestampFromTime(time.Now()).Millis()
 	defer func() {
-		atomic.StoreInt64(&c.lastSync, timestampFromTime(time.Now()).Millis())
+		atomic.StoreInt64(&c.lastSync, syncTime)
 	}()
 
 	// retrieve original last modified time
@@ -158,7 +166,7 @@ func (c *consumer) sync(force bool) (*ConsumerSync, error) {
 	}
 
 	// skip update if not forced or modified
-	if lastMod.Millis() == atomic.LoadInt64(&c.lastMod) && !force {
+	if !force && lastMod > 0 && lastMod.Millis() == atomic.LoadInt64(&c.lastMod) {
 		return &ConsumerSync{Consumer: c}, nil
 	}
 
@@ -180,6 +188,7 @@ func (c *consumer) sync(force bool) (*ConsumerSync, error) {
 	c.data.Store(data)
 	atomic.StoreInt64(&c.numRead, int64(reader.NumRead()))
 	atomic.StoreInt64(&c.lastMod, lastMod.Millis())
+	atomic.StoreInt64(&c.lastConsumed, syncTime)
 	return &ConsumerSync{
 		Consumer:     c,
 		Updated:      true,

data/consumer_test.go

@@ -8,8 +8,8 @@ import (
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
 	"github.com/bsm/feedx/internal/testdata"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Consumer", func() {
@@ -17,25 +17,26 @@ var _ = Describe("Consumer", func() {
 	var obj *bfs.Object
 	var ctx = context.Background()
 
+	consume := func(r *feedx.Reader) (interface{}, error) {
+		var msgs []*testdata.MockMessage
+		for {
+			var msg testdata.MockMessage
+			if err := r.Decode(&msg); err == io.EOF {
+				break
+			} else if err != nil {
+				return nil, err
+			}
+			msgs = append(msgs, &msg)
+		}
+		return msgs, nil
+	}
+
 	BeforeEach(func() {
 		obj = bfs.NewInMemObject("path/to/file.jsonz")
-		Expect(writeMulti(obj, 2)).To(Succeed())
+		Expect(writeMulti(obj, 2, mockTime)).To(Succeed())
 
 		var err error
-		subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
-			var msgs []*testdata.MockMessage
-			for {
-				var msg testdata.MockMessage
-				if err := r.Decode(&msg); err == io.EOF {
-					break
-				}
-				if err != nil {
-					return nil, err
-				}
-				msgs = append(msgs, &msg)
-			}
-			return msgs, nil
-		})
+		subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, consume)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
@@ -43,11 +44,41 @@ var _ = Describe("Consumer", func() {
 		Expect(subject.Close()).To(Succeed())
 	})
 
-	It("should sync and retrieve feeds from remote", func() {
+	It("syncs/retrieves feeds from remote", func() {
 		Expect(subject.LastSync()).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.LastModified()).To(BeTemporally("~", time.Unix(1515151515, 0), time.Second))
+		Expect(subject.LastConsumed()).To(BeTemporally("==", subject.LastSync()))
+		Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
 		Expect(subject.NumRead()).To(Equal(2))
 		Expect(subject.Data()).To(ConsistOf(seed(), seed()))
 		Expect(subject.Close()).To(Succeed())
 	})
+
+	It("consumes feeds only if necessary", func() {
+		prevSync := subject.LastSync()
+		time.Sleep(2 * time.Millisecond)
+
+		testable := subject.(interface{ TestSync() error })
+		Expect(testable.TestSync()).To(Succeed())
+		Expect(subject.LastSync()).To(BeTemporally(">", prevSync))
+		Expect(subject.LastConsumed()).To(BeTemporally("==", prevSync)) // skipped on last sync
+		Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
+		Expect(subject.NumRead()).To(Equal(2))
+	})
+
+	It("always consumes if LastModified not set", func() {
+		noModTime := bfs.NewInMemObject("path/to/file.json")
+		Expect(writeMulti(noModTime, 2, time.Time{})).To(Succeed())
+
+		csmr, err := feedx.NewConsumerForRemote(ctx, noModTime, nil, consume)
+		Expect(err).NotTo(HaveOccurred())
+
+		prevSync := csmr.LastSync()
+		time.Sleep(2 * time.Millisecond)
+
+		testable := csmr.(interface{ TestSync() error })
+		Expect(testable.TestSync()).To(Succeed())
+		Expect(csmr.LastSync()).To(BeTemporally(">", prevSync))
+		Expect(csmr.LastConsumed()).To(BeTemporally("==", csmr.LastSync())) // consumed on last sync
+		Expect(csmr.LastModified()).To(BeTemporally("==", time.Unix(0, 0)))
+	})
 })

data/ext/parquet/decoder.go

@@ -0,0 +1,59 @@
+package parquet
+
+import (
+	"io"
+
+	goparquet "github.com/fraugster/parquet-go"
+	"github.com/fraugster/parquet-go/floor"
+)
+
+type decoder struct {
+	pfr *goparquet.FileReader
+	ffr *floor.Reader
+	tmp *tempFile
+}
+
+func newDecoder(rs io.ReadSeeker) (*decoder, error) {
+	pfr, err := goparquet.NewFileReader(rs)
+	if err != nil {
+		return nil, err
+	}
+
+	ffr := floor.NewReader(pfr)
+
+	return &decoder{
+			pfr: pfr,
+			ffr: ffr,
+		},
+		nil
+}
+
+func (w *decoder) Decode(v interface{}) error {
+	// read the next value and scan
+	if w.ffr.Next() {
+		return w.ffr.Scan(v)
+	}
+
+	// check for errors
+	if err := w.ffr.Err(); err != nil {
+		return err
+	}
+
+	// end of file
+	return io.EOF
+}
+
+func (w *decoder) Close() (err error) {
+	// close the tmp file if present
+	if w.tmp != nil {
+		if e := w.tmp.Close(); e != nil {
+			err = e
+		}
+	}
+
+	// close the reader
+	if e := w.ffr.Close(); e != nil {
+		err = e
+	}
+	return
+}