feedx 0.12.0 → 0.12.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 78c9ab1f475804897ee6610107515f598c14590878a3134f14abecad23e151ab
-  data.tar.gz: 89d4d70a336b47ccba2db349fdcc4283e45ddb877599ac6618ba3da173ba8547
+  metadata.gz: ab1b8d12efde331dcd1ca96c686c29e1ef16ab895f049121078b9f0503d23276
+  data.tar.gz: 3d51319d3ea39cbceeb480252e06c37238fce0cda68aa6b65cf1f59c0f0af38b
 SHA512:
-  metadata.gz: 12a539f376dd128eb4eb9d26d7a31812f656c0601375eec8721c814f404326d52feaafa80c6856fc9cf7fe66f1e1e956b9c74621d3ee88f0f8a47307f2ef04c1
-  data.tar.gz: c1de172489a3404f3c811aa3c41bf6d00708f36e756f4db0d4e58db0adfe8aa3908189cc07a25703682c48293c747351781650582ca4f1f1e5d3ab5681237f7c
+  metadata.gz: d08c34b59158c69d500b7cffe8019550dafa87b760e4b0cb3b25afc367801a3f3473486987dcb62e5c1bc62c05d3653bf26692fcbdcbb552d2af1073f4eced50
+  data.tar.gz: cfa3f24458cead0c12c1f03b559ded816095e1e9cf29a1918c142cd255f782fa3c51d7c45ca5833ab25b5a745e0e68f4b1b1062c7760c56ff1b57da122767135

data/.editorconfig

@@ -7,3 +7,6 @@ end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
+
+[{*.go,Makefile}]
+indent_style = tab

data/.github/workflows/test.yml

@@ -0,0 +1,60 @@
+name: Test
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  go:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x]
+    steps:
+      - name: Fetch staticcheck
+        uses: engineerd/configurator@v0.0.6
+        with:
+          name: staticcheck
+          fromGitHubReleases: true
+          repo: dominikh/go-tools
+          version: latest
+          urlTemplate: "https://github.com/dominikh/go-tools/releases/download/2020.2.3/staticcheck_linux_amd64.tar.gz"
+          pathInArchive: staticcheck/staticcheck
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Run tests
+        run: make test
+      - name: Run staticcheck
+        run: make staticcheck
+  ruby:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ["2.6", "2.7", "3.0"]
+    steps:
+      - name: Install libarrow
+        run: |
+          wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
+          sudo apt install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
+          sudo apt update
+          sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+      - run: bundle exec rake

data/.gitignore

@@ -1,3 +1,4 @@
 .rubocop-*
 pkg/
 *~
+*.makefile

data/.rubocop.yml

@@ -1,8 +1,17 @@
-require: rubocop-performance
-inherit_from:
-  - https://gitlab.com/bsm/misc/raw/master/rubocop/default.yml
+inherit_gem:
+  rubocop-bsm:
+    - default.yml
+inherit_mode:
+  merge:
+    - Exclude
 
 AllCops:
-  TargetRubyVersion: "2.4"
+  TargetRubyVersion: 2.6
+Naming/MethodParameterName:
+  MinNameLength: 2
+Naming/MemoizedInstanceVariableName:
+  Enabled: false
+Naming/FileName:
+  Exclude: [lib/sortable-by.rb]
 Metrics/ParameterLists:
-  Max: 10
+  Max: 8

data/Gemfile

@@ -1,4 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-
-gem 'google-protobuf', '>= 3.7.0-rc2'

data/Gemfile.lock

@@ -1,70 +1,83 @@
 PATH
   remote: .
   specs:
-    feedx (0.12.0)
-      bfs (>= 0.5.0)
+    feedx (0.12.6)
+      bfs (>= 0.8.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.1)
-    bfs (0.7.0)
-    diff-lcs (1.4)
-    extpp (0.0.8)
-    gio2 (3.4.3)
-      gobject-introspection (= 3.4.3)
-    glib2 (3.4.3)
+    ast (2.4.2)
+    bfs (0.8.4)
+    bigdecimal (3.0.2)
+    diff-lcs (1.4.4)
+    extpp (0.0.9)
+    gio2 (3.4.4)
+      gobject-introspection (= 3.4.4)
+    glib2 (3.4.4)
       native-package-installer (>= 1.0.3)
       pkg-config (>= 1.3.5)
-    gobject-introspection (3.4.3)
-      glib2 (= 3.4.3)
-    google-protobuf (3.12.2)
-    native-package-installer (1.0.9)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    gobject-introspection (3.4.4)
+      glib2 (= 3.4.4)
+    google-protobuf (3.17.3)
+    native-package-installer (1.1.1)
+    parallel (1.20.1)
+    parser (3.0.1.1)
       ast (~> 2.4.1)
-    pbio (0.2.1)
+    pbio (0.3.0)
       google-protobuf
-    pkg-config (1.4.1)
+    pkg-config (1.4.6)
     rainbow (3.0.0)
-    rake (13.0.1)
-    red-arrow (0.17.1)
+    rake (13.0.3)
+    red-arrow (3.0.0)
+      bigdecimal (>= 2.0.3)
       extpp (>= 0.0.7)
       gio2 (>= 3.3.6)
       native-package-installer
       pkg-config
-    red-parquet (0.17.1)
-      red-arrow (= 0.17.1)
-    regexp_parser (1.7.1)
-    rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    red-parquet (3.0.0)
+      red-arrow (= 3.0.0)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rubocop (0.86.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.17.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.0.3, < 1.0)
+      rubocop-ast (>= 1.7.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.0.3)
-      parser (>= 2.7.0.1)
-    rubocop-performance (1.6.1)
-      rubocop (>= 0.71.0)
-    ruby-progressbar (1.10.1)
-    unicode-display_width (1.7.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.7.0)
+      parser (>= 3.0.1.1)
+    rubocop-bsm (0.6.0)
+      rubocop (~> 1.0)
+      rubocop-performance
+      rubocop-rake
+      rubocop-rspec
+    rubocop-performance (1.11.3)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rake (0.5.1)
+      rubocop
+    rubocop-rspec (2.4.0)
+      rubocop (~> 1.0)
+      rubocop-ast (>= 1.1.0)
+    ruby-progressbar (1.11.0)
+    unicode-display_width (2.0.0)
 
 PLATFORMS
   ruby
@@ -72,13 +85,11 @@ PLATFORMS
 DEPENDENCIES
   bundler
   feedx!
-  google-protobuf (>= 3.7.0.pre.rc2)
   pbio
   rake
-  red-parquet
+  red-parquet (>= 3.0, < 4.0)
   rspec
-  rubocop
-  rubocop-performance
+  rubocop-bsm
 
 BUNDLED WITH
    2.1.4

data/Makefile

@@ -1,12 +1,12 @@
-default: vet test
+default: test
 
-test:
-	go test ./...
+.common.makefile:
+	curl -fsSL -o $@ https://gitlab.com/bsm/misc/raw/master/make/go/common.makefile
 
-vet:
-	go vet ./...
+include .common.makefile
 
 proto: internal/testdata/testdata.pb.go
 
 %.pb.go: %.proto
-	protoc -I=. --gogo_out=paths=source_relative:. $<
+	# may need to `go install google.golang.org/protobuf/cmd/protoc-gen-go`
+	protoc -I=. --go_out=paths=source_relative:. $<

data/README.md

@@ -1,6 +1,6 @@
 # Feedx
 
-[![Build Status](https://travis-ci.org/bsm/feedx.png?branch=master)](https://travis-ci.org/bsm/feedx)
+[![Test](https://github.com/bsm/feedx/actions/workflows/test.yml/badge.svg)](https://github.com/bsm/feedx/actions/workflows/test.yml)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 Feed-based data exchange between services.

data/compression.go

@@ -1,6 +1,7 @@
 package feedx
 
 import (
+	"compress/flate"
 	"compress/gzip"
 	"io"
 	"path"
@@ -20,6 +21,8 @@ func DetectCompression(name string) Compression {
 		ext := path.Ext(path.Base(name))
 		if ext != "" && ext[0] == '.' && ext[len(ext)-1] == 'z' {
 			return GZipCompression
+		} else if ext == ".flate" {
+			return FlateCompression
 		}
 	}
 	return NoCompression
@@ -61,3 +64,18 @@ func (gzipCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
 func (gzipCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
 	return gzip.NewWriter(w), nil
 }
+
+// --------------------------------------------------------------------
+
+// FlateCompression supports flate compression format.
+var FlateCompression = flateCompression{}
+
+type flateCompression struct{}
+
+func (flateCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
+	return flate.NewReader(r), nil
+}
+
+func (flateCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
+	return flate.NewWriter(w, flate.BestSpeed)
+}

data/compression_test.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 
 	"github.com/bsm/feedx"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Compression", func() {
@@ -32,7 +32,7 @@ var _ = Describe("Compression", func() {
 		Expect(r.Close()).To(Succeed())
 	}
 
-	It("should detect the format", func() {
+	It("detects the format", func() {
 		Expect(feedx.DetectCompression("/path/to/file.json")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file.json.gz")).To(Equal(feedx.GZipCompression))
 		Expect(feedx.DetectCompression("/path/to/file.jsonz")).To(Equal(feedx.GZipCompression))
@@ -41,6 +41,9 @@ var _ = Describe("Compression", func() {
 		Expect(feedx.DetectCompression("/path/to/file.pb.gz")).To(Equal(feedx.GZipCompression))
 		Expect(feedx.DetectCompression("/path/to/file.pbz")).To(Equal(feedx.GZipCompression))
 
+		Expect(feedx.DetectCompression("/path/to/file.flate")).To(Equal(feedx.FlateCompression))
+		Expect(feedx.DetectCompression("/path/to/file.whatever.flate")).To(Equal(feedx.FlateCompression))
+
 		Expect(feedx.DetectCompression("")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file.txt")).To(Equal(feedx.NoCompression))
@@ -50,7 +53,7 @@ var _ = Describe("Compression", func() {
 		var subject = feedx.NoCompression
 		var _ feedx.Compression = subject
 
-		It("should write/read", func() {
+		It("writes/reads", func() {
 			runSharedTest(subject)
 		})
 	})
@@ -59,7 +62,16 @@ var _ = Describe("Compression", func() {
 		var subject = feedx.GZipCompression
 		var _ feedx.Compression = subject
 
-		It("should write/read", func() {
+		It("writes/reads", func() {
+			runSharedTest(subject)
+		})
+	})
+
+	Describe("FlateCompression", func() {
+		var subject = feedx.FlateCompression
+		var _ feedx.Compression = subject
+
+		It("writes/reads", func() {
 			runSharedTest(subject)
 		})
 	})

data/consumer.go

@@ -50,6 +50,8 @@ type Consumer interface {
 	Data() interface{}
 	// LastSync returns time of last sync attempt.
 	LastSync() time.Time
+	// LastConsumed returns time of last feed consumption.
+	LastConsumed() time.Time
 	// LastModified returns time at which the remote feed was last modified.
 	LastModified() time.Time
 	// NumRead returns the number of values consumed during the last sync.
@@ -114,7 +116,7 @@ type consumer struct {
 	cfn  ConsumeFunc
 	data atomic.Value
 
-	numRead, lastMod, lastSync int64
+	numRead, lastMod, lastSync, lastConsumed int64
 }
 
 // Data implements Consumer interface.
@@ -132,6 +134,11 @@ func (c *consumer) LastSync() time.Time {
 	return timestamp(atomic.LoadInt64(&c.lastSync)).Time()
 }
 
+// LastConsumed implements Consumer interface.
+func (c *consumer) LastConsumed() time.Time {
+	return timestamp(atomic.LoadInt64(&c.lastConsumed)).Time()
+}
+
 // LastModified implements Consumer interface.
 func (c *consumer) LastModified() time.Time {
 	return timestamp(atomic.LoadInt64(&c.lastMod)).Time()
@@ -147,8 +154,9 @@ func (c *consumer) Close() error {
 }
 
 func (c *consumer) sync(force bool) (*ConsumerSync, error) {
+	syncTime := timestampFromTime(time.Now()).Millis()
 	defer func() {
-		atomic.StoreInt64(&c.lastSync, timestampFromTime(time.Now()).Millis())
+		atomic.StoreInt64(&c.lastSync, syncTime)
 	}()
 
 	// retrieve original last modified time
@@ -158,7 +166,7 @@ func (c *consumer) sync(force bool) (*ConsumerSync, error) {
 	}
 
 	// skip update if not forced or modified
-	if lastMod.Millis() == atomic.LoadInt64(&c.lastMod) && !force {
+	if !force && lastMod > 0 && lastMod.Millis() == atomic.LoadInt64(&c.lastMod) {
 		return &ConsumerSync{Consumer: c}, nil
 	}
 
@@ -180,6 +188,7 @@ func (c *consumer) sync(force bool) (*ConsumerSync, error) {
 	c.data.Store(data)
 	atomic.StoreInt64(&c.numRead, int64(reader.NumRead()))
 	atomic.StoreInt64(&c.lastMod, lastMod.Millis())
+	atomic.StoreInt64(&c.lastConsumed, syncTime)
 	return &ConsumerSync{
 		Consumer:     c,
 		Updated:      true,

data/consumer_test.go

@@ -8,8 +8,8 @@ import (
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
 	"github.com/bsm/feedx/internal/testdata"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Consumer", func() {
@@ -17,25 +17,26 @@ var _ = Describe("Consumer", func() {
 	var obj *bfs.Object
 	var ctx = context.Background()
 
+	consume := func(r *feedx.Reader) (interface{}, error) {
+		var msgs []*testdata.MockMessage
+		for {
+			var msg testdata.MockMessage
+			if err := r.Decode(&msg); err == io.EOF {
+				break
+			} else if err != nil {
+				return nil, err
+			}
+			msgs = append(msgs, &msg)
+		}
+		return msgs, nil
+	}
+
 	BeforeEach(func() {
 		obj = bfs.NewInMemObject("path/to/file.jsonz")
-		Expect(writeMulti(obj, 2)).To(Succeed())
+		Expect(writeMulti(obj, 2, mockTime)).To(Succeed())
 
 		var err error
-		subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
-			var msgs []*testdata.MockMessage
-			for {
-				var msg testdata.MockMessage
-				if err := r.Decode(&msg); err == io.EOF {
-					break
-				}
-				if err != nil {
-					return nil, err
-				}
-				msgs = append(msgs, &msg)
-			}
-			return msgs, nil
-		})
+		subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, consume)
 		Expect(err).NotTo(HaveOccurred())
 	})
 
@@ -43,11 +44,41 @@ var _ = Describe("Consumer", func() {
 		Expect(subject.Close()).To(Succeed())
 	})
 
-	It("should sync and retrieve feeds from remote", func() {
+	It("syncs/retrieves feeds from remote", func() {
 		Expect(subject.LastSync()).To(BeTemporally("~", time.Now(), time.Second))
-		Expect(subject.LastModified()).To(BeTemporally("~", time.Unix(1515151515, 0), time.Second))
+		Expect(subject.LastConsumed()).To(BeTemporally("==", subject.LastSync()))
+		Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
 		Expect(subject.NumRead()).To(Equal(2))
 		Expect(subject.Data()).To(ConsistOf(seed(), seed()))
 		Expect(subject.Close()).To(Succeed())
 	})
+
+	It("consumes feeds only if necessary", func() {
+		prevSync := subject.LastSync()
+		time.Sleep(2 * time.Millisecond)
+
+		testable := subject.(interface{ TestSync() error })
+		Expect(testable.TestSync()).To(Succeed())
+		Expect(subject.LastSync()).To(BeTemporally(">", prevSync))
+		Expect(subject.LastConsumed()).To(BeTemporally("==", prevSync)) // skipped on last sync
+		Expect(subject.LastModified()).To(BeTemporally("==", mockTime.Truncate(time.Millisecond)))
+		Expect(subject.NumRead()).To(Equal(2))
+	})
+
+	It("always consumes if LastModified not set", func() {
+		noModTime := bfs.NewInMemObject("path/to/file.json")
+		Expect(writeMulti(noModTime, 2, time.Time{})).To(Succeed())
+
+		csmr, err := feedx.NewConsumerForRemote(ctx, noModTime, nil, consume)
+		Expect(err).NotTo(HaveOccurred())
+
+		prevSync := csmr.LastSync()
+		time.Sleep(2 * time.Millisecond)
+
+		testable := csmr.(interface{ TestSync() error })
+		Expect(testable.TestSync()).To(Succeed())
+		Expect(csmr.LastSync()).To(BeTemporally(">", prevSync))
+		Expect(csmr.LastConsumed()).To(BeTemporally("==", csmr.LastSync())) // consumed on last sync
+		Expect(csmr.LastModified()).To(BeTemporally("==", time.Unix(0, 0)))
+	})
 })