feedx 0.11.0 → 0.12.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5662b9773c03d14b3d7d68b8a1ad208986b95c675cbf61d445f74ebdf3b86b09
-  data.tar.gz: 074c6e5b86975aed1b31caa5250d7132a2d6bb68af7c221f89043f5f574d0268
+  metadata.gz: c1c91575fbae6b3b5df7c840d8355ec2d27916212a57d19192fe63288b39ed9e
+  data.tar.gz: e21dffcc3da9ebadd5f936380a00427339566bde0d058e66498528f9418942d6
 SHA512:
-  metadata.gz: 3c39e851c7e9266c9b6248f6c6010d351ae9d57809c5f6d153bd6defdf4933fa6f138637e8ad6b0ee5f8cab07dec002b2995ecc0a7ff02222974a49ab1d646e3
-  data.tar.gz: 1846a1ba29397928281d1ba4b5c683c6c4e097e2879acedc52de21d722033290e5aac72a8f001d2012643f67661aa9fd76bac27c925782b6760b2d96295e1e80
+  metadata.gz: d8f0e86db26b77894b062a9919c6f902ddb6123f218fdb9233ab8126b0905c023ba8a32d69cc90b3f035345e637cb6719bd43bee984f3f8c62460772f941405f
+  data.tar.gz: 7451b7e3305ce7ec05a8f7c7d4a8cd9a8b65b1eef417f419903f139d01db94cc75045e0fac105eac4dc78699e2012c7c1450b16ad213615fe563799cb7ab1fab

data/.editorconfig

@@ -7,3 +7,6 @@ end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
+
+[{*.go,Makefile}]
+indent_style = tab

data/.github/workflows/test.yml

@@ -0,0 +1,60 @@
+name: Test
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  go:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        go-version: [1.15.x, 1.16.x]
+    steps:
+      - name: Fetch staticcheck
+        uses: engineerd/configurator@v0.0.6
+        with:
+          name: staticcheck
+          fromGitHubReleases: true
+          repo: dominikh/go-tools
+          version: latest
+          urlTemplate: "https://github.com/dominikh/go-tools/releases/download/2020.2.3/staticcheck_linux_amd64.tar.gz"
+          pathInArchive: staticcheck/staticcheck
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache dependencies
+        uses: actions/cache@v2
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - name: Run tests
+        run: make test
+      - name: Run staticcheck
+        run: make staticcheck
+  ruby:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ["2.6", "2.7", "3.0"]
+    steps:
+      - name: Install libarrow
+        run: |
+          wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
+          sudo apt install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
+          sudo apt update
+          sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler-cache: true
+      - run: bundle exec rake

data/.gitignore

@@ -1,3 +1,4 @@
 .rubocop-*
 pkg/
 *~
+*.makefile

data/.rubocop.yml

@@ -1,6 +1,17 @@
-require: rubocop-performance
-inherit_from:
-  - https://gitlab.com/bsm/misc/raw/master/rubocop/default.yml
+inherit_gem:
+  rubocop-bsm:
+    - default.yml
+inherit_mode:
+  merge:
+    - Exclude
 
 AllCops:
-  TargetRubyVersion: "2.4"
+  TargetRubyVersion: 2.6
+Naming/MethodParameterName:
+  MinNameLength: 2
+Naming/MemoizedInstanceVariableName:
+  Enabled: false
+Naming/FileName:
+  Exclude: [lib/sortable-by.rb]
+Metrics/ParameterLists:
+  Max: 8

data/Gemfile

@@ -1,4 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-
-gem 'google-protobuf', '>= 3.7.0-rc2'

data/Gemfile.lock

@@ -1,70 +1,102 @@
 PATH
   remote: .
   specs:
-    feedx (0.11.0)
-      bfs (>= 0.5.0)
+    feedx (0.12.5)
+      bfs (>= 0.8.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.1)
-    bfs (0.7.0)
-    diff-lcs (1.3)
-    extpp (0.0.8)
-    gio2 (3.4.3)
-      gobject-introspection (= 3.4.3)
-    glib2 (3.4.3)
+    activesupport (6.1.3.1)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    ast (2.4.2)
+    bfs (0.8.1)
+    bigdecimal (3.0.2)
+    concurrent-ruby (1.1.8)
+    diff-lcs (1.4.4)
+    extpp (0.0.9)
+    gio2 (3.4.4)
+      gobject-introspection (= 3.4.4)
+    glib2 (3.4.4)
       native-package-installer (>= 1.0.3)
       pkg-config (>= 1.3.5)
-    gobject-introspection (3.4.3)
-      glib2 (= 3.4.3)
-    google-protobuf (3.12.2)
-    native-package-installer (1.0.9)
-    parallel (1.19.1)
-    parser (2.7.1.3)
-      ast (~> 2.4.0)
-    pbio (0.2.1)
+    gobject-introspection (3.4.4)
+      glib2 (= 3.4.4)
+    google-protobuf (3.15.8)
+    i18n (1.8.10)
+      concurrent-ruby (~> 1.0)
+    minitest (5.14.4)
+    native-package-installer (1.1.1)
+    parallel (1.20.1)
+    parser (3.0.1.1)
+      ast (~> 2.4.1)
+    pbio (0.2.2)
       google-protobuf
-    pkg-config (1.4.1)
+    pkg-config (1.4.6)
+    rack (2.2.3)
     rainbow (3.0.0)
-    rake (13.0.1)
-    red-arrow (0.17.1)
+    rake (13.0.3)
+    red-arrow (3.0.0)
+      bigdecimal (>= 2.0.3)
       extpp (>= 0.0.7)
       gio2 (>= 3.3.6)
       native-package-installer
       pkg-config
-    red-parquet (0.17.1)
-      red-arrow (= 0.17.1)
-    regexp_parser (1.7.1)
-    rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    red-parquet (3.0.0)
+      red-arrow (= 3.0.0)
+    regexp_parser (2.1.1)
+    rexml (3.2.5)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.1)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rubocop (0.85.1)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.2)
+    rubocop (1.13.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 0.0.3)
+      rubocop-ast (>= 1.2.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.0.3)
-      parser (>= 2.7.0.1)
-    rubocop-performance (1.6.1)
-      rubocop (>= 0.71.0)
-    ruby-progressbar (1.10.1)
-    unicode-display_width (1.7.0)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.5.0)
+      parser (>= 3.0.1.1)
+    rubocop-bsm (0.5.9)
+      rubocop (~> 1.0)
+      rubocop-performance
+      rubocop-rails
+      rubocop-rake
+      rubocop-rspec
+    rubocop-performance (1.11.1)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    rubocop-rails (2.9.1)
+      activesupport (>= 4.2.0)
+      rack (>= 1.1)
+      rubocop (>= 0.90.0, < 2.0)
+    rubocop-rake (0.5.1)
+      rubocop
+    rubocop-rspec (2.3.0)
+      rubocop (~> 1.0)
+      rubocop-ast (>= 1.1.0)
+    ruby-progressbar (1.11.0)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
+    unicode-display_width (2.0.0)
+    zeitwerk (2.4.2)
 
 PLATFORMS
   ruby
@@ -72,13 +104,11 @@ PLATFORMS
 DEPENDENCIES
   bundler
   feedx!
-  google-protobuf (>= 3.7.0.pre.rc2)
   pbio
   rake
-  red-parquet
+  red-parquet (>= 3.0, < 4.0)
   rspec
-  rubocop
-  rubocop-performance
+  rubocop-bsm
 
 BUNDLED WITH
    2.1.4

data/Makefile

@@ -1,12 +1,12 @@
-default: vet test
+default: test
 
-test:
-	go test ./...
+.common.makefile:
+	curl -fsSL -o $@ https://gitlab.com/bsm/misc/raw/master/make/go/common.makefile
 
-vet:
-	go vet ./...
+include .common.makefile
 
 proto: internal/testdata/testdata.pb.go
 
 %.pb.go: %.proto
-	protoc -I=. --gogo_out=paths=source_relative:. $<
+	# may need to `go install google.golang.org/protobuf/cmd/protoc-gen-go`
+	protoc -I=. --go_out=paths=source_relative:. $<

data/README.md

@@ -1,6 +1,6 @@
 # Feedx
 
-[![Build Status](https://travis-ci.org/bsm/feedx.png?branch=master)](https://travis-ci.org/bsm/feedx)
+[![Test](https://github.com/bsm/feedx/actions/workflows/test.yml/badge.svg)](https://github.com/bsm/feedx/actions/workflows/test.yml)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 Feed-based data exchange between services.

data/compression.go

@@ -1,6 +1,7 @@
 package feedx
 
 import (
+	"compress/flate"
 	"compress/gzip"
 	"io"
 	"path"
@@ -20,6 +21,8 @@ func DetectCompression(name string) Compression {
 		ext := path.Ext(path.Base(name))
 		if ext != "" && ext[0] == '.' && ext[len(ext)-1] == 'z' {
 			return GZipCompression
+		} else if ext == ".flate" {
+			return FlateCompression
 		}
 	}
 	return NoCompression
@@ -61,3 +64,18 @@ func (gzipCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
 func (gzipCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
 	return gzip.NewWriter(w), nil
 }
+
+// --------------------------------------------------------------------
+
+// FlateCompression supports flate compression format.
+var FlateCompression = flateCompression{}
+
+type flateCompression struct{}
+
+func (flateCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
+	return flate.NewReader(r), nil
+}
+
+func (flateCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
+	return flate.NewWriter(w, flate.BestSpeed)
+}

data/compression_test.go

@@ -4,8 +4,8 @@ import (
 	"bytes"
 
 	"github.com/bsm/feedx"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Compression", func() {
@@ -41,6 +41,9 @@ var _ = Describe("Compression", func() {
 		Expect(feedx.DetectCompression("/path/to/file.pb.gz")).To(Equal(feedx.GZipCompression))
 		Expect(feedx.DetectCompression("/path/to/file.pbz")).To(Equal(feedx.GZipCompression))
 
+		Expect(feedx.DetectCompression("/path/to/file.flate")).To(Equal(feedx.FlateCompression))
+		Expect(feedx.DetectCompression("/path/to/file.whatever.flate")).To(Equal(feedx.FlateCompression))
+
 		Expect(feedx.DetectCompression("")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file.txt")).To(Equal(feedx.NoCompression))
@@ -63,4 +66,13 @@ var _ = Describe("Compression", func() {
 			runSharedTest(subject)
 		})
 	})
+
+	Describe("FlateCompression", func() {
+		var subject = feedx.FlateCompression
+		var _ feedx.Compression = subject
+
+		It("should write/read", func() {
+			runSharedTest(subject)
+		})
+	})
 })

data/consumer_test.go

@@ -8,8 +8,8 @@ import (
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
 	"github.com/bsm/feedx/internal/testdata"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
 )
 
 var _ = Describe("Consumer", func() {

data/ext/parquet/decoder.go

@@ -0,0 +1,170 @@
+package parquet
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"reflect"
+	"time"
+
+	kpq "github.com/bsm/parquet-go/parquet"
+)
+
+type decoder struct {
+	cols    []*columnReader
+	closers []io.Closer
+}
+
+func newDecoder(rs io.ReadSeeker, names []string, batchSize int) (*decoder, error) {
+	file, err := kpq.FileFromReader(rs)
+	if err != nil {
+		return nil, err
+	}
+
+	// normalise column names
+	if len(names) == 0 {
+		for _, c := range file.Schema.Columns() {
+			names = append(names, c.String())
+		}
+	}
+
+	// normalise batch size
+	if batchSize < 1 {
+		batchSize = 1000
+	}
+
+	// initialise column buffers
+	cols := make([]*columnReader, 0, len(names))
+	for _, name := range names {
+		col, ok := file.Schema.ColumnByName(name)
+		if !ok {
+			_ = file.Close()
+			return nil, fmt.Errorf("column %q does not exist", name)
+		}
+		cols = append(cols, newColumnReader(file, col, batchSize))
+	}
+
+	return &decoder{cols: cols, closers: []io.Closer{file}}, nil
+}
+
+func (w *decoder) Decode(v interface{}) error {
+	rv := reflect.ValueOf(v)
+	rt := rv.Type()
+	if rt.Kind() != reflect.Ptr {
+		return fmt.Errorf("cannot decode non-pointer %s type", rt.String())
+	}
+
+	// field index by name
+	fidx := cachedTypeFields(rt.Elem())
+	elem := rv.Elem()
+
+	for _, r := range w.cols {
+		// next column value
+		val, err := r.Next()
+		if err != nil {
+			return err
+		}
+
+		// skip if value is NULL
+		if val == nil {
+			continue
+		}
+
+		// set field if exists
+		if fi, ok := fidx[r.Name()]; ok {
+			fv := elem.Field(fi)
+			if ok := setValue(fv, val); !ok {
+				return fmt.Errorf("cannot assign value of type %T to %s", val, fv.Type())
+			}
+		}
+	}
+
+	return nil
+}
+
+func (w *decoder) Close() (err error) {
+	for _, c := range w.closers {
+		if e := c.Close(); e != nil {
+			err = e
+		}
+	}
+	return
+}
+
+// --------------------------------------------------------------------
+
+func setValue(rv reflect.Value, v interface{}) bool {
+	if rv.Kind() == reflect.Ptr {
+		if rv.IsNil() {
+			if ev := reflect.New(rv.Type().Elem()); setValue(ev, v) {
+				rv.Set(ev)
+				return true
+			}
+			return false
+		}
+		return setValue(rv.Elem(), v)
+	}
+
+	switch vv := v.(type) {
+	case bool:
+		switch rv.Kind() {
+		case reflect.Bool:
+			rv.SetBool(vv)
+			return true
+		}
+	case []byte:
+		switch rv.Kind() {
+		case reflect.String:
+			rv.SetString(string(vv))
+			return true
+		case reflect.Slice:
+			if rv.Type() == byteSliceType {
+				rv.SetBytes(vv)
+				return true
+			}
+		}
+	case int, int8, int16, int32, int64:
+		switch rv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			rv.SetInt(reflect.ValueOf(v).Int())
+			return true
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			rv.SetUint(uint64(reflect.ValueOf(v).Int()))
+			return true
+		}
+	case uint, uint8, uint16, uint32, uint64:
+		switch rv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			rv.SetInt(int64(reflect.ValueOf(v).Uint()))
+			return true
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			rv.SetUint(reflect.ValueOf(v).Uint())
+			return true
+		}
+	case float32, float64:
+		switch rv.Kind() {
+		case reflect.Float32, reflect.Float64:
+			rv.SetFloat(reflect.ValueOf(v).Float())
+			return true
+		}
+	case kpq.Int96:
+		if rt := rv.Type(); rt == timeType {
+			ns := int64(binary.LittleEndian.Uint64(vv[:8]))
+			jd := int64(binary.LittleEndian.Uint32(vv[8:]))
+			ts := time.Unix((jd-2440588)*86400, ns)
+			rv.Set(reflect.ValueOf(ts))
+			return true
+		} else if rt == int96Type {
+			rv.Set(reflect.ValueOf(v))
+			return true
+		}
+	}
+
+	return false
+}
+
+var (
+	byteSliceType = reflect.TypeOf(([]byte)(nil))
+	int96Type     = reflect.TypeOf(kpq.Int96{})
+	timeType      = reflect.TypeOf(time.Time{})
+)