feedx 0.10.2 → 0.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73577178d9531fd5397bca23cb9e85cb0d6ab8ea9f97e32b0ce5b97dbebe43d4
-  data.tar.gz: e1439be97329b54309831cd7c19c19982058e71879dbc612058f0bb5fce1a578
+  metadata.gz: c600f733aa3a0663939488c86cfd1d29abb4dde277ceb582a3e3d35c970830fd
+  data.tar.gz: 4ab6bbe4a53e31c4c2c408670f82a72fceb63128c0a22c7f57d0bf8c0eca95d2
 SHA512:
-  metadata.gz: 673e28b0f7d0e01543796f7aa5ea869a80e119c2fbbcbb21eb9d6918ea4e051ecc20e6f1634e79fcc56acf1991ae68b26f7f69e1ec43c47a8b96d8a359d20834
-  data.tar.gz: f2b9d69ef7dbc915c4a08c798e0da5f65a044907b27c3e742a2c4a7c1a1b5c514e648fc1924a335b99ca65d56454718b0a9041827306f6f19ce0a7c2d6507a97
+  metadata.gz: b3e1ef899a5dfafabad3eb45ebb56c672fc9348e52455e04c054967ea3e75b15e2cba3b74449868104204fa3bbf9d4bd4ff3e8c4b1e4db74cc8349982359e1f6
+  data.tar.gz: 3722d116710b5b16315af4fa7ac7e052f9c87efd1531e18d15000bc747119f7dececdd9bae02cac12b9dd363b00d033f710bf9e169986f3a036aab54194e64ef

data/.editorconfig

@@ -7,3 +7,6 @@ end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
+
+[{*.go,Makefile}]
+indent_style = tab

data/.gitignore

@@ -1,3 +1,4 @@
 .rubocop-*
 pkg/
 *~
+*.makefile

data/.rubocop.yml

@@ -4,3 +4,5 @@ inherit_from:
 
 AllCops:
   TargetRubyVersion: "2.4"
+Metrics/ParameterLists:
+  Max: 10

data/.travis.yml

@@ -3,12 +3,22 @@ matrix:
     - language: ruby
       rvm:
         - 2.7
+      before_install:
+        - wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
+        - sudo apt install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
+        - sudo apt update
+        - sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
     - language: ruby
       rvm:
         - 2.6
+      before_install:
+        - wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
+        - sudo apt install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
+        - sudo apt update
+        - sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
     - language: go
       go:
-        - 1.14.x
+        - 1.15.x
     - language: go
       go:
-        - 1.13.x
+        - 1.14.x

data/Gemfile

@@ -1,4 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-
-gem 'google-protobuf', '>= 3.7.0-rc2'

data/Gemfile.lock

@@ -1,49 +1,69 @@
 PATH
   remote: .
   specs:
-    feedx (0.10.2)
-      bfs (>= 0.5.0)
+    feedx (0.12.3)
+      bfs (>= 0.8.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    ast (2.4.0)
-    bfs (0.7.0)
-    diff-lcs (1.3)
-    google-protobuf (3.12.2)
-    parallel (1.19.1)
-    parser (2.7.1.3)
-      ast (~> 2.4.0)
-    pbio (0.2.1)
+    ast (2.4.1)
+    bfs (0.8.0)
+    diff-lcs (1.4.4)
+    extpp (0.0.9)
+    gio2 (3.4.3)
+      gobject-introspection (= 3.4.3)
+    glib2 (3.4.3)
+      native-package-installer (>= 1.0.3)
+      pkg-config (>= 1.3.5)
+    gobject-introspection (3.4.3)
+      glib2 (= 3.4.3)
+    google-protobuf (3.14.0)
+    native-package-installer (1.0.9)
+    parallel (1.20.1)
+    parser (2.7.2.0)
+      ast (~> 2.4.1)
+    pbio (0.2.2)
       google-protobuf
+    pkg-config (1.4.4)
     rainbow (3.0.0)
     rake (13.0.1)
+    red-arrow (2.0.0)
+      extpp (>= 0.0.7)
+      gio2 (>= 3.3.6)
+      native-package-installer
+      pkg-config
+    red-parquet (2.0.0)
+      red-arrow (= 2.0.0)
+    regexp_parser (2.0.0)
     rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rubocop (0.84.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.0)
+    rubocop (1.4.2)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
       rexml
-      rubocop-ast (>= 0.0.3)
+      rubocop-ast (>= 1.1.1)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.0.3)
-      parser (>= 2.7.0.1)
-    rubocop-performance (1.6.0)
-      rubocop (>= 0.71.0)
+    rubocop-ast (1.3.0)
+      parser (>= 2.7.1.5)
+    rubocop-performance (1.9.1)
+      rubocop (>= 0.90.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.10.1)
     unicode-display_width (1.7.0)
 
@@ -53,12 +73,12 @@ PLATFORMS
 DEPENDENCIES
   bundler
   feedx!
-  google-protobuf (>= 3.7.0.pre.rc2)
   pbio
   rake
+  red-parquet (>= 1.0.0)
   rspec
   rubocop
   rubocop-performance
 
 BUNDLED WITH
-   2.1.2
+   2.1.4

data/Makefile

@@ -1,7 +1,12 @@
-default: vet test
+default: test
 
-test:
-	go test ./...
+.common.makefile:
+	curl -fsSL -o $@ https://gitlab.com/bsm/misc/raw/master/make/go/common.makefile
 
-vet:
-	go vet ./...
+include .common.makefile
+
+proto: internal/testdata/testdata.pb.go
+
+%.pb.go: %.proto
+	# may need to `go install google.golang.org/protobuf/cmd/protoc-gen-go`
+	protoc -I=. --go_out=paths=source_relative:. $<

data/compression.go

@@ -1,6 +1,7 @@
 package feedx
 
 import (
+	"compress/flate"
 	"compress/gzip"
 	"io"
 	"path"
@@ -20,6 +21,8 @@ func DetectCompression(name string) Compression {
 		ext := path.Ext(path.Base(name))
 		if ext != "" && ext[0] == '.' && ext[len(ext)-1] == 'z' {
 			return GZipCompression
+		} else if ext == ".flate" {
+			return FlateCompression
 		}
 	}
 	return NoCompression
@@ -61,3 +64,18 @@ func (gzipCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
 func (gzipCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
 	return gzip.NewWriter(w), nil
 }
+
+// --------------------------------------------------------------------
+
+// FlateCompression supports flate compression format.
+var FlateCompression = flateCompression{}
+
+type flateCompression struct{}
+
+func (flateCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
+	return flate.NewReader(r), nil
+}
+
+func (flateCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
+	return flate.NewWriter(w, flate.BestSpeed)
+}

data/compression_test.go

@@ -41,6 +41,9 @@ var _ = Describe("Compression", func() {
 		Expect(feedx.DetectCompression("/path/to/file.pb.gz")).To(Equal(feedx.GZipCompression))
 		Expect(feedx.DetectCompression("/path/to/file.pbz")).To(Equal(feedx.GZipCompression))
 
+		Expect(feedx.DetectCompression("/path/to/file.flate")).To(Equal(feedx.FlateCompression))
+		Expect(feedx.DetectCompression("/path/to/file.whatever.flate")).To(Equal(feedx.FlateCompression))
+
 		Expect(feedx.DetectCompression("")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file.txt")).To(Equal(feedx.NoCompression))
@@ -63,4 +66,13 @@ var _ = Describe("Compression", func() {
 			runSharedTest(subject)
 		})
 	})
+
+	Describe("FlateCompression", func() {
+		var subject = feedx.FlateCompression
+		var _ feedx.Compression = subject
+
+		It("should write/read", func() {
+			runSharedTest(subject)
+		})
+	})
 })

data/consumer_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/bsm/bfs"
 	"github.com/bsm/feedx"
+	"github.com/bsm/feedx/internal/testdata"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -22,16 +23,16 @@ var _ = Describe("Consumer", func() {
 
 		var err error
 		subject, err = feedx.NewConsumerForRemote(ctx, obj, nil, func(r *feedx.Reader) (interface{}, error) {
-			var msgs []MockMessage
+			var msgs []*testdata.MockMessage
 			for {
-				var msg MockMessage
+				var msg testdata.MockMessage
 				if err := r.Decode(&msg); err == io.EOF {
 					break
 				}
 				if err != nil {
 					return nil, err
 				}
-				msgs = append(msgs, msg)
+				msgs = append(msgs, &msg)
 			}
 			return msgs, nil
 		})
@@ -46,7 +47,7 @@ var _ = Describe("Consumer", func() {
 		Expect(subject.LastSync()).To(BeTemporally("~", time.Now(), time.Second))
 		Expect(subject.LastModified()).To(BeTemporally("~", time.Unix(1515151515, 0), time.Second))
 		Expect(subject.NumRead()).To(Equal(2))
-		Expect(subject.Data()).To(Equal([]MockMessage{fixture, fixture}))
+		Expect(subject.Data()).To(ConsistOf(seed(), seed()))
 		Expect(subject.Close()).To(Succeed())
 	})
 })

data/ext/parquet/decoder.go

@@ -0,0 +1,170 @@
+package parquet
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"reflect"
+	"time"
+
+	kpq "github.com/bsm/parquet-go/parquet"
+)
+
+type decoder struct {
+	cols    []*columnReader
+	closers []io.Closer
+}
+
+func newDecoder(rs io.ReadSeeker, names []string, batchSize int) (*decoder, error) {
+	file, err := kpq.FileFromReader(rs)
+	if err != nil {
+		return nil, err
+	}
+
+	// normalise column names
+	if len(names) == 0 {
+		for _, c := range file.Schema.Columns() {
+			names = append(names, c.String())
+		}
+	}
+
+	// normalise batch size
+	if batchSize < 1 {
+		batchSize = 1000
+	}
+
+	// initialise column buffers
+	cols := make([]*columnReader, 0, len(names))
+	for _, name := range names {
+		col, ok := file.Schema.ColumnByName(name)
+		if !ok {
+			_ = file.Close()
+			return nil, fmt.Errorf("column %q does not exist", name)
+		}
+		cols = append(cols, newColumnReader(file, col, batchSize))
+	}
+
+	return &decoder{cols: cols, closers: []io.Closer{file}}, nil
+}
+
+func (w *decoder) Decode(v interface{}) error {
+	rv := reflect.ValueOf(v)
+	rt := rv.Type()
+	if rt.Kind() != reflect.Ptr {
+		return fmt.Errorf("cannot decode non-pointer %s type", rt.String())
+	}
+
+	// field index by name
+	fidx := cachedTypeFields(rt.Elem())
+	elem := rv.Elem()
+
+	for _, r := range w.cols {
+		// next column value
+		val, err := r.Next()
+		if err != nil {
+			return err
+		}
+
+		// skip if value is NULL
+		if val == nil {
+			continue
+		}
+
+		// set field if exists
+		if fi, ok := fidx[r.Name()]; ok {
+			fv := elem.Field(fi)
+			if ok := setValue(fv, val); !ok {
+				return fmt.Errorf("cannot assign value of type %T to %s", val, fv.Type())
+			}
+		}
+	}
+
+	return nil
+}
+
+func (w *decoder) Close() (err error) {
+	for _, c := range w.closers {
+		if e := c.Close(); e != nil {
+			err = e
+		}
+	}
+	return
+}
+
+// --------------------------------------------------------------------
+
+func setValue(rv reflect.Value, v interface{}) bool {
+	if rv.Kind() == reflect.Ptr {
+		if rv.IsNil() {
+			if ev := reflect.New(rv.Type().Elem()); setValue(ev, v) {
+				rv.Set(ev)
+				return true
+			}
+			return false
+		}
+		return setValue(rv.Elem(), v)
+	}
+
+	switch vv := v.(type) {
+	case bool:
+		switch rv.Kind() {
+		case reflect.Bool:
+			rv.SetBool(vv)
+			return true
+		}
+	case []byte:
+		switch rv.Kind() {
+		case reflect.String:
+			rv.SetString(string(vv))
+			return true
+		case reflect.Slice:
+			if rv.Type() == byteSliceType {
+				rv.SetBytes(vv)
+				return true
+			}
+		}
+	case int, int8, int16, int32, int64:
+		switch rv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			rv.SetInt(reflect.ValueOf(v).Int())
+			return true
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			rv.SetUint(uint64(reflect.ValueOf(v).Int()))
+			return true
+		}
+	case uint, uint8, uint16, uint32, uint64:
+		switch rv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			rv.SetInt(int64(reflect.ValueOf(v).Uint()))
+			return true
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			rv.SetUint(reflect.ValueOf(v).Uint())
+			return true
+		}
+	case float32, float64:
+		switch rv.Kind() {
+		case reflect.Float32, reflect.Float64:
+			rv.SetFloat(reflect.ValueOf(v).Float())
+			return true
+		}
+	case kpq.Int96:
+		if rt := rv.Type(); rt == timeType {
+			ns := int64(binary.LittleEndian.Uint64(vv[:8]))
+			jd := int64(binary.LittleEndian.Uint32(vv[8:]))
+			ts := time.Unix((jd-2440588)*86400, ns)
+			rv.Set(reflect.ValueOf(ts))
+			return true
+		} else if rt == int96Type {
+			rv.Set(reflect.ValueOf(v))
+			return true
+		}
+	}
+
+	return false
+}
+
+var (
+	byteSliceType = reflect.TypeOf(([]byte)(nil))
+	int96Type     = reflect.TypeOf(kpq.Int96{})
+	timeType      = reflect.TypeOf(time.Time{})
+)