feedx 0.12.2 → 0.12.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 898f80c359182952ab192f95fb05dc1e6ae806f0d661d71244813b779595d0f8
-  data.tar.gz: 43778c6084131e8ffa9ba59b7836fe198f7595e5dc07af1c4e11a9b03ac49419
+  metadata.gz: c600f733aa3a0663939488c86cfd1d29abb4dde277ceb582a3e3d35c970830fd
+  data.tar.gz: 4ab6bbe4a53e31c4c2c408670f82a72fceb63128c0a22c7f57d0bf8c0eca95d2
 SHA512:
-  metadata.gz: fa8f2e4e26c140bb4c2732655ac73c39ef20055f215ac39c8c62bd77e7cf27d7d8140287ed6c90a4c4da6bf461d491123f76784ce5d770189a45da7fea897965
-  data.tar.gz: a35d5296c5aec452b450e95985535c98b6b99f1a872624b3ad3974cbb0f56b0e6d0ae8395edd849c97242ae0a77ee77537404cb3dce22882a3a310a6e0f04a04
+  metadata.gz: b3e1ef899a5dfafabad3eb45ebb56c672fc9348e52455e04c054967ea3e75b15e2cba3b74449868104204fa3bbf9d4bd4ff3e8c4b1e4db74cc8349982359e1f6
+  data.tar.gz: 3722d116710b5b16315af4fa7ac7e052f9c87efd1531e18d15000bc747119f7dececdd9bae02cac12b9dd363b00d033f710bf9e169986f3a036aab54194e64ef

data/.editorconfig

@@ -7,3 +7,6 @@ end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
+
+[{*.go,Makefile}]
+indent_style = tab

data/.gitignore

@@ -1,3 +1,4 @@
 .rubocop-*
 pkg/
 *~
+*.makefile

data/.travis.yml

@@ -18,7 +18,7 @@ matrix:
         - sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
     - language: go
       go:
-        - 1.14.x
+        - 1.15.x
     - language: go
       go:
-        - 1.13.x
+        - 1.14.x

data/Gemfile

@@ -1,4 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-
-gem 'google-protobuf', '>= 3.7.0-rc2'

data/Gemfile.lock

@@ -1,16 +1,16 @@
 PATH
   remote: .
   specs:
-    feedx (0.12.2)
-      bfs (>= 0.5.0)
+    feedx (0.12.3)
+      bfs (>= 0.8.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.1)
-    bfs (0.7.2)
+    bfs (0.8.0)
     diff-lcs (1.4.4)
-    extpp (0.0.8)
+    extpp (0.0.9)
     gio2 (3.4.3)
       gobject-introspection (= 3.4.3)
     glib2 (3.4.3)
@@ -18,51 +18,52 @@ GEM
       pkg-config (>= 1.3.5)
     gobject-introspection (3.4.3)
       glib2 (= 3.4.3)
-    google-protobuf (3.12.2)
+    google-protobuf (3.14.0)
     native-package-installer (1.0.9)
-    parallel (1.19.2)
-    parser (2.7.1.4)
+    parallel (1.20.1)
+    parser (2.7.2.0)
       ast (~> 2.4.1)
-    pbio (0.2.1)
+    pbio (0.2.2)
       google-protobuf
-    pkg-config (1.4.1)
+    pkg-config (1.4.4)
     rainbow (3.0.0)
     rake (13.0.1)
-    red-arrow (0.17.1)
+    red-arrow (2.0.0)
       extpp (>= 0.0.7)
       gio2 (>= 3.3.6)
       native-package-installer
       pkg-config
-    red-parquet (0.17.1)
-      red-arrow (= 0.17.1)
-    regexp_parser (1.7.1)
+    red-parquet (2.0.0)
+      red-arrow (= 2.0.0)
+    regexp_parser (2.0.0)
     rexml (3.2.4)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.2)
-      rspec-support (~> 3.9.3)
-    rspec-expectations (3.9.2)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.3)
-    rubocop (0.86.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.0)
+    rubocop (1.4.2)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.7)
+      regexp_parser (>= 1.8)
       rexml
-      rubocop-ast (>= 0.0.3, < 1.0)
+      rubocop-ast (>= 1.1.1)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 2.0)
-    rubocop-ast (0.1.0)
-      parser (>= 2.7.0.1)
-    rubocop-performance (1.6.1)
-      rubocop (>= 0.71.0)
+    rubocop-ast (1.3.0)
+      parser (>= 2.7.1.5)
+    rubocop-performance (1.9.1)
+      rubocop (>= 0.90.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
     ruby-progressbar (1.10.1)
     unicode-display_width (1.7.0)
 
@@ -72,10 +73,9 @@ PLATFORMS
 DEPENDENCIES
   bundler
   feedx!
-  google-protobuf (>= 3.7.0.pre.rc2)
   pbio
   rake
-  red-parquet
+  red-parquet (>= 1.0.0)
   rspec
   rubocop
   rubocop-performance

data/Makefile

@@ -1,12 +1,12 @@
-default: vet test
+default: test
 
-test:
-	go test ./...
+.common.makefile:
+	curl -fsSL -o $@ https://gitlab.com/bsm/misc/raw/master/make/go/common.makefile
 
-vet:
-	go vet ./...
+include .common.makefile
 
 proto: internal/testdata/testdata.pb.go
 
 %.pb.go: %.proto
-	protoc -I=. --gogo_out=paths=source_relative:. $<
+	# may need to `go install google.golang.org/protobuf/cmd/protoc-gen-go`
+	protoc -I=. --go_out=paths=source_relative:. $<

data/compression.go

@@ -1,6 +1,7 @@
 package feedx
 
 import (
+	"compress/flate"
 	"compress/gzip"
 	"io"
 	"path"
@@ -20,6 +21,8 @@ func DetectCompression(name string) Compression {
 		ext := path.Ext(path.Base(name))
 		if ext != "" && ext[0] == '.' && ext[len(ext)-1] == 'z' {
 			return GZipCompression
+		} else if ext == ".flate" {
+			return FlateCompression
 		}
 	}
 	return NoCompression
@@ -61,3 +64,18 @@ func (gzipCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
 func (gzipCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
 	return gzip.NewWriter(w), nil
 }
+
+// --------------------------------------------------------------------
+
+// FlateCompression supports flate compression format.
+var FlateCompression = flateCompression{}
+
+type flateCompression struct{}
+
+func (flateCompression) NewReader(r io.Reader) (io.ReadCloser, error) {
+	return flate.NewReader(r), nil
+}
+
+func (flateCompression) NewWriter(w io.Writer) (io.WriteCloser, error) {
+	return flate.NewWriter(w, flate.BestSpeed)
+}

data/compression_test.go

@@ -41,6 +41,9 @@ var _ = Describe("Compression", func() {
 		Expect(feedx.DetectCompression("/path/to/file.pb.gz")).To(Equal(feedx.GZipCompression))
 		Expect(feedx.DetectCompression("/path/to/file.pbz")).To(Equal(feedx.GZipCompression))
 
+		Expect(feedx.DetectCompression("/path/to/file.flate")).To(Equal(feedx.FlateCompression))
+		Expect(feedx.DetectCompression("/path/to/file.whatever.flate")).To(Equal(feedx.FlateCompression))
+
 		Expect(feedx.DetectCompression("")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file")).To(Equal(feedx.NoCompression))
 		Expect(feedx.DetectCompression("/path/to/file.txt")).To(Equal(feedx.NoCompression))
@@ -63,4 +66,13 @@ var _ = Describe("Compression", func() {
 			runSharedTest(subject)
 		})
 	})
+
+	Describe("FlateCompression", func() {
+		var subject = feedx.FlateCompression
+		var _ feedx.Compression = subject
+
+		It("should write/read", func() {
+			runSharedTest(subject)
+		})
+	})
 })

data/ext/parquet/decoder.go

@@ -0,0 +1,170 @@
+package parquet
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"reflect"
+	"time"
+
+	kpq "github.com/bsm/parquet-go/parquet"
+)
+
+type decoder struct {
+	cols    []*columnReader
+	closers []io.Closer
+}
+
+func newDecoder(rs io.ReadSeeker, names []string, batchSize int) (*decoder, error) {
+	file, err := kpq.FileFromReader(rs)
+	if err != nil {
+		return nil, err
+	}
+
+	// normalise column names
+	if len(names) == 0 {
+		for _, c := range file.Schema.Columns() {
+			names = append(names, c.String())
+		}
+	}
+
+	// normalise batch size
+	if batchSize < 1 {
+		batchSize = 1000
+	}
+
+	// initialise column buffers
+	cols := make([]*columnReader, 0, len(names))
+	for _, name := range names {
+		col, ok := file.Schema.ColumnByName(name)
+		if !ok {
+			_ = file.Close()
+			return nil, fmt.Errorf("column %q does not exist", name)
+		}
+		cols = append(cols, newColumnReader(file, col, batchSize))
+	}
+
+	return &decoder{cols: cols, closers: []io.Closer{file}}, nil
+}
+
+func (w *decoder) Decode(v interface{}) error {
+	rv := reflect.ValueOf(v)
+	rt := rv.Type()
+	if rt.Kind() != reflect.Ptr {
+		return fmt.Errorf("cannot decode non-pointer %s type", rt.String())
+	}
+
+	// field index by name
+	fidx := cachedTypeFields(rt.Elem())
+	elem := rv.Elem()
+
+	for _, r := range w.cols {
+		// next column value
+		val, err := r.Next()
+		if err != nil {
+			return err
+		}
+
+		// skip if value is NULL
+		if val == nil {
+			continue
+		}
+
+		// set field if exists
+		if fi, ok := fidx[r.Name()]; ok {
+			fv := elem.Field(fi)
+			if ok := setValue(fv, val); !ok {
+				return fmt.Errorf("cannot assign value of type %T to %s", val, fv.Type())
+			}
+		}
+	}
+
+	return nil
+}
+
+func (w *decoder) Close() (err error) {
+	for _, c := range w.closers {
+		if e := c.Close(); e != nil {
+			err = e
+		}
+	}
+	return
+}
+
+// --------------------------------------------------------------------
+
+func setValue(rv reflect.Value, v interface{}) bool {
+	if rv.Kind() == reflect.Ptr {
+		if rv.IsNil() {
+			if ev := reflect.New(rv.Type().Elem()); setValue(ev, v) {
+				rv.Set(ev)
+				return true
+			}
+			return false
+		}
+		return setValue(rv.Elem(), v)
+	}
+
+	switch vv := v.(type) {
+	case bool:
+		switch rv.Kind() {
+		case reflect.Bool:
+			rv.SetBool(vv)
+			return true
+		}
+	case []byte:
+		switch rv.Kind() {
+		case reflect.String:
+			rv.SetString(string(vv))
+			return true
+		case reflect.Slice:
+			if rv.Type() == byteSliceType {
+				rv.SetBytes(vv)
+				return true
+			}
+		}
+	case int, int8, int16, int32, int64:
+		switch rv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			rv.SetInt(reflect.ValueOf(v).Int())
+			return true
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			rv.SetUint(uint64(reflect.ValueOf(v).Int()))
+			return true
+		}
+	case uint, uint8, uint16, uint32, uint64:
+		switch rv.Kind() {
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			rv.SetInt(int64(reflect.ValueOf(v).Uint()))
+			return true
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			rv.SetUint(reflect.ValueOf(v).Uint())
+			return true
+		}
+	case float32, float64:
+		switch rv.Kind() {
+		case reflect.Float32, reflect.Float64:
+			rv.SetFloat(reflect.ValueOf(v).Float())
+			return true
+		}
+	case kpq.Int96:
+		if rt := rv.Type(); rt == timeType {
+			ns := int64(binary.LittleEndian.Uint64(vv[:8]))
+			jd := int64(binary.LittleEndian.Uint32(vv[8:]))
+			ts := time.Unix((jd-2440588)*86400, ns)
+			rv.Set(reflect.ValueOf(ts))
+			return true
+		} else if rt == int96Type {
+			rv.Set(reflect.ValueOf(v))
+			return true
+		}
+	}
+
+	return false
+}
+
+var (
+	byteSliceType = reflect.TypeOf(([]byte)(nil))
+	int96Type     = reflect.TypeOf(kpq.Int96{})
+	timeType      = reflect.TypeOf(time.Time{})
+)

data/ext/parquet/decoder_test.go

@@ -0,0 +1,88 @@
+package parquet_test
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"os"
+	"time"
+
+	"github.com/bsm/feedx"
+	"github.com/bsm/feedx/ext/parquet"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Decoder", func() {
+	var subject feedx.FormatDecoder
+	var fixture *os.File
+
+	f32ptr := func(f float32) *float32 { return &f }
+
+	BeforeEach(func() {
+		var err error
+		fixture, err = os.Open("testdata/alltypes_plain.parquet")
+		Expect(err).NotTo(HaveOccurred())
+
+		format := &parquet.Format{BatchSize: 3}
+		subject, err = format.NewDecoder(fixture)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(subject.Close()).To(Succeed())
+		Expect(fixture.Close()).To(Succeed())
+	})
+
+	It("should decode", func() {
+		v1 := new(mockStruct)
+		Expect(subject.Decode(v1)).To(Succeed())
+		Expect(v1).To(Equal(&mockStruct{
+			ID:         4,
+			Bool:       true,
+			Float:      f32ptr(0),
+			DateString: "03/01/09", ByteString: []byte("0"),
+			Timestamp: time.Unix(1235865600, 0),
+		}))
+
+		v2 := new(mockStruct)
+		Expect(subject.Decode(v2)).To(Succeed())
+		Expect(v2).To(Equal(&mockStruct{
+			ID:      5,
+			TinyInt: 1, SmallUint: 1, StdInt: 1, BigInt: 10,
+			Float: f32ptr(1.1), Double: 10.1,
+			DateString: "03/01/09", ByteString: []byte("1"),
+			Timestamp: time.Unix(1235865660, 0),
+		}))
+
+		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v3
+		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v4
+		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v5
+
+		v6 := new(mockStruct)
+		Expect(subject.Decode(v6)).To(Succeed())
+		Expect(v6).To(Equal(&mockStruct{
+			ID:      3,
+			Bool:    false,
+			TinyInt: 1, SmallUint: 1, StdInt: 1, BigInt: 10,
+			Float: f32ptr(1.1), Double: 10.1,
+			DateString: "02/01/09", ByteString: []byte("1"),
+			Timestamp: time.Unix(1233446460, 0),
+		}))
+
+		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v7
+		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v8
+
+		v9 := new(mockStruct)
+		Expect(subject.Decode(v9)).To(MatchError(io.EOF))
+	})
+
+	It("should open from non-file readers", func() {
+		bin, err := ioutil.ReadFile("testdata/alltypes_plain.parquet")
+		Expect(err).NotTo(HaveOccurred())
+
+		dec, err := new(parquet.Format).NewDecoder(bytes.NewReader(bin))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(dec.Close()).To(Succeed())
+	})
+})