feedx 0.12.6 → 0.12.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ab1b8d12efde331dcd1ca96c686c29e1ef16ab895f049121078b9f0503d23276
-  data.tar.gz: 3d51319d3ea39cbceeb480252e06c37238fce0cda68aa6b65cf1f59c0f0af38b
+  metadata.gz: 6a8328c3af95657d103cf62048d13cb5a14cc50466351f2178082612c347a8cf
+  data.tar.gz: f2fdb20a577c88f165f2cbac3dadf217889716b67b9ae20e4941ed9ef396e864
 SHA512:
-  metadata.gz: d08c34b59158c69d500b7cffe8019550dafa87b760e4b0cb3b25afc367801a3f3473486987dcb62e5c1bc62c05d3653bf26692fcbdcbb552d2af1073f4eced50
-  data.tar.gz: cfa3f24458cead0c12c1f03b559ded816095e1e9cf29a1918c142cd255f782fa3c51d7c45ca5833ab25b5a745e0e68f4b1b1062c7760c56ff1b57da122767135
+  metadata.gz: 6cf78b2fb0917c7afb33014e6b3e30d045ffca874f4c8463092f25a7b22b268ab9f2cbd03a1dfda9df7a9314fe213d11c332ff965eb7b81436768cb94d80bf83
+  data.tar.gz: 9131ba95738e600ab4e731546b6e876cae77120782e148a6eac6e3ea4b90e398975f15f2af00b325e86f3fc73a79c39c4a6be25e80bc7298881ff636d61d150b

data/.github/workflows/lint.yml

@@ -0,0 +1,18 @@
+name: Lint
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+jobs:
+  golangci:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Run lint
+        uses: golangci/golangci-lint-action@v2
+        with:
+          version: latest

data/.github/workflows/test.yml

@@ -11,18 +11,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go-version: [1.15.x, 1.16.x]
+        go-version: [1.16.x, 1.17.x]
     steps:
-      - name: Fetch staticcheck
-        uses: engineerd/configurator@v0.0.6
-        with:
-          name: staticcheck
-          fromGitHubReleases: true
-          repo: dominikh/go-tools
-          version: latest
-          urlTemplate: "https://github.com/dominikh/go-tools/releases/download/2020.2.3/staticcheck_linux_amd64.tar.gz"
-          pathInArchive: staticcheck/staticcheck
-          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Checkout
         uses: actions/checkout@v2
       - name: Cache dependencies
@@ -38,18 +28,16 @@ jobs:
           go-version: ${{ matrix.go-version }}
       - name: Run tests
         run: make test
-      - name: Run staticcheck
-        run: make staticcheck
   ruby:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby-version: ["2.6", "2.7", "3.0"]
+        ruby-version: ["2.7", "3.0", "3.1"]
     steps:
       - name: Install libarrow
         run: |
-          wget https://apache.bintray.com/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
-          sudo apt install -y ./apache-arrow-archive-keyring-latest-$(lsb_release --codename --short).deb
+          wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
+          sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
           sudo apt update
           sudo apt install -y libarrow-dev libarrow-glib-dev libarrow-dataset-dev libplasma-dev libplasma-glib-dev libgandiva-dev libgandiva-glib-dev libparquet-dev libparquet-glib-dev
       - uses: actions/checkout@v2

data/.golangci.yml

@@ -0,0 +1,4 @@
+linters-settings:
+  errcheck:
+    exclude-functions:
+      - (*github.com/bsm/feedx.Writer).Discard

data/.rubocop.yml

@@ -6,7 +6,7 @@ inherit_mode:
     - Exclude
 
 AllCops:
-  TargetRubyVersion: 2.6
+  TargetRubyVersion: 2.7
 Naming/MethodParameterName:
   MinNameLength: 2
 Naming/MemoizedInstanceVariableName:

data/Gemfile.lock

@@ -1,34 +1,34 @@
 PATH
   remote: .
   specs:
-    feedx (0.12.6)
+    feedx (0.12.7)
       bfs (>= 0.8.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
     ast (2.4.2)
-    bfs (0.8.4)
-    bigdecimal (3.0.2)
-    diff-lcs (1.4.4)
-    extpp (0.0.9)
-    gio2 (3.4.4)
-      gobject-introspection (= 3.4.4)
-    glib2 (3.4.4)
+    bfs (0.9.0)
+    bigdecimal (3.1.1)
+    diff-lcs (1.5.0)
+    extpp (0.1.0)
+    gio2 (3.4.9)
+      gobject-introspection (= 3.4.9)
+    glib2 (3.4.9)
       native-package-installer (>= 1.0.3)
       pkg-config (>= 1.3.5)
-    gobject-introspection (3.4.4)
-      glib2 (= 3.4.4)
-    google-protobuf (3.17.3)
+    gobject-introspection (3.4.9)
+      glib2 (= 3.4.9)
+    google-protobuf (3.19.2)
     native-package-installer (1.1.1)
-    parallel (1.20.1)
-    parser (3.0.1.1)
+    parallel (1.21.0)
+    parser (3.1.0.0)
       ast (~> 2.4.1)
     pbio (0.3.0)
       google-protobuf
     pkg-config (1.4.6)
     rainbow (3.0.0)
-    rake (13.0.3)
+    rake (13.0.6)
     red-arrow (3.0.0)
       bigdecimal (>= 2.0.3)
       extpp (>= 0.0.7)
@@ -37,7 +37,7 @@ GEM
       pkg-config
     red-parquet (3.0.0)
       red-arrow (= 3.0.0)
-    regexp_parser (2.1.1)
+    regexp_parser (2.2.0)
     rexml (3.2.5)
     rspec (3.10.0)
       rspec-core (~> 3.10.0)
@@ -51,33 +51,32 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.17.0)
+    rspec-support (3.10.3)
+    rubocop (1.24.1)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml
-      rubocop-ast (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 1.15.1, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.7.0)
+    rubocop-ast (1.15.1)
       parser (>= 3.0.1.1)
     rubocop-bsm (0.6.0)
       rubocop (~> 1.0)
       rubocop-performance
       rubocop-rake
       rubocop-rspec
-    rubocop-performance (1.11.3)
+    rubocop-performance (1.13.1)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
-    rubocop-rake (0.5.1)
-      rubocop
-    rubocop-rspec (2.4.0)
+    rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-      rubocop-ast (>= 1.1.0)
+    rubocop-rspec (2.7.0)
+      rubocop (~> 1.19)
     ruby-progressbar (1.11.0)
-    unicode-display_width (2.0.0)
+    unicode-display_width (2.1.0)
 
 PLATFORMS
   ruby
@@ -92,4 +91,4 @@ DEPENDENCIES
   rubocop-bsm
 
 BUNDLED WITH
-   2.1.4
+   2.2.27

data/ext/parquet/decoder.go

@@ -1,170 +1,59 @@
 package parquet
 
 import (
-	"encoding/binary"
-	"fmt"
 	"io"
-	"reflect"
-	"time"
 
-	kpq "github.com/bsm/parquet-go/parquet"
+	goparquet "github.com/fraugster/parquet-go"
+	"github.com/fraugster/parquet-go/floor"
 )
 
 type decoder struct {
-	cols    []*columnReader
-	closers []io.Closer
+	pfr *goparquet.FileReader
+	ffr *floor.Reader
+	tmp *tempFile
 }
 
-func newDecoder(rs io.ReadSeeker, names []string, batchSize int) (*decoder, error) {
-	file, err := kpq.FileFromReader(rs)
+func newDecoder(rs io.ReadSeeker) (*decoder, error) {
+	pfr, err := goparquet.NewFileReader(rs)
 	if err != nil {
 		return nil, err
 	}
 
-	// normalise column names
-	if len(names) == 0 {
-		for _, c := range file.Schema.Columns() {
-			names = append(names, c.String())
-		}
-	}
-
-	// normalise batch size
-	if batchSize < 1 {
-		batchSize = 1000
-	}
+	ffr := floor.NewReader(pfr)
 
-	// initialise column buffers
-	cols := make([]*columnReader, 0, len(names))
-	for _, name := range names {
-		col, ok := file.Schema.ColumnByName(name)
-		if !ok {
-			_ = file.Close()
-			return nil, fmt.Errorf("column %q does not exist", name)
-		}
-		cols = append(cols, newColumnReader(file, col, batchSize))
-	}
-
-	return &decoder{cols: cols, closers: []io.Closer{file}}, nil
+	return &decoder{
+			pfr: pfr,
+			ffr: ffr,
+		},
+		nil
 }
 
 func (w *decoder) Decode(v interface{}) error {
-	rv := reflect.ValueOf(v)
-	rt := rv.Type()
-	if rt.Kind() != reflect.Ptr {
-		return fmt.Errorf("cannot decode non-pointer %s type", rt.String())
+	// read the next value and scan
+	if w.ffr.Next() {
+		return w.ffr.Scan(v)
 	}
 
-	// field index by name
-	fidx := cachedTypeFields(rt.Elem())
-	elem := rv.Elem()
-
-	for _, r := range w.cols {
-		// next column value
-		val, err := r.Next()
-		if err != nil {
-			return err
-		}
-
-		// skip if value is NULL
-		if val == nil {
-			continue
-		}
-
-		// set field if exists
-		if fi, ok := fidx[r.Name()]; ok {
-			fv := elem.Field(fi)
-			if ok := setValue(fv, val); !ok {
-				return fmt.Errorf("cannot assign value of type %T to %s", val, fv.Type())
-			}
-		}
+	// check for errors
+	if err := w.ffr.Err(); err != nil {
+		return err
 	}
 
-	return nil
+	// end of file
+	return io.EOF
 }
 
 func (w *decoder) Close() (err error) {
-	for _, c := range w.closers {
-		if e := c.Close(); e != nil {
+	// close the tmp file if present
+	if w.tmp != nil {
+		if e := w.tmp.Close(); e != nil {
 			err = e
 		}
 	}
-	return
-}
 
-// --------------------------------------------------------------------
-
-func setValue(rv reflect.Value, v interface{}) bool {
-	if rv.Kind() == reflect.Ptr {
-		if rv.IsNil() {
-			if ev := reflect.New(rv.Type().Elem()); setValue(ev, v) {
-				rv.Set(ev)
-				return true
-			}
-			return false
-		}
-		return setValue(rv.Elem(), v)
-	}
-
-	switch vv := v.(type) {
-	case bool:
-		switch rv.Kind() {
-		case reflect.Bool:
-			rv.SetBool(vv)
-			return true
-		}
-	case []byte:
-		switch rv.Kind() {
-		case reflect.String:
-			rv.SetString(string(vv))
-			return true
-		case reflect.Slice:
-			if rv.Type() == byteSliceType {
-				rv.SetBytes(vv)
-				return true
-			}
-		}
-	case int, int8, int16, int32, int64:
-		switch rv.Kind() {
-		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-			rv.SetInt(reflect.ValueOf(v).Int())
-			return true
-		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-			rv.SetUint(uint64(reflect.ValueOf(v).Int()))
-			return true
-		}
-	case uint, uint8, uint16, uint32, uint64:
-		switch rv.Kind() {
-		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-			rv.SetInt(int64(reflect.ValueOf(v).Uint()))
-			return true
-		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-			rv.SetUint(reflect.ValueOf(v).Uint())
-			return true
-		}
-	case float32, float64:
-		switch rv.Kind() {
-		case reflect.Float32, reflect.Float64:
-			rv.SetFloat(reflect.ValueOf(v).Float())
-			return true
-		}
-	case kpq.Int96:
-		if rt := rv.Type(); rt == timeType {
-			ns := int64(binary.LittleEndian.Uint64(vv[:8]))
-			jd := int64(binary.LittleEndian.Uint32(vv[8:]))
-			ts := time.Unix((jd-2440588)*86400, ns)
-			rv.Set(reflect.ValueOf(ts))
-			return true
-		} else if rt == int96Type {
-			rv.Set(reflect.ValueOf(v))
-			return true
-		}
+	// close the reader
+	if e := w.ffr.Close(); e != nil {
+		err = e
 	}
-
-	return false
+	return
 }
-
-var (
-	byteSliceType = reflect.TypeOf(([]byte)(nil))
-	int96Type     = reflect.TypeOf(kpq.Int96{})
-	timeType      = reflect.TypeOf(time.Time{})
-)

data/ext/parquet/decoder_test.go

@@ -24,7 +24,7 @@ var _ = Describe("Decoder", func() {
 		fixture, err = os.Open("testdata/alltypes_plain.parquet")
 		Expect(err).NotTo(HaveOccurred())
 
-		format := &parquet.Format{BatchSize: 3}
+		format := &parquet.Format{}
 		subject, err = format.NewDecoder(fixture)
 		Expect(err).NotTo(HaveOccurred())
 	})
@@ -42,7 +42,7 @@ var _ = Describe("Decoder", func() {
 			Bool:       true,
 			Float:      f32ptr(0),
 			DateString: "03/01/09", ByteString: []byte("0"),
-			Timestamp: time.Unix(1235865600, 0),
+			Timestamp: time.Unix(1235865600, 0).UTC(),
 		}))
 
 		v2 := new(mockStruct)
@@ -52,7 +52,7 @@ var _ = Describe("Decoder", func() {
 			TinyInt: 1, SmallUint: 1, StdInt: 1, BigInt: 10,
 			Float: f32ptr(1.1), Double: 10.1,
 			DateString: "03/01/09", ByteString: []byte("1"),
-			Timestamp: time.Unix(1235865660, 0),
+			Timestamp: time.Unix(1235865660, 0).UTC(),
 		}))
 
 		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v3
@@ -67,7 +67,7 @@ var _ = Describe("Decoder", func() {
 			TinyInt: 1, SmallUint: 1, StdInt: 1, BigInt: 10,
 			Float: f32ptr(1.1), Double: 10.1,
 			DateString: "02/01/09", ByteString: []byte("1"),
-			Timestamp: time.Unix(1233446460, 0),
+			Timestamp: time.Unix(1233446460, 0).UTC(),
 		}))
 
 		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v7

data/ext/parquet/encoder.go

@@ -0,0 +1,27 @@
+package parquet
+
+import (
+	"io"
+
+	goparquet "github.com/fraugster/parquet-go"
+	"github.com/fraugster/parquet-go/floor"
+)
+
+type encoder struct {
+	*floor.Writer
+}
+
+func newEncoder(w io.Writer, opts []goparquet.FileWriterOption) (*encoder, error) {
+	// create the writer
+	pw := goparquet.NewFileWriter(w, opts...)
+
+	return &encoder{
+		// wrap the parquet writer with a floor writer
+		Writer: floor.NewWriter(pw),
+	}, nil
+}
+
+// implements feedx.FormatEncoder
+func (w *encoder) Encode(v interface{}) error {
+	return w.Writer.Write(v)
+}

data/ext/parquet/encoder_test.go

@@ -0,0 +1,70 @@
+package parquet_test
+
+import (
+	"bytes"
+	"time"
+
+	"github.com/bsm/feedx"
+	"github.com/bsm/feedx/ext/parquet"
+	. "github.com/bsm/ginkgo"
+	. "github.com/bsm/gomega"
+	goparquet "github.com/fraugster/parquet-go"
+	parquetopt "github.com/fraugster/parquet-go/parquet"
+	"github.com/fraugster/parquet-go/parquetschema"
+)
+
+var _ = Describe("Encoder", func() {
+	var subject feedx.FormatEncoder
+
+	f32ptr := func(f float32) *float32 { return &f }
+
+	BeforeEach(func() {
+		var err error
+		buf := new(bytes.Buffer)
+
+		schemaDef, err := parquetschema.ParseSchemaDefinition(`message stat {
+			required int64 bigint_col;
+			optional boolean bool_col;
+			optional int32 tinyint_col;
+			optional int32 smallint_col;
+			optional int32 int_col;
+			optional int64 bigint_col;
+			optional float float_col;
+			optional double double_col;
+			optional binary date_string_col (STRING);
+			optional binary string_col;
+			optional int64 timestamp_col (TIMESTAMP(NANOS, true));
+			}`)
+		Expect(err).NotTo(HaveOccurred())
+
+		format := &parquet.Format{}
+		subject, err = format.NewEncoder(buf,
+			goparquet.WithSchemaDefinition(schemaDef),
+			goparquet.WithCompressionCodec(parquetopt.CompressionCodec_SNAPPY))
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	AfterEach(func() {
+		Expect(subject.Close()).To(Succeed())
+	})
+
+	It("encodes", func() {
+		v1 := &mockStruct{
+			ID:         1,
+			Bool:       true,
+			TinyInt:    int8(5),
+			SmallUint:  uint16(12),
+			StdInt:     5,
+			BigInt:     int64(99),
+			Float:      f32ptr(5.5),
+			Double:     float64(5.5),
+			DateString: "2021-08-11",
+			ByteString: []byte("string"),
+			Timestamp:  time.Now(),
+		}
+		Expect(subject.Encode(v1)).To(Succeed())
+
+		v2 := &mockStruct{ID: 1}
+		Expect(subject.Encode(v2)).To(Succeed())
+	})
+})

data/ext/parquet/go.mod

@@ -3,8 +3,10 @@ module github.com/bsm/feedx/ext/parquet
 go 1.15
 
 require (
-	github.com/bsm/feedx v0.12.5
-	github.com/bsm/ginkgo v1.16.1
-	github.com/bsm/gomega v1.11.0
-	github.com/bsm/parquet-go v0.0.0-20200904113737-792434824f00
+	github.com/bsm/feedx v0.0.0-00010101000000-000000000000
+	github.com/bsm/ginkgo v1.16.5
+	github.com/bsm/gomega v1.17.0
+	github.com/fraugster/parquet-go v0.6.1
 )
+
+replace github.com/bsm/feedx => ../../

data/ext/parquet/go.sum

@@ -1,24 +1,33 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y=
+github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI=
 github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k=
-github.com/bsm/bfs v0.11.3 h1:BTFCftgmuVZwwu6vyjhyKr/Pg1E+cZ5tLodj3wKxr94=
-github.com/bsm/bfs v0.11.3/go.mod h1:sUhBrbc9g0XThRRrT9hiinMhhKbkKIdhLkFljk4fuzM=
-github.com/bsm/feedx v0.12.5 h1:N751MFTRKRgoP5eG1S30EoUuHsI7aAxJ0aQZg+7xf/Y=
-github.com/bsm/feedx v0.12.5/go.mod h1:l5YNhFomuWy9du+8+hznXMH8Hug2qSMik7b5Vipcy58=
-github.com/bsm/ginkgo v1.16.0/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
-github.com/bsm/ginkgo v1.16.1 h1:jp1v1dbmbGZDWmnGXDTN+XK3U1fTTNja9xYa7VBI0l0=
-github.com/bsm/ginkgo v1.16.1/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
-github.com/bsm/gomega v1.11.0 h1:wg9DVGPETNZLIbMsseneMV1a7uo/x+wsCyNXdEcifDI=
-github.com/bsm/gomega v1.11.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
-github.com/bsm/parquet-go v0.0.0-20200904113737-792434824f00 h1:seSUCN95Kgv0mxmBUxx99faZYYQB4+TbE1Do6E5ZwI4=
-github.com/bsm/parquet-go v0.0.0-20200904113737-792434824f00/go.mod h1:7EU1rAae6Y67NlxAbr+Mbxxxc5AFGcxZCSW/vbybDLs=
-github.com/bsm/pbio v0.2.2 h1:Xdj5hQkS0K3kKc1NY6hoSWMvzpq0Mk5j8vcc7irslno=
-github.com/bsm/pbio v0.2.2/go.mod h1:3O4XQFoarlYalkGd+zMFfEUyalP8NBKkQ0Ta4IwhN4w=
+github.com/bsm/bfs v0.11.5 h1:NMWD0hOp/74boNd51Zas4DMh81mvp1As1SjEZrFWHi0=
+github.com/bsm/bfs v0.11.5/go.mod h1:D2N+ly23VsODKiRvkf4HUgKSgOaNjKTOJE53rhL6Vfk=
+github.com/bsm/ginkgo v1.16.4/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
+github.com/bsm/ginkgo v1.16.5 h1:uTeeWv0Yx1PnDeCk76PFyGrOMVw3D+r9bTNKNcIjDdQ=
+github.com/bsm/ginkgo v1.16.5/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
+github.com/bsm/gomega v1.16.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
+github.com/bsm/gomega v1.17.0 h1:Sd6EsHO5d0DU6d41dtx9cK3T7Vjsr89o6zyIVWgi0CI=
+github.com/bsm/gomega v1.17.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
+github.com/bsm/pbio v0.3.0 h1:8XzkUPoJZblPkLp2jHVMnIhGs5WaNHTCmO7gRC45izk=
+github.com/bsm/pbio v0.3.0/go.mod h1:3O4XQFoarlYalkGd+zMFfEUyalP8NBKkQ0Ta4IwhN4w=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fraugster/parquet-go v0.6.1 h1:Kks9Ibly3ZZQPRPGkS1lVmrwndBp8PxamBnDFG5jvEM=
+github.com/fraugster/parquet-go v0.6.1/go.mod h1:1HGhXzpHv7CULzknVNWIY0Ihn2O3qNbD1p+aQvHWhqo=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
@@ -26,6 +35,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
@@ -35,8 +45,9 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
@@ -46,7 +57,9 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@@ -54,6 +67,9 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -64,9 +80,29 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
 github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
@@ -95,6 +131,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -139,14 +176,16 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

data/ext/parquet/parquet.go

@@ -1,25 +1,25 @@
 package parquet
 
 import (
-	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 
 	"github.com/bsm/feedx"
+	goparquet "github.com/fraugster/parquet-go"
 )
 
+// --------------------------------------------------------------------
+
 // Format is a parquet format.
 type Format struct {
-	TempDir   string
-	Columns   []string // column names to include
-	BatchSize int      // batch size, default: 1,000
+	TempDir string
 }
 
 // NewDecoder implements Format.
 func (f *Format) NewDecoder(r io.Reader) (feedx.FormatDecoder, error) {
 	if rs, ok := r.(io.ReadSeeker); ok {
-		return newDecoder(rs, f.Columns, f.BatchSize)
+		return newDecoder(rs)
 	}
 
 	tmp, err := copyToTempFile(f.TempDir, r)
@@ -27,18 +27,18 @@ func (f *Format) NewDecoder(r io.Reader) (feedx.FormatDecoder, error) {
 		return nil, err
 	}
 
-	dec, err := newDecoder(tmp, f.Columns, f.BatchSize)
+	dec, err := newDecoder(tmp)
 	if err != nil {
 		_ = tmp.Close()
 		return nil, err
 	}
-	dec.closers = append(dec.closers, tmp)
+	dec.tmp = tmp
 	return dec, nil
 }
 
 // NewEncoder implements Format.
-func (*Format) NewEncoder(w io.Writer) (feedx.FormatEncoder, error) {
-	return nil, fmt.Errorf("not implemented")
+func (f *Format) NewEncoder(w io.Writer, opts ...goparquet.FileWriterOption) (feedx.FormatEncoder, error) {
+	return newEncoder(w, opts)
 }
 
 // --------------------------------------------------------------------
@@ -69,7 +69,7 @@ func copyToTempFile(dir string, r io.Reader) (*tempFile, error) {
 	return &tempFile{File: f}, nil
 }
 
-func (f tempFile) Close() error {
+func (f *tempFile) Close() error {
 	err := f.File.Close()
 	if e := os.Remove(f.Name()); e != nil {
 		err = e

data/feedx.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'feedx'
-  s.version       = '0.12.6'
+  s.version       = '0.12.7'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Exchange data between components via feeds)
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files -z`.split("\x0").reject {|f| f.start_with?('spec/') }
   s.test_files    = `git ls-files -z -- spec/*`.split("\x0")
   s.require_paths = ['lib']
-  s.required_ruby_version = '>= 2.6'
+  s.required_ruby_version = '>= 2.7'
 
   s.add_dependency 'bfs', '>= 0.8.0'
 
@@ -21,4 +21,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'red-parquet', '>= 3.0', '< 4.0'
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'rubocop-bsm'
+  s.metadata['rubygems_mfa_required'] = 'true'
 end

data/feedx_test.go

@@ -48,6 +48,8 @@ func writeMulti(obj *bfs.Object, numEntries int, lastMod time.Time) error {
 	return w.Commit()
 }
 
+// ------------------------------------------------------------------------
+
 func TestSuite(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "feedx")

data/go.mod

@@ -3,13 +3,11 @@ module github.com/bsm/feedx
 go 1.15
 
 require (
-	github.com/bsm/bfs v0.11.3
-	github.com/bsm/ginkgo v1.16.0
-	github.com/bsm/gomega v1.11.0
-	github.com/bsm/pbio v0.2.2
+	github.com/bsm/bfs v0.11.5
+	github.com/bsm/ginkgo v1.16.5
+	github.com/bsm/gomega v1.17.0
+	github.com/bsm/pbio v0.3.0
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/protobuf v1.5.0
-	github.com/kr/pretty v0.1.0 // indirect
-	google.golang.org/protobuf v1.26.0
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	github.com/golang/protobuf v1.5.2
+	google.golang.org/protobuf v1.27.1
 )

data/go.sum

@@ -2,14 +2,16 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI=
 github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k=
-github.com/bsm/bfs v0.11.3 h1:BTFCftgmuVZwwu6vyjhyKr/Pg1E+cZ5tLodj3wKxr94=
-github.com/bsm/bfs v0.11.3/go.mod h1:sUhBrbc9g0XThRRrT9hiinMhhKbkKIdhLkFljk4fuzM=
-github.com/bsm/ginkgo v1.16.0 h1:d0FSr//fhUpdWr4uBHBLG+xN1bOYWERSBnfu1hGknfU=
-github.com/bsm/ginkgo v1.16.0/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
-github.com/bsm/gomega v1.11.0 h1:wg9DVGPETNZLIbMsseneMV1a7uo/x+wsCyNXdEcifDI=
-github.com/bsm/gomega v1.11.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
-github.com/bsm/pbio v0.2.2 h1:Xdj5hQkS0K3kKc1NY6hoSWMvzpq0Mk5j8vcc7irslno=
-github.com/bsm/pbio v0.2.2/go.mod h1:3O4XQFoarlYalkGd+zMFfEUyalP8NBKkQ0Ta4IwhN4w=
+github.com/bsm/bfs v0.11.5 h1:NMWD0hOp/74boNd51Zas4DMh81mvp1As1SjEZrFWHi0=
+github.com/bsm/bfs v0.11.5/go.mod h1:D2N+ly23VsODKiRvkf4HUgKSgOaNjKTOJE53rhL6Vfk=
+github.com/bsm/ginkgo v1.16.4/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
+github.com/bsm/ginkgo v1.16.5 h1:uTeeWv0Yx1PnDeCk76PFyGrOMVw3D+r9bTNKNcIjDdQ=
+github.com/bsm/ginkgo v1.16.5/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
+github.com/bsm/gomega v1.16.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
+github.com/bsm/gomega v1.17.0 h1:Sd6EsHO5d0DU6d41dtx9cK3T7Vjsr89o6zyIVWgi0CI=
+github.com/bsm/gomega v1.17.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
+github.com/bsm/pbio v0.3.0 h1:8XzkUPoJZblPkLp2jHVMnIhGs5WaNHTCmO7gRC45izk=
+github.com/bsm/pbio v0.3.0/go.mod h1:3O4XQFoarlYalkGd+zMFfEUyalP8NBKkQ0Ta4IwhN4w=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -30,8 +32,9 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -42,11 +45,6 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
@@ -132,11 +130,11 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.12.6
+  version: 0.12.7
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-06-18 00:00:00.000000000 Z
+date: 2022-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -122,8 +122,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".editorconfig"
+- ".github/workflows/lint.yml"
 - ".github/workflows/test.yml"
 - ".gitignore"
+- ".golangci.yml"
 - ".rubocop.yml"
 - Gemfile
 - Gemfile.lock
@@ -137,13 +139,13 @@ files:
 - consumer_test.go
 - ext/parquet/decoder.go
 - ext/parquet/decoder_test.go
+- ext/parquet/encoder.go
+- ext/parquet/encoder_test.go
 - ext/parquet/go.mod
 - ext/parquet/go.sum
 - ext/parquet/parquet.go
 - ext/parquet/parquet_test.go
-- ext/parquet/reader.go
 - ext/parquet/testdata/alltypes_plain.parquet
-- ext/parquet/types.go
 - feedx.gemspec
 - feedx.go
 - feedx_ext_test.go
@@ -195,7 +197,8 @@ files:
 homepage: https://github.com/bsm/feedx
 licenses:
 - Apache-2.0
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -204,14 +207,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.15
 signing_key: 
 specification_version: 4
 summary: Exchange data between components via feeds

data/ext/parquet/reader.go

@@ -1,89 +0,0 @@
-package parquet
-
-import (
-	"io"
-
-	kpq "github.com/bsm/parquet-go/parquet"
-)
-
-type columnReader struct {
-	file     *kpq.File
-	col      kpq.Column
-	rowGroup int
-
-	chunk   *kpq.ColumnChunkReader
-	values  []interface{}
-	dLevels []uint16
-	rLevels []uint16
-
-	n, i, vi int // chunk stats
-}
-
-func newColumnReader(file *kpq.File, col kpq.Column, batchSize int) *columnReader {
-	return &columnReader{
-		file:    file,
-		col:     col,
-		values:  make([]interface{}, batchSize),
-		dLevels: make([]uint16, batchSize),
-		rLevels: make([]uint16, batchSize),
-	}
-}
-
-func (c *columnReader) Name() string {
-	return c.col.String()
-}
-
-func (c *columnReader) Next() (interface{}, error) {
-	if err := c.ensureChunk(); err != nil {
-		return nil, err
-	}
-
-	if err := c.ensureValues(); err == kpq.EndOfChunk {
-		c.chunk = nil
-		return c.Next()
-	} else if err != nil {
-		return nil, err
-	}
-
-	dLevel := c.dLevels[c.i]
-	c.i++
-
-	if notNull := dLevel == c.col.MaxD(); notNull {
-		val := c.values[c.vi]
-		c.vi++
-		return val, nil
-	}
-
-	return nil, nil
-}
-
-func (c *columnReader) ensureChunk() error {
-	if c.chunk != nil {
-		return nil
-	}
-	if c.rowGroup >= len(c.file.MetaData.RowGroups) {
-		return io.EOF
-	}
-
-	rd, err := c.file.NewReader(c.col, c.rowGroup)
-	if err != nil {
-		return err
-	}
-	c.chunk = rd
-	c.rowGroup++
-	return nil
-}
-
-func (c *columnReader) ensureValues() error {
-	if c.n != 0 && c.i < c.n {
-		return nil
-	}
-
-	n, err := c.chunk.Read(c.values, c.dLevels, c.rLevels)
-	if err != nil {
-		return err
-	}
-
-	c.n, c.i, c.vi = n, 0, 0
-	return nil
-}

data/ext/parquet/types.go

@@ -1,51 +0,0 @@
-package parquet
-
-import (
-	"reflect"
-	"strings"
-	"sync"
-)
-
-type structFields map[string]int
-
-var fieldCache sync.Map // map[reflect.Type]structFields
-
-// cachedTypeFields is like typeFields but uses a cache to avoid repeated work.
-//
-// "Inspired" by https://golang.org/src/encoding/json/encode.go
-// Copyright 2010 The Go Authors. All rights reserved.
-func cachedTypeFields(t reflect.Type) structFields {
-	if f, ok := fieldCache.Load(t); ok {
-		return f.(structFields)
-	}
-
-	f, _ := fieldCache.LoadOrStore(t, typeFields(t))
-	return f.(structFields)
-}
-
-func tagName(tag string) string {
-	if pos := strings.Index(tag, ","); pos != -1 {
-		return tag[:pos]
-	}
-	return tag
-}
-
-// "Inspired" by https://golang.org/src/encoding/json/encode.go
-// Copyright 2010 The Go Authors. All rights reserved.
-func typeFields(t reflect.Type) structFields {
-	index := make(map[string]int, t.NumField())
-	for i := 0; i < t.NumField(); i++ {
-		field := t.Field(i)
-		tag := field.Tag.Get("parquet")
-		if tag == "-" {
-			continue
-		}
-
-		name := field.Name
-		if s := tagName(tag); s != "" {
-			name = s
-		}
-		index[name] = i
-	}
-	return index
-}