feedx 0.12.7 → 0.14.0

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: feedx
 version: !ruby/object:Gem::Version
-  version: 0.12.7
+  version: 0.14.0
 platform: ruby
 authors:
 - Black Square Media Ltd
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-10 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bfs
@@ -24,96 +23,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.8.0
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: pbio
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: red-parquet
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '3.0'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '4.0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop-bsm
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Use feeds to exchange data between (micro-)services.
 email:
 - info@blacksquaremedia.com
@@ -122,11 +31,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".editorconfig"
-- ".github/workflows/lint.yml"
 - ".github/workflows/test.yml"
 - ".gitignore"
 - ".golangci.yml"
 - ".rubocop.yml"
+- ".tool-versions"
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -137,15 +46,7 @@ files:
 - compression_test.go
 - consumer.go
 - consumer_test.go
-- ext/parquet/decoder.go
-- ext/parquet/decoder_test.go
-- ext/parquet/encoder.go
-- ext/parquet/encoder_test.go
-- ext/parquet/go.mod
-- ext/parquet/go.sum
-- ext/parquet/parquet.go
-- ext/parquet/parquet_test.go
-- ext/parquet/testdata/alltypes_plain.parquet
+- example_test.go
 - feedx.gemspec
 - feedx.go
 - feedx_ext_test.go
@@ -154,6 +55,8 @@ files:
 - format_test.go
 - go.mod
 - go.sum
+- incremental.go
+- incremental_test.go
 - internal/testdata/testdata.pb.go
 - internal/testdata/testdata.proto
 - lib/feedx.rb
@@ -169,29 +72,17 @@ files:
 - lib/feedx/format.rb
 - lib/feedx/format/abstract.rb
 - lib/feedx/format/json.rb
-- lib/feedx/format/parquet.rb
 - lib/feedx/format/protobuf.rb
 - lib/feedx/producer.rb
 - lib/feedx/pusher.rb
 - lib/feedx/stream.rb
+- manifest.go
 - producer.go
 - producer_test.go
 - reader.go
 - reader_test.go
-- spec/feedx/cache/memory_spec.rb
-- spec/feedx/cache/value_spec.rb
-- spec/feedx/compression/gzip_spec.rb
-- spec/feedx/compression/none_spec.rb
-- spec/feedx/compression_spec.rb
-- spec/feedx/consumer_spec.rb
-- spec/feedx/format/abstract_spec.rb
-- spec/feedx/format/json_spec.rb
-- spec/feedx/format/parquet_spec.rb
-- spec/feedx/format/protobuf_spec.rb
-- spec/feedx/format_spec.rb
-- spec/feedx/producer_spec.rb
-- spec/feedx/stream_spec.rb
-- spec/spec_helper.rb
+- scheduler.go
+- scheduler_test.go
 - writer.go
 - writer_test.go
 homepage: https://github.com/bsm/feedx
@@ -199,7 +90,6 @@ licenses:
 - Apache-2.0
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -207,29 +97,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
-signing_key: 
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Exchange data between components via feeds
-test_files:
-- spec/feedx/cache/memory_spec.rb
-- spec/feedx/cache/value_spec.rb
-- spec/feedx/compression/gzip_spec.rb
-- spec/feedx/compression/none_spec.rb
-- spec/feedx/compression_spec.rb
-- spec/feedx/consumer_spec.rb
-- spec/feedx/format/abstract_spec.rb
-- spec/feedx/format/json_spec.rb
-- spec/feedx/format/parquet_spec.rb
-- spec/feedx/format/protobuf_spec.rb
-- spec/feedx/format_spec.rb
-- spec/feedx/producer_spec.rb
-- spec/feedx/stream_spec.rb
-- spec/spec_helper.rb
+test_files: []

data/.github/workflows/lint.yml

@@ -1,18 +0,0 @@
-name: Lint
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-jobs:
-  golangci:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Run lint
-        uses: golangci/golangci-lint-action@v2
-        with:
-          version: latest

data/ext/parquet/decoder.go

@@ -1,59 +0,0 @@
-package parquet
-
-import (
-	"io"
-
-	goparquet "github.com/fraugster/parquet-go"
-	"github.com/fraugster/parquet-go/floor"
-)
-
-type decoder struct {
-	pfr *goparquet.FileReader
-	ffr *floor.Reader
-	tmp *tempFile
-}
-
-func newDecoder(rs io.ReadSeeker) (*decoder, error) {
-	pfr, err := goparquet.NewFileReader(rs)
-	if err != nil {
-		return nil, err
-	}
-
-	ffr := floor.NewReader(pfr)
-
-	return &decoder{
-			pfr: pfr,
-			ffr: ffr,
-		},
-		nil
-}
-
-func (w *decoder) Decode(v interface{}) error {
-	// read the next value and scan
-	if w.ffr.Next() {
-		return w.ffr.Scan(v)
-	}
-
-	// check for errors
-	if err := w.ffr.Err(); err != nil {
-		return err
-	}
-
-	// end of file
-	return io.EOF
-}
-
-func (w *decoder) Close() (err error) {
-	// close the tmp file if present
-	if w.tmp != nil {
-		if e := w.tmp.Close(); e != nil {
-			err = e
-		}
-	}
-
-	// close the reader
-	if e := w.ffr.Close(); e != nil {
-		err = e
-	}
-	return
-}

data/ext/parquet/decoder_test.go

@@ -1,88 +0,0 @@
-package parquet_test
-
-import (
-	"bytes"
-	"io"
-	"io/ioutil"
-	"os"
-	"time"
-
-	"github.com/bsm/feedx"
-	"github.com/bsm/feedx/ext/parquet"
-	. "github.com/bsm/ginkgo"
-	. "github.com/bsm/gomega"
-)
-
-var _ = Describe("Decoder", func() {
-	var subject feedx.FormatDecoder
-	var fixture *os.File
-
-	f32ptr := func(f float32) *float32 { return &f }
-
-	BeforeEach(func() {
-		var err error
-		fixture, err = os.Open("testdata/alltypes_plain.parquet")
-		Expect(err).NotTo(HaveOccurred())
-
-		format := &parquet.Format{}
-		subject, err = format.NewDecoder(fixture)
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	AfterEach(func() {
-		Expect(subject.Close()).To(Succeed())
-		Expect(fixture.Close()).To(Succeed())
-	})
-
-	It("decodes", func() {
-		v1 := new(mockStruct)
-		Expect(subject.Decode(v1)).To(Succeed())
-		Expect(v1).To(Equal(&mockStruct{
-			ID:         4,
-			Bool:       true,
-			Float:      f32ptr(0),
-			DateString: "03/01/09", ByteString: []byte("0"),
-			Timestamp: time.Unix(1235865600, 0).UTC(),
-		}))
-
-		v2 := new(mockStruct)
-		Expect(subject.Decode(v2)).To(Succeed())
-		Expect(v2).To(Equal(&mockStruct{
-			ID:      5,
-			TinyInt: 1, SmallUint: 1, StdInt: 1, BigInt: 10,
-			Float: f32ptr(1.1), Double: 10.1,
-			DateString: "03/01/09", ByteString: []byte("1"),
-			Timestamp: time.Unix(1235865660, 0).UTC(),
-		}))
-
-		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v3
-		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v4
-		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v5
-
-		v6 := new(mockStruct)
-		Expect(subject.Decode(v6)).To(Succeed())
-		Expect(v6).To(Equal(&mockStruct{
-			ID:      3,
-			Bool:    false,
-			TinyInt: 1, SmallUint: 1, StdInt: 1, BigInt: 10,
-			Float: f32ptr(1.1), Double: 10.1,
-			DateString: "02/01/09", ByteString: []byte("1"),
-			Timestamp: time.Unix(1233446460, 0).UTC(),
-		}))
-
-		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v7
-		Expect(subject.Decode(new(mockStruct))).To(Succeed()) // v8
-
-		v9 := new(mockStruct)
-		Expect(subject.Decode(v9)).To(MatchError(io.EOF))
-	})
-
-	It("opens from non-file readers", func() {
-		bin, err := ioutil.ReadFile("testdata/alltypes_plain.parquet")
-		Expect(err).NotTo(HaveOccurred())
-
-		dec, err := new(parquet.Format).NewDecoder(bytes.NewReader(bin))
-		Expect(err).NotTo(HaveOccurred())
-		Expect(dec.Close()).To(Succeed())
-	})
-})

data/ext/parquet/encoder.go

@@ -1,27 +0,0 @@
-package parquet
-
-import (
-	"io"
-
-	goparquet "github.com/fraugster/parquet-go"
-	"github.com/fraugster/parquet-go/floor"
-)
-
-type encoder struct {
-	*floor.Writer
-}
-
-func newEncoder(w io.Writer, opts []goparquet.FileWriterOption) (*encoder, error) {
-	// create the writer
-	pw := goparquet.NewFileWriter(w, opts...)
-
-	return &encoder{
-		// wrap the parquet writer with a floor writer
-		Writer: floor.NewWriter(pw),
-	}, nil
-}
-
-// implements feedx.FormatEncoder
-func (w *encoder) Encode(v interface{}) error {
-	return w.Writer.Write(v)
-}

data/ext/parquet/encoder_test.go

@@ -1,70 +0,0 @@
-package parquet_test
-
-import (
-	"bytes"
-	"time"
-
-	"github.com/bsm/feedx"
-	"github.com/bsm/feedx/ext/parquet"
-	. "github.com/bsm/ginkgo"
-	. "github.com/bsm/gomega"
-	goparquet "github.com/fraugster/parquet-go"
-	parquetopt "github.com/fraugster/parquet-go/parquet"
-	"github.com/fraugster/parquet-go/parquetschema"
-)
-
-var _ = Describe("Encoder", func() {
-	var subject feedx.FormatEncoder
-
-	f32ptr := func(f float32) *float32 { return &f }
-
-	BeforeEach(func() {
-		var err error
-		buf := new(bytes.Buffer)
-
-		schemaDef, err := parquetschema.ParseSchemaDefinition(`message stat {
-			required int64 bigint_col;
-			optional boolean bool_col;
-			optional int32 tinyint_col;
-			optional int32 smallint_col;
-			optional int32 int_col;
-			optional int64 bigint_col;
-			optional float float_col;
-			optional double double_col;
-			optional binary date_string_col (STRING);
-			optional binary string_col;
-			optional int64 timestamp_col (TIMESTAMP(NANOS, true));
-			}`)
-		Expect(err).NotTo(HaveOccurred())
-
-		format := &parquet.Format{}
-		subject, err = format.NewEncoder(buf,
-			goparquet.WithSchemaDefinition(schemaDef),
-			goparquet.WithCompressionCodec(parquetopt.CompressionCodec_SNAPPY))
-		Expect(err).NotTo(HaveOccurred())
-	})
-
-	AfterEach(func() {
-		Expect(subject.Close()).To(Succeed())
-	})
-
-	It("encodes", func() {
-		v1 := &mockStruct{
-			ID:         1,
-			Bool:       true,
-			TinyInt:    int8(5),
-			SmallUint:  uint16(12),
-			StdInt:     5,
-			BigInt:     int64(99),
-			Float:      f32ptr(5.5),
-			Double:     float64(5.5),
-			DateString: "2021-08-11",
-			ByteString: []byte("string"),
-			Timestamp:  time.Now(),
-		}
-		Expect(subject.Encode(v1)).To(Succeed())
-
-		v2 := &mockStruct{ID: 1}
-		Expect(subject.Encode(v2)).To(Succeed())
-	})
-})

data/ext/parquet/go.mod

@@ -1,12 +0,0 @@
-module github.com/bsm/feedx/ext/parquet
-
-go 1.15
-
-require (
-	github.com/bsm/feedx v0.0.0-00010101000000-000000000000
-	github.com/bsm/ginkgo v1.16.5
-	github.com/bsm/gomega v1.17.0
-	github.com/fraugster/parquet-go v0.6.1
-)
-
-replace github.com/bsm/feedx => ../../