feedx 0.12.7 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +2 -37
- data/.golangci.yml +13 -4
- data/.rubocop.yml +8 -14
- data/.tool-versions +1 -0
- data/Gemfile +8 -0
- data/Gemfile.lock +54 -68
- data/Makefile +3 -3
- data/README.md +3 -1
- data/compression.go +29 -0
- data/compression_test.go +73 -61
- data/consumer.go +96 -152
- data/consumer_test.go +124 -59
- data/example_test.go +140 -0
- data/feedx.gemspec +2 -10
- data/feedx.go +16 -31
- data/feedx_ext_test.go +13 -3
- data/feedx_test.go +24 -26
- data/format.go +29 -19
- data/format_test.go +84 -56
- data/go.mod +11 -7
- data/go.sum +16 -138
- data/incremental.go +122 -0
- data/incremental_test.go +62 -0
- data/lib/feedx/cache/abstract.rb +3 -3
- data/lib/feedx/cache/value.rb +6 -6
- data/lib/feedx/compression/abstract.rb +2 -2
- data/lib/feedx/compression/gzip.rb +4 -4
- data/lib/feedx/consumer.rb +8 -8
- data/lib/feedx/format/abstract.rb +6 -6
- data/lib/feedx/format/json.rb +2 -2
- data/lib/feedx/format/protobuf.rb +6 -6
- data/lib/feedx/format.rb +1 -3
- data/lib/feedx/producer.rb +11 -11
- data/lib/feedx/stream.rb +2 -2
- data/lib/feedx.rb +2 -3
- data/manifest.go +65 -0
- data/producer.go +34 -137
- data/producer_test.go +46 -60
- data/reader.go +142 -41
- data/reader_test.go +86 -35
- data/scheduler.go +176 -0
- data/scheduler_test.go +128 -0
- data/writer.go +13 -13
- data/writer_test.go +61 -44
- metadata +12 -137
- data/.github/workflows/lint.yml +0 -18
- data/ext/parquet/decoder.go +0 -59
- data/ext/parquet/decoder_test.go +0 -88
- data/ext/parquet/encoder.go +0 -27
- data/ext/parquet/encoder_test.go +0 -70
- data/ext/parquet/go.mod +0 -12
- data/ext/parquet/go.sum +0 -193
- data/ext/parquet/parquet.go +0 -78
- data/ext/parquet/parquet_test.go +0 -28
- data/ext/parquet/testdata/alltypes_plain.parquet +0 -0
- data/lib/feedx/format/parquet.rb +0 -102
- data/spec/feedx/cache/memory_spec.rb +0 -23
- data/spec/feedx/cache/value_spec.rb +0 -19
- data/spec/feedx/compression/gzip_spec.rb +0 -17
- data/spec/feedx/compression/none_spec.rb +0 -15
- data/spec/feedx/compression_spec.rb +0 -19
- data/spec/feedx/consumer_spec.rb +0 -49
- data/spec/feedx/format/abstract_spec.rb +0 -21
- data/spec/feedx/format/json_spec.rb +0 -27
- data/spec/feedx/format/parquet_spec.rb +0 -30
- data/spec/feedx/format/protobuf_spec.rb +0 -23
- data/spec/feedx/format_spec.rb +0 -21
- data/spec/feedx/producer_spec.rb +0 -74
- data/spec/feedx/stream_spec.rb +0 -109
- data/spec/spec_helper.rb +0 -57
data/ext/parquet/go.sum
DELETED
|
@@ -1,193 +0,0 @@
|
|
|
1
|
-
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
|
2
|
-
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
|
3
|
-
github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y=
|
|
4
|
-
github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
|
|
5
|
-
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
|
|
6
|
-
github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI=
|
|
7
|
-
github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k=
|
|
8
|
-
github.com/bsm/bfs v0.11.5 h1:NMWD0hOp/74boNd51Zas4DMh81mvp1As1SjEZrFWHi0=
|
|
9
|
-
github.com/bsm/bfs v0.11.5/go.mod h1:D2N+ly23VsODKiRvkf4HUgKSgOaNjKTOJE53rhL6Vfk=
|
|
10
|
-
github.com/bsm/ginkgo v1.16.4/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
|
|
11
|
-
github.com/bsm/ginkgo v1.16.5 h1:uTeeWv0Yx1PnDeCk76PFyGrOMVw3D+r9bTNKNcIjDdQ=
|
|
12
|
-
github.com/bsm/ginkgo v1.16.5/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
|
|
13
|
-
github.com/bsm/gomega v1.16.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
|
|
14
|
-
github.com/bsm/gomega v1.17.0 h1:Sd6EsHO5d0DU6d41dtx9cK3T7Vjsr89o6zyIVWgi0CI=
|
|
15
|
-
github.com/bsm/gomega v1.17.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
|
|
16
|
-
github.com/bsm/pbio v0.3.0 h1:8XzkUPoJZblPkLp2jHVMnIhGs5WaNHTCmO7gRC45izk=
|
|
17
|
-
github.com/bsm/pbio v0.3.0/go.mod h1:3O4XQFoarlYalkGd+zMFfEUyalP8NBKkQ0Ta4IwhN4w=
|
|
18
|
-
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
|
19
|
-
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
|
20
|
-
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
|
|
21
|
-
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
|
|
22
|
-
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
|
23
|
-
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
|
|
24
|
-
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
25
|
-
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
|
26
|
-
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
|
27
|
-
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
|
28
|
-
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
|
29
|
-
github.com/fraugster/parquet-go v0.6.1 h1:Kks9Ibly3ZZQPRPGkS1lVmrwndBp8PxamBnDFG5jvEM=
|
|
30
|
-
github.com/fraugster/parquet-go v0.6.1/go.mod h1:1HGhXzpHv7CULzknVNWIY0Ihn2O3qNbD1p+aQvHWhqo=
|
|
31
|
-
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
|
32
|
-
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
|
33
|
-
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
|
34
|
-
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
|
35
|
-
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
|
36
|
-
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
|
37
|
-
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
|
38
|
-
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
|
|
39
|
-
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
|
40
|
-
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
|
41
|
-
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
|
42
|
-
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
|
43
|
-
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
|
44
|
-
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
|
45
|
-
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
|
46
|
-
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
|
47
|
-
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
|
48
|
-
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
|
49
|
-
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
|
|
50
|
-
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
|
51
|
-
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
|
52
|
-
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
|
53
|
-
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
|
54
|
-
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
|
55
|
-
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
|
56
|
-
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
|
57
|
-
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
|
58
|
-
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
|
59
|
-
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
|
60
|
-
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
|
|
61
|
-
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
|
62
|
-
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
|
63
|
-
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
|
64
|
-
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
|
65
|
-
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
|
66
|
-
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
|
67
|
-
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
|
68
|
-
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
|
69
|
-
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
|
70
|
-
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
|
71
|
-
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
|
72
|
-
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
|
73
|
-
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
|
74
|
-
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
|
75
|
-
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
|
76
|
-
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
|
77
|
-
github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
|
|
78
|
-
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
|
79
|
-
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
|
80
|
-
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
|
81
|
-
github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
|
|
82
|
-
github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
|
83
|
-
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
|
84
|
-
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
|
85
|
-
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
|
86
|
-
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
|
87
|
-
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
|
88
|
-
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
|
89
|
-
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
|
|
90
|
-
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
|
|
91
|
-
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
|
|
92
|
-
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
|
|
93
|
-
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
|
|
94
|
-
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
|
95
|
-
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
|
96
|
-
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
|
|
97
|
-
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
|
98
|
-
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
|
99
|
-
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
|
100
|
-
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
|
101
|
-
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
|
|
102
|
-
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
|
103
|
-
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
|
104
|
-
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
|
105
|
-
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
|
106
|
-
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
|
107
|
-
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
|
108
|
-
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
|
109
|
-
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
|
110
|
-
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
|
111
|
-
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
|
112
|
-
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
|
113
|
-
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
|
114
|
-
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
|
115
|
-
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
116
|
-
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
117
|
-
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
118
|
-
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
|
119
|
-
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
|
120
|
-
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
|
121
|
-
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
|
122
|
-
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
|
123
|
-
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
|
124
|
-
golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
|
|
125
|
-
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
|
126
|
-
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
|
127
|
-
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
128
|
-
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
129
|
-
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
130
|
-
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
131
|
-
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
|
132
|
-
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
133
|
-
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
134
|
-
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
135
|
-
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
|
136
|
-
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
137
|
-
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
138
|
-
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
139
|
-
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
140
|
-
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
141
|
-
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
142
|
-
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
|
|
143
|
-
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
|
144
|
-
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
|
145
|
-
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
|
146
|
-
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
|
147
|
-
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
|
148
|
-
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
|
149
|
-
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
|
150
|
-
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
|
151
|
-
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
|
152
|
-
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
|
153
|
-
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
|
154
|
-
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
|
155
|
-
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
|
156
|
-
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
157
|
-
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
158
|
-
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
159
|
-
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
|
|
160
|
-
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
|
161
|
-
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
|
162
|
-
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
|
163
|
-
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
|
164
|
-
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
|
165
|
-
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
|
166
|
-
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
|
167
|
-
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
|
168
|
-
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
|
169
|
-
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
|
170
|
-
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
|
171
|
-
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
|
172
|
-
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
|
173
|
-
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
|
174
|
-
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
|
175
|
-
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
|
176
|
-
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
|
177
|
-
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
|
178
|
-
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
|
179
|
-
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
|
180
|
-
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
|
|
181
|
-
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
|
182
|
-
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
183
|
-
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
|
184
|
-
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
|
185
|
-
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
|
186
|
-
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
|
187
|
-
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
|
188
|
-
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
189
|
-
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
190
|
-
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
|
191
|
-
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|
192
|
-
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
|
193
|
-
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
data/ext/parquet/parquet.go
DELETED
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
package parquet
|
|
2
|
-
|
|
3
|
-
import (
|
|
4
|
-
"io"
|
|
5
|
-
"io/ioutil"
|
|
6
|
-
"os"
|
|
7
|
-
|
|
8
|
-
"github.com/bsm/feedx"
|
|
9
|
-
goparquet "github.com/fraugster/parquet-go"
|
|
10
|
-
)
|
|
11
|
-
|
|
12
|
-
// --------------------------------------------------------------------
|
|
13
|
-
|
|
14
|
-
// Format is a parquet format.
|
|
15
|
-
type Format struct {
|
|
16
|
-
TempDir string
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
// NewDecoder implements Format.
|
|
20
|
-
func (f *Format) NewDecoder(r io.Reader) (feedx.FormatDecoder, error) {
|
|
21
|
-
if rs, ok := r.(io.ReadSeeker); ok {
|
|
22
|
-
return newDecoder(rs)
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
tmp, err := copyToTempFile(f.TempDir, r)
|
|
26
|
-
if err != nil {
|
|
27
|
-
return nil, err
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
dec, err := newDecoder(tmp)
|
|
31
|
-
if err != nil {
|
|
32
|
-
_ = tmp.Close()
|
|
33
|
-
return nil, err
|
|
34
|
-
}
|
|
35
|
-
dec.tmp = tmp
|
|
36
|
-
return dec, nil
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
// NewEncoder implements Format.
|
|
40
|
-
func (f *Format) NewEncoder(w io.Writer, opts ...goparquet.FileWriterOption) (feedx.FormatEncoder, error) {
|
|
41
|
-
return newEncoder(w, opts)
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// --------------------------------------------------------------------
|
|
45
|
-
|
|
46
|
-
type tempFile struct{ *os.File }
|
|
47
|
-
|
|
48
|
-
func copyToTempFile(dir string, r io.Reader) (*tempFile, error) {
|
|
49
|
-
w, err := ioutil.TempFile(dir, "feedx-ext-parquet")
|
|
50
|
-
if err != nil {
|
|
51
|
-
return nil, err
|
|
52
|
-
}
|
|
53
|
-
if _, err := io.Copy(w, r); err != nil {
|
|
54
|
-
_ = w.Close()
|
|
55
|
-
_ = os.Remove(w.Name())
|
|
56
|
-
return nil, err
|
|
57
|
-
}
|
|
58
|
-
if err := w.Close(); err != nil {
|
|
59
|
-
_ = os.Remove(w.Name())
|
|
60
|
-
return nil, err
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
f, err := os.Open(w.Name())
|
|
64
|
-
if err != nil {
|
|
65
|
-
_ = os.Remove(w.Name())
|
|
66
|
-
return nil, err
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
return &tempFile{File: f}, nil
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
func (f *tempFile) Close() error {
|
|
73
|
-
err := f.File.Close()
|
|
74
|
-
if e := os.Remove(f.Name()); e != nil {
|
|
75
|
-
err = e
|
|
76
|
-
}
|
|
77
|
-
return err
|
|
78
|
-
}
|
data/ext/parquet/parquet_test.go
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
package parquet_test
|
|
2
|
-
|
|
3
|
-
import (
|
|
4
|
-
"testing"
|
|
5
|
-
"time"
|
|
6
|
-
|
|
7
|
-
. "github.com/bsm/ginkgo"
|
|
8
|
-
. "github.com/bsm/gomega"
|
|
9
|
-
)
|
|
10
|
-
|
|
11
|
-
type mockStruct struct {
|
|
12
|
-
ID int `parquet:"id"`
|
|
13
|
-
Bool bool `parquet:"bool_col"`
|
|
14
|
-
TinyInt int8 `parquet:"tinyint_col"`
|
|
15
|
-
SmallUint uint16 `parquet:"smallint_col"`
|
|
16
|
-
StdInt int `parquet:"int_col"`
|
|
17
|
-
BigInt int64 `parquet:"bigint_col"`
|
|
18
|
-
Float *float32 `parquet:"float_col"`
|
|
19
|
-
Double float64 `parquet:"double_col"`
|
|
20
|
-
DateString string `parquet:"date_string_col"`
|
|
21
|
-
ByteString []byte `parquet:"string_col"`
|
|
22
|
-
Timestamp time.Time `parquet:"timestamp_col"`
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
func TestSuite(t *testing.T) {
|
|
26
|
-
RegisterFailHandler(Fail)
|
|
27
|
-
RunSpecs(t, "feedx/ext/parquet")
|
|
28
|
-
}
|
|
Binary file
|
data/lib/feedx/format/parquet.rb
DELETED
|
@@ -1,102 +0,0 @@
|
|
|
1
|
-
require 'parquet'
|
|
2
|
-
require 'tmpdir'
|
|
3
|
-
|
|
4
|
-
class Feedx::Format::Parquet < Feedx::Format::Abstract
|
|
5
|
-
class Record < Arrow::Record
|
|
6
|
-
def each_pair
|
|
7
|
-
container.columns.each do |col|
|
|
8
|
-
yield col.name, col[index]
|
|
9
|
-
end
|
|
10
|
-
end
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
class Decoder < Feedx::Format::Abstract::Decoder
|
|
14
|
-
def initialize(io, **)
|
|
15
|
-
super(io)
|
|
16
|
-
|
|
17
|
-
@table = read_table
|
|
18
|
-
@cursor = 0
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
def eof?
|
|
22
|
-
@cursor >= @table.n_rows
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
def decode(target, **)
|
|
26
|
-
return if eof?
|
|
27
|
-
|
|
28
|
-
rec = Record.new(@table, @cursor)
|
|
29
|
-
@cursor += 1
|
|
30
|
-
|
|
31
|
-
target = target.allocate if target.is_a?(Class)
|
|
32
|
-
target.from_parquet(rec)
|
|
33
|
-
target
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
private
|
|
37
|
-
|
|
38
|
-
def read_table
|
|
39
|
-
tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
|
|
40
|
-
IO.copy_stream(@io, tmpname)
|
|
41
|
-
|
|
42
|
-
@table = Arrow::Table.load(tmpname, format: 'parquet')
|
|
43
|
-
ensure
|
|
44
|
-
unlink!(tmpname) if tmpname
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
def unlink!(tmpname)
|
|
48
|
-
File.unlink(tmpname)
|
|
49
|
-
rescue Errno::ENOENT
|
|
50
|
-
nil
|
|
51
|
-
end
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
class Encoder < Feedx::Format::Abstract::Encoder
|
|
55
|
-
attr_reader :schema
|
|
56
|
-
|
|
57
|
-
def initialize(io, schema:, buffer_size: 1 << 20, batch_size: 10_000)
|
|
58
|
-
super(io)
|
|
59
|
-
|
|
60
|
-
@schema = schema
|
|
61
|
-
@batch_size = batch_size.to_i
|
|
62
|
-
@buffer_size = buffer_size.to_i
|
|
63
|
-
|
|
64
|
-
@tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
|
|
65
|
-
@output = Arrow::FileOutputStream.new(@tmpname, append: false)
|
|
66
|
-
@writer = Parquet::ArrowFileWriter.new(@schema, @output)
|
|
67
|
-
@batch = []
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
def encode(msg, **opts)
|
|
71
|
-
msg = msg.to_parquet(@schema, **opts) if msg.respond_to?(:to_parquet)
|
|
72
|
-
|
|
73
|
-
res = @batch.push(msg)
|
|
74
|
-
flush_table if @batch.size >= @batch_size
|
|
75
|
-
res
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
def close
|
|
79
|
-
flush_table unless @batch.empty?
|
|
80
|
-
|
|
81
|
-
@writer.close
|
|
82
|
-
@output.close
|
|
83
|
-
IO.copy_stream(@tmpname, @io)
|
|
84
|
-
ensure
|
|
85
|
-
unlink!
|
|
86
|
-
end
|
|
87
|
-
|
|
88
|
-
private
|
|
89
|
-
|
|
90
|
-
def flush_table
|
|
91
|
-
table = Arrow::RecordBatch.new(@schema, @batch).to_table
|
|
92
|
-
@writer.write_table table, @buffer_size
|
|
93
|
-
@batch.clear
|
|
94
|
-
end
|
|
95
|
-
|
|
96
|
-
def unlink!
|
|
97
|
-
File.unlink(@tmpname)
|
|
98
|
-
rescue Errno::ENOENT
|
|
99
|
-
nil
|
|
100
|
-
end
|
|
101
|
-
end
|
|
102
|
-
end
|
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
RSpec.describe Feedx::Cache::Memory do
|
|
4
|
-
it 'read/writes' do
|
|
5
|
-
expect(subject.fetch('key')).to be_nil
|
|
6
|
-
expect(subject.fetch('key') { 'value' }).to eq('value')
|
|
7
|
-
expect(subject.fetch('key')).to eq('value')
|
|
8
|
-
expect(subject.fetch('key') { 'other' }).to eq('value')
|
|
9
|
-
expect(subject.fetch('key')).to eq('value')
|
|
10
|
-
|
|
11
|
-
subject.write('key', 'new-value')
|
|
12
|
-
expect(subject.read('key')).to eq('new-value')
|
|
13
|
-
expect(subject.fetch('key')).to eq('new-value')
|
|
14
|
-
|
|
15
|
-
subject.clear
|
|
16
|
-
expect(subject.fetch('key')).to be_nil
|
|
17
|
-
end
|
|
18
|
-
|
|
19
|
-
it 'writes strings' do
|
|
20
|
-
subject.write('key', 5)
|
|
21
|
-
expect(subject.read('key')).to eq('5')
|
|
22
|
-
end
|
|
23
|
-
end
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
RSpec.describe Feedx::Cache::Value do
|
|
4
|
-
subject do
|
|
5
|
-
described_class.new(Feedx::Cache::Memory.new, 'key')
|
|
6
|
-
end
|
|
7
|
-
|
|
8
|
-
it 'read/writes' do
|
|
9
|
-
expect(subject.fetch).to be_nil
|
|
10
|
-
expect(subject.fetch { 'value' }).to eq('value')
|
|
11
|
-
expect(subject.fetch).to eq('value')
|
|
12
|
-
expect(subject.fetch { 'other' }).to eq('value')
|
|
13
|
-
expect(subject.fetch).to eq('value')
|
|
14
|
-
|
|
15
|
-
subject.write('new-value')
|
|
16
|
-
expect(subject.read).to eq('new-value')
|
|
17
|
-
expect(subject.fetch).to eq('new-value')
|
|
18
|
-
end
|
|
19
|
-
end
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
RSpec.describe Feedx::Compression::Gzip do
|
|
4
|
-
it 'wraps readers/writers' do
|
|
5
|
-
wio = StringIO.new
|
|
6
|
-
subject.writer(wio) {|w| w.write 'xyz' * 1000 }
|
|
7
|
-
expect(wio.size).to be_within(20).of(40)
|
|
8
|
-
expect(wio.string.encoding).to eq(Encoding::BINARY)
|
|
9
|
-
|
|
10
|
-
data = ''
|
|
11
|
-
StringIO.open(wio.string) do |rio|
|
|
12
|
-
subject.reader(rio) {|z| data = z.read }
|
|
13
|
-
end
|
|
14
|
-
expect(data.size).to eq(3000)
|
|
15
|
-
expect(data.encoding).to eq(Encoding.default_external)
|
|
16
|
-
end
|
|
17
|
-
end
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
RSpec.describe Feedx::Compression::None do
|
|
4
|
-
it 'wraps readers/writers' do
|
|
5
|
-
wio = StringIO.new
|
|
6
|
-
subject.writer(wio) {|w| w.write 'xyz' * 1000 }
|
|
7
|
-
expect(wio.size).to eq(3000)
|
|
8
|
-
|
|
9
|
-
data = ''
|
|
10
|
-
StringIO.open(wio.string) do |rio|
|
|
11
|
-
subject.reader(rio) {|z| data = z.read }
|
|
12
|
-
end
|
|
13
|
-
expect(data.size).to eq(3000)
|
|
14
|
-
end
|
|
15
|
-
end
|
|
@@ -1,19 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
RSpec.describe Feedx::Compression do
|
|
4
|
-
it 'resolves' do
|
|
5
|
-
expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
|
|
6
|
-
expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
|
|
7
|
-
expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
|
|
8
|
-
expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
it 'detects' do
|
|
12
|
-
expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
|
|
13
|
-
expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
|
|
14
|
-
expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)
|
|
15
|
-
expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Gzip)
|
|
16
|
-
expect(described_class.detect('path/to/file.pb.gz')).to be_instance_of(described_class::Gzip)
|
|
17
|
-
expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::None)
|
|
18
|
-
end
|
|
19
|
-
end
|
data/spec/feedx/consumer_spec.rb
DELETED
|
@@ -1,49 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
RSpec.describe Feedx::Consumer do
|
|
4
|
-
let(:bucket) { BFS::Bucket::InMem.new }
|
|
5
|
-
let(:klass) { Feedx::TestCase::Model }
|
|
6
|
-
let(:cache) { Feedx::Cache::Memory.new.value('my-consumer') }
|
|
7
|
-
|
|
8
|
-
before { allow(BFS).to receive(:resolve).and_return(bucket) }
|
|
9
|
-
|
|
10
|
-
it 'rejects invalid inputs' do
|
|
11
|
-
expect do
|
|
12
|
-
described_class.each('mock:///dir/file.txt', klass)
|
|
13
|
-
end.to raise_error(/unable to detect format/)
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
it 'consumes feeds' do
|
|
17
|
-
url = mock_produce!
|
|
18
|
-
csm = described_class.new(url, klass)
|
|
19
|
-
expect(csm).to be_a(Enumerable)
|
|
20
|
-
|
|
21
|
-
cnt = csm.count do |rec|
|
|
22
|
-
expect(rec).to be_instance_of(klass)
|
|
23
|
-
true
|
|
24
|
-
end
|
|
25
|
-
expect(cnt).to eq(300)
|
|
26
|
-
end
|
|
27
|
-
|
|
28
|
-
it 'performs conditionally' do
|
|
29
|
-
url = mock_produce! last_modified: Time.at(1515151515)
|
|
30
|
-
expect(described_class.new(url, klass, cache: cache).count).to eq(300)
|
|
31
|
-
expect(described_class.new(url, klass, cache: cache).count).to eq(0)
|
|
32
|
-
|
|
33
|
-
url = mock_produce!
|
|
34
|
-
expect(described_class.new(url, klass, cache: cache).count).to eq(300)
|
|
35
|
-
expect(described_class.new(url, klass, cache: cache).count).to eq(300)
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
private
|
|
39
|
-
|
|
40
|
-
def mock_produce!(enum: mock_enum, **opts)
|
|
41
|
-
url = 'mock:///dir/file.json'
|
|
42
|
-
Feedx::Producer.perform url, enum: enum, **opts
|
|
43
|
-
url
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
def mock_enum
|
|
47
|
-
%w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
|
|
48
|
-
end
|
|
49
|
-
end
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
RSpec.describe Feedx::Format::Abstract do
|
|
4
|
-
subject { Feedx::Format::JSON.new }
|
|
5
|
-
|
|
6
|
-
let(:wio) { StringIO.new }
|
|
7
|
-
let(:rio) { StringIO.open(wio.string) }
|
|
8
|
-
|
|
9
|
-
it 'decodes each' do
|
|
10
|
-
subject.encoder wio do |enc|
|
|
11
|
-
enc.encode(Feedx::TestCase::Model.new('X'))
|
|
12
|
-
enc.encode(Feedx::TestCase::Model.new('Y'))
|
|
13
|
-
enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
subject.decoder rio do |dec|
|
|
17
|
-
acc = dec.decode_each(Feedx::TestCase::Model).to_a
|
|
18
|
-
expect(acc.map(&:title)).to eq(%w[X Y Z])
|
|
19
|
-
end
|
|
20
|
-
end
|
|
21
|
-
end
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
RSpec.describe Feedx::Format::JSON do
|
|
4
|
-
let(:wio) { StringIO.new }
|
|
5
|
-
let(:rio) { StringIO.open(wio.string) }
|
|
6
|
-
|
|
7
|
-
it 'encode/decodes' do
|
|
8
|
-
subject.encoder wio do |enc|
|
|
9
|
-
enc.encode(Feedx::TestCase::Model.new('X'))
|
|
10
|
-
enc.encode(Feedx::TestCase::Model.new('Y'))
|
|
11
|
-
enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
|
|
12
|
-
end
|
|
13
|
-
expect(wio.string).to eq(<<~JSON)
|
|
14
|
-
{"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
|
|
15
|
-
{"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
|
|
16
|
-
{"title":"Z"}
|
|
17
|
-
JSON
|
|
18
|
-
|
|
19
|
-
subject.decoder rio do |dec|
|
|
20
|
-
expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
|
|
21
|
-
expect(dec.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
|
|
22
|
-
expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
|
|
23
|
-
expect(dec.decode(Feedx::TestCase::Model)).to be_nil
|
|
24
|
-
expect(dec).to be_eof
|
|
25
|
-
end
|
|
26
|
-
end
|
|
27
|
-
end
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
require 'spec_helper'
|
|
2
|
-
|
|
3
|
-
RSpec.describe Feedx::Format::Parquet do
|
|
4
|
-
let(:wio) { StringIO.new }
|
|
5
|
-
let(:rio) { StringIO.open(wio.string) }
|
|
6
|
-
|
|
7
|
-
let(:schema) do
|
|
8
|
-
Arrow::Schema.new([
|
|
9
|
-
Arrow::Field.new('title', :string),
|
|
10
|
-
Arrow::Field.new('updated_at', type: :timestamp, unit: :second),
|
|
11
|
-
])
|
|
12
|
-
end
|
|
13
|
-
|
|
14
|
-
it 'encode/decodes' do
|
|
15
|
-
subject.encoder wio, schema: schema, batch_size: 2 do |enc|
|
|
16
|
-
enc.encode(Feedx::TestCase::Model.new('X'))
|
|
17
|
-
enc.encode(Feedx::TestCase::Model.new('Y'))
|
|
18
|
-
enc.encode(Feedx::TestCase::Model.new('Z'))
|
|
19
|
-
end
|
|
20
|
-
expect(wio.string.bytesize).to be_within(100).of(1100)
|
|
21
|
-
|
|
22
|
-
subject.decoder rio do |dec|
|
|
23
|
-
expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
|
|
24
|
-
expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
|
|
25
|
-
expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
|
|
26
|
-
expect(dec.decode(Feedx::TestCase::Model)).to be_nil
|
|
27
|
-
expect(dec).to be_eof
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
end
|