feedx 0.12.7 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/test.yml +2 -37
  3. data/.golangci.yml +13 -4
  4. data/.rubocop.yml +8 -14
  5. data/.tool-versions +1 -0
  6. data/Gemfile +8 -0
  7. data/Gemfile.lock +54 -68
  8. data/Makefile +3 -3
  9. data/README.md +3 -1
  10. data/compression.go +29 -0
  11. data/compression_test.go +73 -61
  12. data/consumer.go +96 -152
  13. data/consumer_test.go +124 -59
  14. data/example_test.go +140 -0
  15. data/feedx.gemspec +2 -10
  16. data/feedx.go +16 -31
  17. data/feedx_ext_test.go +13 -3
  18. data/feedx_test.go +24 -26
  19. data/format.go +29 -19
  20. data/format_test.go +84 -56
  21. data/go.mod +11 -7
  22. data/go.sum +16 -138
  23. data/incremental.go +122 -0
  24. data/incremental_test.go +62 -0
  25. data/lib/feedx/cache/abstract.rb +3 -3
  26. data/lib/feedx/cache/value.rb +6 -6
  27. data/lib/feedx/compression/abstract.rb +2 -2
  28. data/lib/feedx/compression/gzip.rb +4 -4
  29. data/lib/feedx/consumer.rb +8 -8
  30. data/lib/feedx/format/abstract.rb +6 -6
  31. data/lib/feedx/format/json.rb +2 -2
  32. data/lib/feedx/format/protobuf.rb +6 -6
  33. data/lib/feedx/format.rb +1 -3
  34. data/lib/feedx/producer.rb +11 -11
  35. data/lib/feedx/stream.rb +2 -2
  36. data/lib/feedx.rb +2 -3
  37. data/manifest.go +65 -0
  38. data/producer.go +34 -137
  39. data/producer_test.go +46 -60
  40. data/reader.go +142 -41
  41. data/reader_test.go +86 -35
  42. data/scheduler.go +176 -0
  43. data/scheduler_test.go +128 -0
  44. data/writer.go +13 -13
  45. data/writer_test.go +61 -44
  46. metadata +12 -137
  47. data/.github/workflows/lint.yml +0 -18
  48. data/ext/parquet/decoder.go +0 -59
  49. data/ext/parquet/decoder_test.go +0 -88
  50. data/ext/parquet/encoder.go +0 -27
  51. data/ext/parquet/encoder_test.go +0 -70
  52. data/ext/parquet/go.mod +0 -12
  53. data/ext/parquet/go.sum +0 -193
  54. data/ext/parquet/parquet.go +0 -78
  55. data/ext/parquet/parquet_test.go +0 -28
  56. data/ext/parquet/testdata/alltypes_plain.parquet +0 -0
  57. data/lib/feedx/format/parquet.rb +0 -102
  58. data/spec/feedx/cache/memory_spec.rb +0 -23
  59. data/spec/feedx/cache/value_spec.rb +0 -19
  60. data/spec/feedx/compression/gzip_spec.rb +0 -17
  61. data/spec/feedx/compression/none_spec.rb +0 -15
  62. data/spec/feedx/compression_spec.rb +0 -19
  63. data/spec/feedx/consumer_spec.rb +0 -49
  64. data/spec/feedx/format/abstract_spec.rb +0 -21
  65. data/spec/feedx/format/json_spec.rb +0 -27
  66. data/spec/feedx/format/parquet_spec.rb +0 -30
  67. data/spec/feedx/format/protobuf_spec.rb +0 -23
  68. data/spec/feedx/format_spec.rb +0 -21
  69. data/spec/feedx/producer_spec.rb +0 -74
  70. data/spec/feedx/stream_spec.rb +0 -109
  71. data/spec/spec_helper.rb +0 -57
data/ext/parquet/go.sum DELETED
@@ -1,193 +0,0 @@
1
- cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
2
- github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
3
- github.com/apache/thrift v0.15.0 h1:aGvdaR0v1t9XLgjtBYwxcBvBOTMqClzwE26CHOgjW1Y=
4
- github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU=
5
- github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
6
- github.com/bmatcuk/doublestar/v3 v3.0.0 h1:TQtVPlDnAYwcrVNB2JiGuMc++H5qzWZd9PhkNo5WyHI=
7
- github.com/bmatcuk/doublestar/v3 v3.0.0/go.mod h1:6PcTVMw80pCY1RVuoqu3V++99uQB3vsSYKPTd8AWA0k=
8
- github.com/bsm/bfs v0.11.5 h1:NMWD0hOp/74boNd51Zas4DMh81mvp1As1SjEZrFWHi0=
9
- github.com/bsm/bfs v0.11.5/go.mod h1:D2N+ly23VsODKiRvkf4HUgKSgOaNjKTOJE53rhL6Vfk=
10
- github.com/bsm/ginkgo v1.16.4/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
11
- github.com/bsm/ginkgo v1.16.5 h1:uTeeWv0Yx1PnDeCk76PFyGrOMVw3D+r9bTNKNcIjDdQ=
12
- github.com/bsm/ginkgo v1.16.5/go.mod h1:RabIZLzOCPghgHJKUqHZpqrQETA5AnF4aCSIYy5C1bk=
13
- github.com/bsm/gomega v1.16.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
14
- github.com/bsm/gomega v1.17.0 h1:Sd6EsHO5d0DU6d41dtx9cK3T7Vjsr89o6zyIVWgi0CI=
15
- github.com/bsm/gomega v1.17.0/go.mod h1:JifAceMQ4crZIWYUKrlGcmbN3bqHogVTADMD2ATsbwk=
16
- github.com/bsm/pbio v0.3.0 h1:8XzkUPoJZblPkLp2jHVMnIhGs5WaNHTCmO7gRC45izk=
17
- github.com/bsm/pbio v0.3.0/go.mod h1:3O4XQFoarlYalkGd+zMFfEUyalP8NBKkQ0Ta4IwhN4w=
18
- github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
19
- github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
20
- github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
21
- github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
22
- github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
23
- github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
24
- github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
25
- github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
26
- github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
27
- github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
28
- github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
29
- github.com/fraugster/parquet-go v0.6.1 h1:Kks9Ibly3ZZQPRPGkS1lVmrwndBp8PxamBnDFG5jvEM=
30
- github.com/fraugster/parquet-go v0.6.1/go.mod h1:1HGhXzpHv7CULzknVNWIY0Ihn2O3qNbD1p+aQvHWhqo=
31
- github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
32
- github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
33
- github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
34
- github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
35
- github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
36
- github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
37
- github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
38
- github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
39
- github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
40
- github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
41
- github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
42
- github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
43
- github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
44
- github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
45
- github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
46
- github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
47
- github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
48
- github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
49
- github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
50
- github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
51
- github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
52
- github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
53
- github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
54
- github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
55
- github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
56
- github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
57
- github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
58
- github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
59
- github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
60
- github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
61
- github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
62
- github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
63
- github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
64
- github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
65
- github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
66
- github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
67
- github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
68
- github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
69
- github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
70
- github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
71
- github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
72
- github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
73
- github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
74
- github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
75
- github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
76
- github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
77
- github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4=
78
- github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
79
- github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
80
- github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
81
- github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs=
82
- github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
83
- github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
84
- github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
85
- github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
86
- github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
87
- github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
88
- github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
89
- github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
90
- github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
91
- github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
92
- github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
93
- github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
94
- github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
95
- github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
96
- github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
97
- github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
98
- github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
99
- github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
100
- github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
101
- github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
102
- github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
103
- github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
104
- github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
105
- golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
106
- golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
107
- golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
108
- golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
109
- golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
110
- golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
111
- golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
112
- golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
113
- golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
114
- golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
115
- golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
116
- golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
117
- golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
118
- golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
119
- golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
120
- golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
121
- golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
122
- golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
123
- golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
124
- golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI=
125
- golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
126
- golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
127
- golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
128
- golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
129
- golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
130
- golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
131
- golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
132
- golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
133
- golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
134
- golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
135
- golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
136
- golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
137
- golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
138
- golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
139
- golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
140
- golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
141
- golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
142
- golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
143
- golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
144
- golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
145
- golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
146
- golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
147
- golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
148
- golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
149
- golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
150
- golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
151
- golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
152
- golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
153
- golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
154
- golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
155
- golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
156
- golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
157
- golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
158
- golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
159
- golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
160
- golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
161
- google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
162
- google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
163
- google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
164
- google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
165
- google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
166
- google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
167
- google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
168
- google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
169
- google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
170
- google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
171
- google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
172
- google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
173
- google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
174
- google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
175
- google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
176
- google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
177
- google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
178
- google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
179
- google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
180
- google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
181
- google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
182
- gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
183
- gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
184
- gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
185
- gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
186
- gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
187
- gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
188
- gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
189
- gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
190
- gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
191
- gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
192
- honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
193
- honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
@@ -1,78 +0,0 @@
1
- package parquet
2
-
3
- import (
4
- "io"
5
- "io/ioutil"
6
- "os"
7
-
8
- "github.com/bsm/feedx"
9
- goparquet "github.com/fraugster/parquet-go"
10
- )
11
-
12
- // --------------------------------------------------------------------
13
-
14
- // Format is a parquet format.
15
- type Format struct {
16
- TempDir string
17
- }
18
-
19
- // NewDecoder implements Format.
20
- func (f *Format) NewDecoder(r io.Reader) (feedx.FormatDecoder, error) {
21
- if rs, ok := r.(io.ReadSeeker); ok {
22
- return newDecoder(rs)
23
- }
24
-
25
- tmp, err := copyToTempFile(f.TempDir, r)
26
- if err != nil {
27
- return nil, err
28
- }
29
-
30
- dec, err := newDecoder(tmp)
31
- if err != nil {
32
- _ = tmp.Close()
33
- return nil, err
34
- }
35
- dec.tmp = tmp
36
- return dec, nil
37
- }
38
-
39
- // NewEncoder implements Format.
40
- func (f *Format) NewEncoder(w io.Writer, opts ...goparquet.FileWriterOption) (feedx.FormatEncoder, error) {
41
- return newEncoder(w, opts)
42
- }
43
-
44
- // --------------------------------------------------------------------
45
-
46
- type tempFile struct{ *os.File }
47
-
48
- func copyToTempFile(dir string, r io.Reader) (*tempFile, error) {
49
- w, err := ioutil.TempFile(dir, "feedx-ext-parquet")
50
- if err != nil {
51
- return nil, err
52
- }
53
- if _, err := io.Copy(w, r); err != nil {
54
- _ = w.Close()
55
- _ = os.Remove(w.Name())
56
- return nil, err
57
- }
58
- if err := w.Close(); err != nil {
59
- _ = os.Remove(w.Name())
60
- return nil, err
61
- }
62
-
63
- f, err := os.Open(w.Name())
64
- if err != nil {
65
- _ = os.Remove(w.Name())
66
- return nil, err
67
- }
68
-
69
- return &tempFile{File: f}, nil
70
- }
71
-
72
- func (f *tempFile) Close() error {
73
- err := f.File.Close()
74
- if e := os.Remove(f.Name()); e != nil {
75
- err = e
76
- }
77
- return err
78
- }
@@ -1,28 +0,0 @@
1
- package parquet_test
2
-
3
- import (
4
- "testing"
5
- "time"
6
-
7
- . "github.com/bsm/ginkgo"
8
- . "github.com/bsm/gomega"
9
- )
10
-
11
- type mockStruct struct {
12
- ID int `parquet:"id"`
13
- Bool bool `parquet:"bool_col"`
14
- TinyInt int8 `parquet:"tinyint_col"`
15
- SmallUint uint16 `parquet:"smallint_col"`
16
- StdInt int `parquet:"int_col"`
17
- BigInt int64 `parquet:"bigint_col"`
18
- Float *float32 `parquet:"float_col"`
19
- Double float64 `parquet:"double_col"`
20
- DateString string `parquet:"date_string_col"`
21
- ByteString []byte `parquet:"string_col"`
22
- Timestamp time.Time `parquet:"timestamp_col"`
23
- }
24
-
25
- func TestSuite(t *testing.T) {
26
- RegisterFailHandler(Fail)
27
- RunSpecs(t, "feedx/ext/parquet")
28
- }
@@ -1,102 +0,0 @@
1
- require 'parquet'
2
- require 'tmpdir'
3
-
4
- class Feedx::Format::Parquet < Feedx::Format::Abstract
5
- class Record < Arrow::Record
6
- def each_pair
7
- container.columns.each do |col|
8
- yield col.name, col[index]
9
- end
10
- end
11
- end
12
-
13
- class Decoder < Feedx::Format::Abstract::Decoder
14
- def initialize(io, **)
15
- super(io)
16
-
17
- @table = read_table
18
- @cursor = 0
19
- end
20
-
21
- def eof?
22
- @cursor >= @table.n_rows
23
- end
24
-
25
- def decode(target, **)
26
- return if eof?
27
-
28
- rec = Record.new(@table, @cursor)
29
- @cursor += 1
30
-
31
- target = target.allocate if target.is_a?(Class)
32
- target.from_parquet(rec)
33
- target
34
- end
35
-
36
- private
37
-
38
- def read_table
39
- tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
40
- IO.copy_stream(@io, tmpname)
41
-
42
- @table = Arrow::Table.load(tmpname, format: 'parquet')
43
- ensure
44
- unlink!(tmpname) if tmpname
45
- end
46
-
47
- def unlink!(tmpname)
48
- File.unlink(tmpname)
49
- rescue Errno::ENOENT
50
- nil
51
- end
52
- end
53
-
54
- class Encoder < Feedx::Format::Abstract::Encoder
55
- attr_reader :schema
56
-
57
- def initialize(io, schema:, buffer_size: 1 << 20, batch_size: 10_000)
58
- super(io)
59
-
60
- @schema = schema
61
- @batch_size = batch_size.to_i
62
- @buffer_size = buffer_size.to_i
63
-
64
- @tmpname = ::Dir::Tmpname.create('feedx-parquet') {|path, *| path }
65
- @output = Arrow::FileOutputStream.new(@tmpname, append: false)
66
- @writer = Parquet::ArrowFileWriter.new(@schema, @output)
67
- @batch = []
68
- end
69
-
70
- def encode(msg, **opts)
71
- msg = msg.to_parquet(@schema, **opts) if msg.respond_to?(:to_parquet)
72
-
73
- res = @batch.push(msg)
74
- flush_table if @batch.size >= @batch_size
75
- res
76
- end
77
-
78
- def close
79
- flush_table unless @batch.empty?
80
-
81
- @writer.close
82
- @output.close
83
- IO.copy_stream(@tmpname, @io)
84
- ensure
85
- unlink!
86
- end
87
-
88
- private
89
-
90
- def flush_table
91
- table = Arrow::RecordBatch.new(@schema, @batch).to_table
92
- @writer.write_table table, @buffer_size
93
- @batch.clear
94
- end
95
-
96
- def unlink!
97
- File.unlink(@tmpname)
98
- rescue Errno::ENOENT
99
- nil
100
- end
101
- end
102
- end
@@ -1,23 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe Feedx::Cache::Memory do
4
- it 'read/writes' do
5
- expect(subject.fetch('key')).to be_nil
6
- expect(subject.fetch('key') { 'value' }).to eq('value')
7
- expect(subject.fetch('key')).to eq('value')
8
- expect(subject.fetch('key') { 'other' }).to eq('value')
9
- expect(subject.fetch('key')).to eq('value')
10
-
11
- subject.write('key', 'new-value')
12
- expect(subject.read('key')).to eq('new-value')
13
- expect(subject.fetch('key')).to eq('new-value')
14
-
15
- subject.clear
16
- expect(subject.fetch('key')).to be_nil
17
- end
18
-
19
- it 'writes strings' do
20
- subject.write('key', 5)
21
- expect(subject.read('key')).to eq('5')
22
- end
23
- end
@@ -1,19 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe Feedx::Cache::Value do
4
- subject do
5
- described_class.new(Feedx::Cache::Memory.new, 'key')
6
- end
7
-
8
- it 'read/writes' do
9
- expect(subject.fetch).to be_nil
10
- expect(subject.fetch { 'value' }).to eq('value')
11
- expect(subject.fetch).to eq('value')
12
- expect(subject.fetch { 'other' }).to eq('value')
13
- expect(subject.fetch).to eq('value')
14
-
15
- subject.write('new-value')
16
- expect(subject.read).to eq('new-value')
17
- expect(subject.fetch).to eq('new-value')
18
- end
19
- end
@@ -1,17 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe Feedx::Compression::Gzip do
4
- it 'wraps readers/writers' do
5
- wio = StringIO.new
6
- subject.writer(wio) {|w| w.write 'xyz' * 1000 }
7
- expect(wio.size).to be_within(20).of(40)
8
- expect(wio.string.encoding).to eq(Encoding::BINARY)
9
-
10
- data = ''
11
- StringIO.open(wio.string) do |rio|
12
- subject.reader(rio) {|z| data = z.read }
13
- end
14
- expect(data.size).to eq(3000)
15
- expect(data.encoding).to eq(Encoding.default_external)
16
- end
17
- end
@@ -1,15 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe Feedx::Compression::None do
4
- it 'wraps readers/writers' do
5
- wio = StringIO.new
6
- subject.writer(wio) {|w| w.write 'xyz' * 1000 }
7
- expect(wio.size).to eq(3000)
8
-
9
- data = ''
10
- StringIO.open(wio.string) do |rio|
11
- subject.reader(rio) {|z| data = z.read }
12
- end
13
- expect(data.size).to eq(3000)
14
- end
15
- end
@@ -1,19 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe Feedx::Compression do
4
- it 'resolves' do
5
- expect(described_class.resolve(:gzip)).to be_instance_of(described_class::Gzip)
6
- expect(described_class.resolve(:gz)).to be_instance_of(described_class::Gzip)
7
- expect(described_class.resolve(nil)).to be_instance_of(described_class::None)
8
- expect { described_class.resolve(:txt) }.to raise_error(/invalid compression txt/)
9
- end
10
-
11
- it 'detects' do
12
- expect(described_class.detect('path/to/file.jsonz')).to be_instance_of(described_class::Gzip)
13
- expect(described_class.detect('path/to/file.json.gz')).to be_instance_of(described_class::Gzip)
14
- expect(described_class.detect('path/to/file.json')).to be_instance_of(described_class::None)
15
- expect(described_class.detect('path/to/file.pbz')).to be_instance_of(described_class::Gzip)
16
- expect(described_class.detect('path/to/file.pb.gz')).to be_instance_of(described_class::Gzip)
17
- expect(described_class.detect('path/to/file.pb')).to be_instance_of(described_class::None)
18
- end
19
- end
@@ -1,49 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe Feedx::Consumer do
4
- let(:bucket) { BFS::Bucket::InMem.new }
5
- let(:klass) { Feedx::TestCase::Model }
6
- let(:cache) { Feedx::Cache::Memory.new.value('my-consumer') }
7
-
8
- before { allow(BFS).to receive(:resolve).and_return(bucket) }
9
-
10
- it 'rejects invalid inputs' do
11
- expect do
12
- described_class.each('mock:///dir/file.txt', klass)
13
- end.to raise_error(/unable to detect format/)
14
- end
15
-
16
- it 'consumes feeds' do
17
- url = mock_produce!
18
- csm = described_class.new(url, klass)
19
- expect(csm).to be_a(Enumerable)
20
-
21
- cnt = csm.count do |rec|
22
- expect(rec).to be_instance_of(klass)
23
- true
24
- end
25
- expect(cnt).to eq(300)
26
- end
27
-
28
- it 'performs conditionally' do
29
- url = mock_produce! last_modified: Time.at(1515151515)
30
- expect(described_class.new(url, klass, cache: cache).count).to eq(300)
31
- expect(described_class.new(url, klass, cache: cache).count).to eq(0)
32
-
33
- url = mock_produce!
34
- expect(described_class.new(url, klass, cache: cache).count).to eq(300)
35
- expect(described_class.new(url, klass, cache: cache).count).to eq(300)
36
- end
37
-
38
- private
39
-
40
- def mock_produce!(enum: mock_enum, **opts)
41
- url = 'mock:///dir/file.json'
42
- Feedx::Producer.perform url, enum: enum, **opts
43
- url
44
- end
45
-
46
- def mock_enum
47
- %w[x y z].map {|t| Feedx::TestCase::Model.new(t) } * 100
48
- end
49
- end
@@ -1,21 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe Feedx::Format::Abstract do
4
- subject { Feedx::Format::JSON.new }
5
-
6
- let(:wio) { StringIO.new }
7
- let(:rio) { StringIO.open(wio.string) }
8
-
9
- it 'decodes each' do
10
- subject.encoder wio do |enc|
11
- enc.encode(Feedx::TestCase::Model.new('X'))
12
- enc.encode(Feedx::TestCase::Model.new('Y'))
13
- enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
14
- end
15
-
16
- subject.decoder rio do |dec|
17
- acc = dec.decode_each(Feedx::TestCase::Model).to_a
18
- expect(acc.map(&:title)).to eq(%w[X Y Z])
19
- end
20
- end
21
- end
@@ -1,27 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe Feedx::Format::JSON do
4
- let(:wio) { StringIO.new }
5
- let(:rio) { StringIO.open(wio.string) }
6
-
7
- it 'encode/decodes' do
8
- subject.encoder wio do |enc|
9
- enc.encode(Feedx::TestCase::Model.new('X'))
10
- enc.encode(Feedx::TestCase::Model.new('Y'))
11
- enc.encode(Feedx::TestCase::Message.new(title: 'Z'))
12
- end
13
- expect(wio.string).to eq(<<~JSON)
14
- {"title":"X","updated_at":"2018-01-05 11:25:15 UTC"}
15
- {"title":"Y","updated_at":"2018-01-05 11:25:15 UTC"}
16
- {"title":"Z"}
17
- JSON
18
-
19
- subject.decoder rio do |dec|
20
- expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
21
- expect(dec.decode(Feedx::TestCase::Model.new('O'))).to eq(Feedx::TestCase::Model.new('Y'))
22
- expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
23
- expect(dec.decode(Feedx::TestCase::Model)).to be_nil
24
- expect(dec).to be_eof
25
- end
26
- end
27
- end
@@ -1,30 +0,0 @@
1
- require 'spec_helper'
2
-
3
- RSpec.describe Feedx::Format::Parquet do
4
- let(:wio) { StringIO.new }
5
- let(:rio) { StringIO.open(wio.string) }
6
-
7
- let(:schema) do
8
- Arrow::Schema.new([
9
- Arrow::Field.new('title', :string),
10
- Arrow::Field.new('updated_at', type: :timestamp, unit: :second),
11
- ])
12
- end
13
-
14
- it 'encode/decodes' do
15
- subject.encoder wio, schema: schema, batch_size: 2 do |enc|
16
- enc.encode(Feedx::TestCase::Model.new('X'))
17
- enc.encode(Feedx::TestCase::Model.new('Y'))
18
- enc.encode(Feedx::TestCase::Model.new('Z'))
19
- end
20
- expect(wio.string.bytesize).to be_within(100).of(1100)
21
-
22
- subject.decoder rio do |dec|
23
- expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('X'))
24
- expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Y'))
25
- expect(dec.decode(Feedx::TestCase::Model)).to eq(Feedx::TestCase::Model.new('Z'))
26
- expect(dec.decode(Feedx::TestCase::Model)).to be_nil
27
- expect(dec).to be_eof
28
- end
29
- end
30
- end