stream_lines 0.3.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5f86f4c30e240b9076fa0ede333d89a0974e48b57ca3c75c145031314b524973
-  data.tar.gz: 7ee39a6d12b3e3e54eac67f6ea1d0108ac8ecc57ad78676187fe4c3dfcfed293
+  metadata.gz: d12669d8a8c0f786d731b6fb98374e4105b483379e6ad9b0cb346a62289c5481
+  data.tar.gz: 67607b92517c8b6151e576258314ed4896344893b1a5160602e018aa0aae7d28
 SHA512:
-  metadata.gz: 2edb6bdd8ebccbf8d99c9769303c132ae2016c628dec9fc7b499ea3cf95ef579f9781ac7c205cbf526a1a10062af5d14c56a4927058a9580b0b6c0636b186978
-  data.tar.gz: b56a5173a42cc761337d4d1f30116bc6e5d08f1e07ff32eb8c4edeef58a537d42949a21d696be21f3913f51413242e03a0bbe5f7e4176f8b8f63b3204722e1ee
+  metadata.gz: 178c5fc570b8dba9330efcd6493b2437fdbd6ed277e3ec653d7c6e436fcb959fea89110781059e5a6acf3e821fa5cb098afcee35664dbac028469baf3542f5cf
+  data.tar.gz: a9e08b3b01ad37fd2daf0e1e10ed729fb8ceabe0e151c2e147c0d131216bf2f1cca2562240890db816a869fc2c7f8c52618796769b20cb8c92e920b16cf11ff0

data/.github/dependabot.yml

@@ -0,0 +1,12 @@
+version: 2
+updates:
+- package-ecosystem: bundler
+  directory: "/"
+  schedule:
+    interval: weekly
+    time: "10:00"
+  open-pull-requests-limit: 10
+  ignore:
+  - dependency-name: rubocop
+    versions:
+    - 1.12.0

data/.github/workflows/ruby-tests.yml

@@ -0,0 +1,32 @@
+name: Tests (Ruby)
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby_version:
+          - 2.6
+          - 2.7
+          - 3.0
+          - 3.1
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby_version }}
+      - name: Ruby version
+        run: ruby -v
+      - name: Install dependencies
+        run: bundle install
+      - name: Run tests
+        run: bundle exec rake

data/.github/workflows/style-check.yml

@@ -0,0 +1,30 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Style Check
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 3.0
+    - name: Install dependencies
+      run: bundle install
+    - name: Run Rubocop
+      run: bundle exec rubocop -D

data/.rubocop.yml

@@ -1,4 +1,6 @@
 AllCops:
+  NewCops: 'enable'
+  TargetRubyVersion: 2.5
   Exclude:
     - '*.gemspec'
     - 'vendor/bundle/**/*'

data/.ruby-version

@@ -1 +1 @@
-ruby-2.7.0
+3.1.2

data/Gemfile.lock

@@ -1,78 +1,85 @@
 PATH
   remote: .
   specs:
-    stream_lines (0.3.0)
+    stream_lines (0.4.1)
       httparty (~> 0.14)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    addressable (2.7.0)
+    addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.0)
-    awesome_print (1.8.0)
-    backports (3.15.0)
-    bundler-audit (0.6.1)
+    ast (2.4.2)
+    awesome_print (1.9.2)
+    backports (3.17.2)
+    bundler-audit (0.9.0.1)
       bundler (>= 1.2.0, < 3)
-      thor (~> 0.18)
+      thor (~> 1.0)
     bundler-gem_version_tasks (0.2.1)
-    byebug (11.1.1)
-    crack (0.4.3)
-      safe_yaml (~> 1.0.0)
-    diff-lcs (1.3)
-    docile (1.3.2)
-    ffi (1.11.3)
-    get_process_mem (0.2.5)
+    byebug (11.1.3)
+    charlock_holmes (0.7.7)
+    crack (0.4.5)
+      rexml
+    diff-lcs (1.5.0)
+    docile (1.3.4)
+    ffi (1.13.1)
+    get_process_mem (0.2.7)
       ffi (~> 1.0)
-    hashdiff (1.0.0)
-    httparty (0.17.3)
+    hashdiff (1.0.1)
+    httparty (0.20.0)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
-    jaro_winkler (1.5.4)
-    memory_profiler (0.9.14)
+    memory_profiler (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2021.0901)
     multi_json (1.14.1)
     multi_xml (0.6.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
-    parallel (1.19.1)
-    parser (2.7.0.2)
-      ast (~> 2.4.0)
-    public_suffix (4.0.3)
-    rack (2.0.8)
+    parallel (1.22.1)
+    parser (3.1.1.0)
+      ast (~> 2.4.1)
+    public_suffix (4.0.6)
+    rack (2.2.3)
     rack-protection (2.0.8.1)
       rack
-    rainbow (3.0.0)
-    rake (13.0.1)
-    rspec (3.9.0)
-      rspec-core (~> 3.9.0)
-      rspec-expectations (~> 3.9.0)
-      rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.0)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.3.0)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-mocks (3.9.1)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
-    rubocop (0.79.0)
-      jaro_winkler (~> 1.5.1)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.27.0)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.16.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
-    ruby-progressbar (1.10.1)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.17.0)
+      parser (>= 3.1.1.0)
+    ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.2)
-    safe_yaml (1.0.5)
-    simplecov (0.18.1)
+    simplecov (0.21.2)
       docile (~> 1.1)
-      simplecov-html (~> 0.11.0)
-    simplecov-html (0.11.0)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-html (0.12.3)
+    simplecov_json_formatter (0.1.2)
     sinatra (2.0.8.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
@@ -85,13 +92,14 @@ GEM
       rack-protection (= 2.0.8.1)
       sinatra (= 2.0.8.1)
       tilt (~> 2.0)
-    thor (0.20.3)
+    thor (1.1.0)
     tilt (2.0.10)
-    unicode-display_width (1.6.0)
-    webmock (3.8.1)
-      addressable (>= 2.3.6)
+    unicode-display_width (2.1.0)
+    webmock (3.14.0)
+      addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
+    webrick (1.7.0)
 
 PLATFORMS
   ruby
@@ -102,16 +110,18 @@ DEPENDENCIES
   bundler-audit
   bundler-gem_version_tasks
   byebug
+  charlock_holmes
   get_process_mem
   memory_profiler
   rake (~> 13.0)
   rspec (~> 3.0)
-  rubocop (~> 0.79.0)
+  rubocop (~> 1.27.0)
   simplecov (~> 0.17)
   sinatra (~> 2.0)
   sinatra-contrib (~> 2.0)
   stream_lines!
   webmock (~> 3.0)
+  webrick (~> 1.7)
 
 BUNDLED WITH
-   2.0.2
+   2.3.11

data/README.md

@@ -1,6 +1,7 @@
 # StreamLines
 
-[![Build Status](https://travis-ci.com/jdlubrano/stream_lines.svg?branch=master)](https://travis-ci.com/jdlubrano/stream_lines)
+[![Gem Version](https://badge.fury.io/rb/stream_lines.svg)](https://badge.fury.io/rb/stream_lines)
+[![Tests (Ruby)](https://github.com/jdlubrano/stream_lines/actions/workflows/ruby-tests.yml/badge.svg)](https://github.com/jdlubrano/stream_lines/actions/workflows/ruby-tests.yml)
 
 An API for streaming files from remote locations one line at a time.
 
@@ -54,6 +55,37 @@ stream.each_slice(100) do |lines|
 end
 ```
 
+##### Caveats
+
+This library strives to provide streamed data via an `Enumerable` interface.
+In order to be memory-efficient, however, each time the stream is iterated over,
+a new GET request is made to fetch the data from its remote URL.  For example,
+
+```ruby
+url = 'https://my.remote.file/file.txt'
+stream = StreamLines::Reading::Stream.new(url)
+do_something_with_first_row(stream.first) # GET request made
+
+stream.each do |line| # same GET request made
+  # Do something with the line of data (the line will be a String)
+end
+```
+
+makes two GET requests.  The call to `first` makes a GET request to fetch
+the first row of data.  The subsequent call to `each` makes the same GET
+request.  To avoid unnecessary requests, I recommend a slightly different
+approach, which may not be intuitive but does make only one network request:
+
+```
+url = 'https://my.remote.file/file.txt'
+stream = StreamLines::Reading::Stream.new(url)
+
+stream.each_with_index do |line, i|
+  do_something_with_first_row(line) if i.zero?
+  # Do something with the line of data (the line will be a String)
+end
+```
+
 ##### CSVs
 
 This gem provides first-class support for streaming CSVs from a remote URL.
@@ -118,10 +150,10 @@ To install this gem onto your local machine, run `bundle exec rake install`.
 
 ## Releasing
 
-After merging in the new functionality to the master branch:
+After merging in the new functionality to the main branch:
 
 ```
-git checkout master
+git checkout main
 git pull --prune
 bundle exec rake version:bump:<major, minor, or patch>
 bundle exec rubocop -a
@@ -134,7 +166,7 @@ bundle exec rake release
 Bug reports and pull requests are welcome on GitHub at
 https://github.com/jdlubrano/stream_lines. This project is intended to be a
 safe, welcoming space for collaboration, and contributors are expected to
-adhere to the [code of conduct](https://github.com/jdlubrano/stream_lines/blob/master/CODE_OF_CONDUCT.md).
+adhere to the [code of conduct](https://github.com/jdlubrano/stream_lines/blob/main/CODE_OF_CONDUCT.md).
 
 ## License
 

data/bin/console

@@ -3,7 +3,9 @@
 # frozen_string_literal: true
 
 require 'bundler/setup'
+require 'byebug'
 require 'stream_lines'
+require 'charlock_holmes/string'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.

data/lib/stream_lines/reading/csv.rb

@@ -23,7 +23,10 @@ module StreamLines
       def initialize(url, **csv_options)
         @url = url
         @csv_options = accepted_csv_options(csv_options)
-        @stream = Stream.new(url)
+        @first_row_headers = @csv_options[:headers] == true
+
+        encoding = @csv_options[:encoding] || Encoding.default_external
+        @stream = Stream.new(url, encoding: encoding)
       end
 
       def each(&block)
@@ -39,7 +42,7 @@ module StreamLines
       attr_reader :url
 
       def first_row_headers?
-        @csv_options[:headers] == true
+        @first_row_headers
       end
 
       def assign_first_row_headers(first_line)

data/lib/stream_lines/reading/json_lines.rb

@@ -8,10 +8,10 @@ module StreamLines
     class JSONLines
       include Enumerable
 
-      def initialize(url, **json_options)
+      def initialize(url, encoding: Encoding.default_external, **json_options)
         @url = url
         @json_options = json_options
-        @stream = Stream.new(url)
+        @stream = Stream.new(url, encoding: encoding)
       end
 
       def each(&block)

data/lib/stream_lines/reading/stream.rb

@@ -12,9 +12,10 @@ module StreamLines
 
       raise_on 400..599
 
-      def initialize(url)
+      def initialize(url, encoding: Encoding.default_external)
         @url = url
-        @buffer = StringIO.new
+        @encoding = encoding
+        @buffer = String.new(encoding: @encoding)
       end
 
       def each(&block)
@@ -33,22 +34,36 @@ module StreamLines
           lines.each { |line| block.call(line) }
         end
 
-        @buffer.rewind
-        block.call(@buffer.read) if @buffer.size.positive?
+        block.call(@buffer) if @buffer.size.positive?
       end
 
       def extract_lines(chunk)
-        lines = chunk.split($INPUT_RECORD_SEPARATOR, -1)
+        encoded_chunk = @buffer + chunk.to_s.dup.force_encoding(@encoding)
+        lines = split_lines(encoded_chunk)
+        @buffer = String.new(encoding: @encoding)
+        @buffer << lines.pop.to_s
 
-        if lines.length > 1
-          @buffer.rewind
-          lines.first.prepend(@buffer.read)
-          @buffer.truncate(0)
-        end
-
-        @buffer << lines.pop
         lines
       end
+
+      def split_lines(encoded_chunk)
+        encoded_chunk.split($INPUT_RECORD_SEPARATOR, -1)
+      rescue ArgumentError => e
+        raise e unless /invalid byte sequence/.match?(e.message)
+
+        # NOTE: (jdlubrano)
+        # The last byte in the chunk is most likely a part of a multibyte
+        # character that, on its own, is an invalid byte sequence.  So, we
+        # want to split the lines containing all valid bytes and make the
+        # trailing bytes the last line.  The last line eventually gets added
+        # to the buffer, prepended to the next chunk, and, hopefully, restores
+        # a valid byte sequence.
+        last_newline_index = encoded_chunk.rindex($INPUT_RECORD_SEPARATOR)
+        return [encoded_chunk] if last_newline_index.nil?
+
+        valid_lines = encoded_chunk[0...last_newline_index].split($INPUT_RECORD_SEPARATOR, -1)
+        valid_lines + [encoded_chunk[(last_newline_index + 1)..-1]].compact
+      end
     end
   end
 end

data/lib/stream_lines/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module StreamLines
-  VERSION = '0.3.0'
+  VERSION = '0.4.1'
 end

data/stream_lines.gemspec

@@ -33,13 +33,15 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler-audit'
   spec.add_development_dependency 'bundler-gem_version_tasks'
   spec.add_development_dependency 'byebug'
+  spec.add_development_dependency 'charlock_holmes'
   spec.add_development_dependency 'get_process_mem'
   spec.add_development_dependency 'memory_profiler'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '~> 0.79.0'
+  spec.add_development_dependency 'rubocop', '~> 1.27.0'
   spec.add_development_dependency 'simplecov', '~> 0.17'
   spec.add_development_dependency 'sinatra', '~> 2.0'
   spec.add_development_dependency 'sinatra-contrib', '~> 2.0'
+  spec.add_development_dependency 'webrick', '~> 1.7'
   spec.add_development_dependency 'webmock', '~> 3.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stream_lines
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Joel Lubrano
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-14 00:00:00.000000000 Z
+date: 2022-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: get_process_mem
   requirement: !ruby/object:Gem::Requirement
@@ -156,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.79.0
+        version: 1.27.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.79.0
+        version: 1.27.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -206,6 +220,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -220,19 +248,21 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-description: 
+description:
 email:
 - joel.lubrano@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/ruby-tests.yml"
+- ".github/workflows/style-check.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-gemset"
 - ".ruby-version"
-- ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
@@ -256,7 +286,7 @@ metadata:
   homepage_uri: https://github.com/jdlubrano/stream_lines
   source_code_uri: https://github.com/jdlubrano/stream_lines
   changelog_uri: https://github.com/jdlubrano/stream_lines/releases
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -271,8 +301,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: A utility to stream lines of a file over HTTP
 test_files: []

data/.travis.yml

@@ -1,15 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.7.0
-  - 2.6.0
-  - 2.5.0
-before_install: gem install bundler -v 2.0.2
-script:
-  - bundle exec bundle-audit check --update
-  - bundle exec rubocop
-  - bundle exec rspec
-branches:
-  only:
-    - master