stream_lines 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 222d5f5ba77ac40c17fe4faf3ba9aab8d611e042e432ecbb2b6328c63b3856fe
-  data.tar.gz: 79664c06171bfcc6c253f01c912aa7862193f875e4dc2b8002f52f55c017b994
+  metadata.gz: 9e832d9790c71431c43722f116cca885d3fee985001b12f5d0c53cc17e78c7b8
+  data.tar.gz: 76338bc7e203bfa59f3aed2469db3694951dfd5905c6bcfe416b08f1512f3289
 SHA512:
-  metadata.gz: dff1372c1048260c71d11fbab457ae9bd45723e52bf0485b141f8cc8885687d9cafe09d7a241d957a4f66ff6e8644fd65562a102b82ef73244607089e6774517
-  data.tar.gz: 6c22a6aa984ac5c216e1ecfc290b79f4fbb2f330bf6327497d50cf71a3d0ef527b518c18efb319c542b89297b77c22fe7fe4bf067094dda9f6d1efe8bc1e4053
+  metadata.gz: 3d980bdddd41227418da10447e57599ee94cea52b7ea926e1cb3f0efd8a3b71b7e8f44c70ed7a62dc37ea31ac97e84a1f5d20677ecb933bd26be7c2348e533c5
+  data.tar.gz: 721610a59f59862772b0ea259b6f22c0e84f058d458538ae993478324e9e315dc6fa0b469eaebb56fd630f59bd43b3bb43956f2111855c22cb00bb744296018c

data/.github/workflows/ci-tests.yml

@@ -0,0 +1,30 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - '*'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.6
+    - name: Install dependencies
+      run: bundle install
+    - name: Run tests
+      run: bundle exec rake

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    stream_lines (0.3.1)
+    stream_lines (0.4.0)
       httparty (~> 0.14)
 
 GEM
@@ -9,70 +9,76 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
-    ast (2.4.0)
+    ast (2.4.1)
     awesome_print (1.8.0)
-    backports (3.15.0)
-    bundler-audit (0.6.1)
+    backports (3.17.2)
+    bundler-audit (0.7.0.1)
       bundler (>= 1.2.0, < 3)
-      thor (~> 0.18)
+      thor (>= 0.18, < 2)
     bundler-gem_version_tasks (0.2.1)
-    byebug (11.1.1)
+    byebug (11.1.3)
+    charlock_holmes (0.7.7)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     diff-lcs (1.3)
     docile (1.3.2)
-    ffi (1.11.3)
+    ffi (1.13.0)
     get_process_mem (0.2.5)
       ffi (~> 1.0)
-    hashdiff (1.0.0)
-    httparty (0.17.3)
+    hashdiff (1.0.1)
+    httparty (0.18.1)
       mime-types (~> 3.0)
       multi_xml (>= 0.5.2)
-    jaro_winkler (1.5.4)
     memory_profiler (0.9.14)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
-    mime-types-data (3.2019.1009)
+    mime-types-data (3.2020.0512)
     multi_json (1.14.1)
     multi_xml (0.6.0)
     mustermann (1.1.1)
       ruby2_keywords (~> 0.0.1)
-    parallel (1.19.1)
-    parser (2.7.0.2)
-      ast (~> 2.4.0)
-    public_suffix (4.0.3)
-    rack (2.0.8)
+    parallel (1.19.2)
+    parser (2.7.1.4)
+      ast (~> 2.4.1)
+    public_suffix (4.0.5)
+    rack (2.2.3)
     rack-protection (2.0.8.1)
       rack
     rainbow (3.0.0)
     rake (13.0.1)
+    regexp_parser (1.7.1)
+    rexml (3.2.4)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
       rspec-mocks (~> 3.9.0)
-    rspec-core (3.9.1)
-      rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
-    rspec-support (3.9.2)
-    rubocop (0.79.0)
-      jaro_winkler (~> 1.5.1)
+    rspec-support (3.9.3)
+    rubocop (0.87.1)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.1)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.7)
+      rexml
+      rubocop-ast (>= 0.1.0, < 1.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.1.0)
+      parser (>= 2.7.0.1)
     ruby-progressbar (1.10.1)
     ruby2_keywords (0.0.2)
     safe_yaml (1.0.5)
-    simplecov (0.18.1)
+    simplecov (0.18.5)
       docile (~> 1.1)
-      simplecov-html (~> 0.11.0)
-    simplecov-html (0.11.0)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.2)
     sinatra (2.0.8.1)
       mustermann (~> 1.0)
       rack (~> 2.0)
@@ -85,10 +91,10 @@ GEM
       rack-protection (= 2.0.8.1)
       sinatra (= 2.0.8.1)
       tilt (~> 2.0)
-    thor (0.20.3)
+    thor (1.0.1)
     tilt (2.0.10)
-    unicode-display_width (1.6.0)
-    webmock (3.8.1)
+    unicode-display_width (1.7.0)
+    webmock (3.8.3)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
@@ -102,11 +108,12 @@ DEPENDENCIES
   bundler-audit
   bundler-gem_version_tasks
   byebug
+  charlock_holmes
   get_process_mem
   memory_profiler
   rake (~> 13.0)
   rspec (~> 3.0)
-  rubocop (~> 0.79.0)
+  rubocop (~> 0.87.1)
   simplecov (~> 0.17)
   sinatra (~> 2.0)
   sinatra-contrib (~> 2.0)

data/README.md

@@ -1,7 +1,7 @@
 # StreamLines
 
 [![Gem Version](https://badge.fury.io/rb/stream_lines.svg)](https://badge.fury.io/rb/stream_lines)
-[![Build Status](https://travis-ci.com/jdlubrano/stream_lines.svg?branch=master)](https://travis-ci.com/jdlubrano/stream_lines)
+![](https://github.com/jdlubrano/stream_lines/workflows/Tests/badge.svg)
 
 An API for streaming files from remote locations one line at a time.
 
@@ -119,10 +119,10 @@ To install this gem onto your local machine, run `bundle exec rake install`.
 
 ## Releasing
 
-After merging in the new functionality to the master branch:
+After merging in the new functionality to the main branch:
 
 ```
-git checkout master
+git checkout main
 git pull --prune
 bundle exec rake version:bump:<major, minor, or patch>
 bundle exec rubocop -a
@@ -135,7 +135,7 @@ bundle exec rake release
 Bug reports and pull requests are welcome on GitHub at
 https://github.com/jdlubrano/stream_lines. This project is intended to be a
 safe, welcoming space for collaboration, and contributors are expected to
-adhere to the [code of conduct](https://github.com/jdlubrano/stream_lines/blob/master/CODE_OF_CONDUCT.md).
+adhere to the [code of conduct](https://github.com/jdlubrano/stream_lines/blob/main/CODE_OF_CONDUCT.md).
 
 ## License
 

data/bin/console

@@ -3,7 +3,9 @@
 # frozen_string_literal: true
 
 require 'bundler/setup'
+require 'byebug'
 require 'stream_lines'
+require 'charlock_holmes/string'
 
 # You can add fixtures and/or initialization code here to make experimenting
 # with your gem easier. You can also use a different console, if you like.

data/lib/stream_lines/reading/csv.rb

@@ -23,7 +23,9 @@ module StreamLines
       def initialize(url, **csv_options)
         @url = url
         @csv_options = accepted_csv_options(csv_options)
-        @stream = Stream.new(url)
+
+        encoding = @csv_options[:encoding] || Encoding.default_external
+        @stream = Stream.new(url, encoding: encoding)
       end
 
       def each(&block)

data/lib/stream_lines/reading/json_lines.rb

@@ -8,10 +8,10 @@ module StreamLines
     class JSONLines
       include Enumerable
 
-      def initialize(url, **json_options)
+      def initialize(url, encoding: Encoding.default_external, **json_options)
         @url = url
         @json_options = json_options
-        @stream = Stream.new(url)
+        @stream = Stream.new(url, encoding: encoding)
       end
 
       def each(&block)

data/lib/stream_lines/reading/stream.rb

@@ -12,9 +12,10 @@ module StreamLines
 
       raise_on 400..599
 
-      def initialize(url)
+      def initialize(url, encoding: Encoding.default_external)
         @url = url
-        @buffer = StringIO.new
+        @encoding = encoding
+        @buffer = String.new(encoding: @encoding)
       end
 
       def each(&block)
@@ -33,22 +34,36 @@ module StreamLines
           lines.each { |line| block.call(line) }
         end
 
-        @buffer.rewind
-        block.call(@buffer.read) if @buffer.size.positive?
+        block.call(@buffer) if @buffer.size.positive?
       end
 
       def extract_lines(chunk)
-        lines = chunk.split($INPUT_RECORD_SEPARATOR, -1)
+        encoded_chunk = @buffer + chunk.to_s.dup.force_encoding(@encoding)
+        lines = split_lines(encoded_chunk)
+        @buffer = String.new(encoding: @encoding)
+        @buffer << lines.pop.to_s
 
-        if lines.length > 1
-          @buffer.rewind
-          lines.first.prepend(@buffer.read)
-          @buffer = StringIO.new
-        end
-
-        @buffer << lines.pop
         lines
       end
+
+      def split_lines(encoded_chunk)
+        encoded_chunk.split($INPUT_RECORD_SEPARATOR, -1)
+      rescue ArgumentError => e
+        raise e unless /invalid byte sequence/.match?(e.message)
+
+        # NOTE: (jdlubrano)
+        # The last byte in the chunk is most likely a part of a multibyte
+        # character that, on its own, is an invalid byte sequence.  So, we
+        # want to split the lines containing all valid bytes and make the
+        # trailing bytes the last line.  The last line eventually gets added
+        # to the buffer, prepended to the next chunk, and, hopefully, restores
+        # a valid byte sequence.
+        last_newline_index = encoded_chunk.rindex($INPUT_RECORD_SEPARATOR)
+        return [encoded_chunk] if last_newline_index.nil?
+
+        valid_lines = encoded_chunk[0...last_newline_index].split($INPUT_RECORD_SEPARATOR, -1)
+        valid_lines + [encoded_chunk[(last_newline_index + 1)..-1]].compact
+      end
     end
   end
 end

data/lib/stream_lines/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module StreamLines
-  VERSION = '0.3.1'
+  VERSION = '0.4.0'
 end

data/stream_lines.gemspec

@@ -33,11 +33,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bundler-audit'
   spec.add_development_dependency 'bundler-gem_version_tasks'
   spec.add_development_dependency 'byebug'
+  spec.add_development_dependency 'charlock_holmes'
   spec.add_development_dependency 'get_process_mem'
   spec.add_development_dependency 'memory_profiler'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rspec', '~> 3.0'
-  spec.add_development_dependency 'rubocop', '~> 0.79.0'
+  spec.add_development_dependency 'rubocop', '~> 0.87.1'
   spec.add_development_dependency 'simplecov', '~> 0.17'
   spec.add_development_dependency 'sinatra', '~> 2.0'
   spec.add_development_dependency 'sinatra-contrib', '~> 2.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stream_lines
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Joel Lubrano
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-04 00:00:00.000000000 Z
+date: 2020-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -94,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: charlock_holmes
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: get_process_mem
   requirement: !ruby/object:Gem::Requirement
@@ -156,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.79.0
+        version: 0.87.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.79.0
+        version: 0.87.1
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -227,12 +241,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci-tests.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
 - ".ruby-gemset"
 - ".ruby-version"
-- ".travis.yml"
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock

data/.travis.yml

@@ -1,15 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.7.0
-  - 2.6.0
-  - 2.5.0
-before_install: gem install bundler -v 2.0.2
-script:
-  - bundle exec bundle-audit check --update
-  - bundle exec rubocop
-  - bundle exec rspec
-branches:
-  only:
-    - master