json_stream_trigger 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 11dbed3cd388bb5830c6956a4792bafb172d249a
+  data.tar.gz: 73ff8ca8f6301a079c9ebdddb531b292e683e641
+SHA512:
+  metadata.gz: 78a5fcc5b1a352fe79cbfbc0218d5d5f41ae3d34a83a69eb4c0cc85c94fb6004e440d0705c90de0822591b1fc6eca00ee12cf59a082386f4517a99882cc9e23b
+  data.tar.gz: ffbcc34b8cdb0564293d548368f13b233804819d691746ca7266b3c84759cae38b1f75e17572f5e9d6e028bd219553ef4744c2e9ec85bd6f1e2b406f9cfe3d39

data/README.md

@@ -0,0 +1,76 @@
+# Ruby `json_stream_trigger` Gem
+
+Instead of parsing a huge JSON files and loading it into memory,
+this library will stream the bytes through
+[json-stream](https://github.com/dgraham/json-stream) and only
+creates a small buffer for objects whose JSONPath matches a pattern you specify.
+When the object is completed, the specified block will be called.
+
+Install with `gem "json_stream_trigger"` in your Gemfile.
+
+## Example:
+
+```ruby
+f = File.open('really_big_file.json')
+stream = JsonStreamTrigger.new()
+
+# Match each array item. If you wanted to whole array use $.data
+stream.on('$.data[*]') do |json_string|
+  import JSON.parse(json_string, :quirks_mode => true)
+end
+
+# Will match for $.any.sub[*].item.meta
+stream.on('$..meta') do |json_string|
+  save_meta JSON.parse(json_string, :quirks_mode => true)
+end
+
+# read in 1MB chunks
+while chunk = f.read(1024)
+  stream << chunk
+end
+
+```
+
+The captured JSON strinb buffer will be passed to the block. Note, Ruby's JSON library expects
+JSON documents to be passed to it - not primatives - this is why `:quirks_mode => true` has been added.
+
+## Path Details
+The JSONPaths are similar to XPath notation. `$` is the root,
+single wild card keys can be done with `$.*.version`,
+or you can do muli-level wildcard with `$.docs..name`.
+[More info on JSONPath](http://goessner.net/articles/JsonPath/)
+
+A few more examples:
+
+```javascript
+{
+  meta: {version: 0.1},
+  docs: [
+    {id: 1},
+    {id: 2},
+    {id: 3},
+    {id: 4},
+    {
+      id: 5,
+      user: {
+        name: "Tyler"
+      }
+    }
+  ]
+}
+```
+
+```ruby
+on('$.docs[*].id') # triggers for id property of every item in docs array
+on('$.docs') # returns full array of items
+on('$.docs[*]') # triggers for each item in the array
+on('$.docs[1].id') # returns value of ID 1
+on('$.docs[*].*.name') # returns 'Tyler'
+on('$..name') # matches any value who's key is 'name'
+```
+
+
+
+## Tests
+`rake test`
+

data/lib/json_path.rb

@@ -0,0 +1,22 @@
+
+class JsonPath
+
+  def self.matches?(path, pattern)
+    re = self.convert_to_re(pattern)
+    !path.match(re).nil?
+  end
+
+  def self.convert_to_re(pattern)
+    re = pattern.dup
+    re = re.gsub('[', '\[').gsub(']', '\]')# escape brackets
+    re.gsub!(/^\$/, '^\$') # escape $ and fix it to root
+    re.gsub!(/ \\\[  \*  \\\] /x, '\[\d+\]') # change [*] to [\d+]
+    re.gsub!('..', '(?<=[\.\$\]]).*[\.\]]') # change .. to match a dot, $, or ] followed by anything, and ending in a . or ]
+    re.gsub!('.*.', '\.[^\.\[\]]+\.') # wild card will match any key
+    
+    re += '(?=$)' #'(?=$|\.)
+    return Regexp.new(re)
+  end
+  
+end
+

data/lib/json_stream_trigger.rb

@@ -0,0 +1,188 @@
+require 'json'
+require 'json/stream'
+require_relative 'json_path'
+
+class JsonStreamTrigger
+  attr_reader :key_path, :triggers, :full_buffer
+  DEBUG=false
+  
+  def initialize()
+    @parser = JSON::Stream::Parser.new
+    
+    @parser.start_object   &method(:start_object)
+    @parser.end_object     &method(:end_object)
+    @parser.start_array    &method(:start_array)
+    @parser.end_array      &method(:end_array)
+    @parser.key            &method(:key)
+    @parser.value          &method(:value)
+
+    @last_call      = nil
+    @key_path       = ''
+    @triggers       = {}
+    @active_buffers = {}
+    @full_buffer    = ''
+  end
+
+  def on(pattern, &block)
+    @triggers[pattern] = block
+  end
+  
+  def <<(bytes)
+    debug "bytes: #{bytes.inspect}"
+    @parser << bytes
+    @full_buffer << bytes if DEBUG
+  end
+  
+  def path_matches?(pattern)
+    JsonPath.matches?(@key_path, pattern)
+  end
+  
+  protected
+
+
+  ################################  PARSING TRIGGERS ###########################
+  #def start_document
+  #end
+  #def end_document
+  #end
+  
+  def start_object
+    debug "start object"
+    @key_path << (@key_path.empty? ? "$" : '')
+    increment_path_array() do
+      activate_buffers_for_matching()
+    end
+    append_buffers ',{'
+    @last_call = :start_object
+  end
+  
+  def end_object
+    debug "end object"
+    append_buffers '}' if @last_call == :start_object
+    trigger_block_for_matching()
+    append_buffers '}' if @last_call != :start_object
+    trim_segment(/[\.\$][^\.\[\[]+$/) # remove last .key
+    @last_call = :end_object
+  end
+  
+  def start_array
+    debug "start array"
+    increment_path_array() do
+      activate_buffers_for_matching()
+    end
+    append_buffers ',['
+    @key_path << (@key_path.empty? ? "$[]" : "[]")
+    activate_buffers_for_matching()
+    @last_call = :start_array
+  end
+  
+  def end_array
+    debug "end array"
+    append_buffers ']'
+    
+    trim_segment(/\[\d*\]+$/) # remove last [\d] and check triggers to match .my-array
+    trigger_block_for_matching()
+    trim_segment(/[\.$][^\.\[\]]+$/) # remove last .my-array
+    @last_call = :end_array
+  end
+  
+  def key(k)
+    debug "new key '#{k}'"
+    trim_segment(/\.[^\.\[\]]+$/) unless @last_call == :start_object# remove last .key[\d]
+    @key_path << ".#{k}"
+    append_buffers ",\"#{k}\":"
+    activate_buffers_for_matching()
+    @last_call = :key
+  end
+  
+  def value(v)
+    debug "value '#{v}'"
+    increment_path_array() do
+      activate_buffers_for_matching()
+    end
+    
+    append_buffers ','
+    append_buffers JSON.dump(v)
+    
+    trigger_block_for_matching()
+    @last_call = :value
+  end
+  
+  ################################ BUFFER STUFF ###########################
+  
+  # Called when we know the name of the object/array we are workign with
+  def activate_buffers_for_matching
+    @triggers.keys.each do |pattern|
+      debug "checking #{@key_path} matches #{pattern}"
+      if JsonPath.matches?(@key_path, pattern) && !@active_buffers.keys.include?(pattern)
+        debug ">> Activating buffer for #{pattern.inspect}"
+        @active_buffers[pattern] = ''
+      end
+    end
+  end
+
+  # To be called when exiting an object or array so the buffer is completed
+  def trigger_block_for_matching
+    active_patterns = @active_buffers.keys
+    active_patterns.each do |pattern|
+      if JsonPath.matches?(@key_path, pattern)
+        debug "<< Calling trigger for '#{pattern}'"
+        @triggers[pattern].call @active_buffers[pattern]
+        if pattern[-3..3] == '[*]'
+          @active_buffers[pattern] = ''
+          debug "Clearing buffer for '#{pattern}'"
+        else
+          @active_buffers.delete(pattern)
+          debug "Stopping buffer for '#{pattern}'"
+        end
+      end
+    end
+  end
+
+  def append_buffers(bytes)
+    @active_buffers.keys.each do |k|
+      # remove comma if it's not needed
+      if bytes[0] == ',' && [nil, '[', '{', ':'].include?(@active_buffers[k][-1, 1])
+        bytes = bytes[1..-1]
+      end
+      @active_buffers[k] += "#{bytes}" 
+      debug "Appended to #{k} => '#{bytes}' to buffer '#{@active_buffers[k]}'"
+    end
+  end
+
+  ############################### PATH STUFF #######################
+  
+  # trim off the last segment of the key_path
+  def trim_segment(re)
+    @key_path.sub!(re, '')
+    @key_path << '$' if @key_path == '' # add back the $ if we trimmed it off
+    debug "  trimmed off #{re}"
+  end
+  
+  def increment_path_array(&block)
+    # Increment the array index if we are in an array
+    # Note: Xpath indexes start at 1
+    # pull out the [\d] from the last array index
+    did_update = false
+    @key_path.sub!(/\[(\d*)\]$/) do |m, x|
+      debug "incrementing path array: #{@key_path}"
+      new_i = m.match(/\[(\d*)\]/)[1].to_i + 1
+      did_update = true
+      debug "  new array i = #{new_i}"
+      '[' + new_i.to_s + ']'
+    end
+
+    if did_update
+      block.call()
+    end
+    
+  end
+
+
+  
+  def debug(msg)
+    indent = 60
+    puts msg + (" " * [0, (indent - msg.length)].max ) + "PATH: #{@key_path}" if DEBUG
+  end
+  
+end

metadata

@@ -0,0 +1,47 @@
+--- !ruby/object:Gem::Specification
+name: json_stream_trigger
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Tyler Roberts
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-01-12 00:00:00.000000000 Z
+dependencies: []
+description: ''
+email: code@polar-concepts.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/json_path.rb
+- lib/json_stream_trigger.rb
+homepage: http://github.com/bdevel/json_stream_trigger
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Parse large JSON files as a stream and trigger events upon key matching.
+test_files: []
+has_rdoc: