nemweb 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 365f80b7362e4ef860617c2901c610f2b73480e798f9aecc70f4f97bb7fdb755
+  data.tar.gz: 25941839fbd9b5bda7a163d101b60ca36a5ac5040899918f43a081b777c741d9
+SHA512:
+  metadata.gz: c445e7f378714cd116569814f4341f66b59566a5ff5ca28d240e74a190c59a888db8fec12d15ecf856d83ce212ce151aa03cbf39f0f71cb5087b9348c0e5a13b
+  data.tar.gz: 10c2c08f7ffad64e142f565084484920e834a2480ff31126a0650680883b22ace76fa87b9ef0708bef4b46c1d0a9a0a23b4bf0afad0903b03fc447ef80cc2bd1

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.5.3
+before_install: gem install bundler -v 2.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in nemweb.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,43 @@
+PATH
+  remote: .
+  specs:
+    nemweb (0.2.0)
+      clamp
+      nokogiri
+      rubyzip
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    clamp (1.3.1)
+    diff-lcs (1.3)
+    mini_portile2 (2.4.0)
+    nokogiri (1.10.3)
+      mini_portile2 (~> 2.4.0)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubyzip (1.2.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.0)
+  nemweb!
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   2.0.1

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Mike Williams
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,39 @@
+# Nemweb
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/nemweb`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'nemweb'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install nemweb
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/nemweb.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "nemweb"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/nemweb

@@ -0,0 +1,145 @@
+#! /usr/bin/env ruby
+
+require 'clamp'
+require 'json'
+require 'set'
+require 'time'
+
+$stdout.sync = true
+$stderr.sync = true
+
+$LOAD_PATH << File.expand_path('../lib', __dir__)
+
+require 'nemweb/client'
+
+Clamp do
+
+  subcommand "dirs", "List NEMWEB data directories" do
+
+    def execute
+      nemweb.dirs.each do |dir|
+        puts dir.path.split("/").last
+      end
+    end
+
+  end
+
+  subcommand "ls", "List a NEMWEB data directory" do
+
+    option %w(-l --long), :flag, "print details"
+
+    parameter "DIR", "directory to list"
+
+    def execute
+      lags = []
+      nemweb.list_files(dir).each do |data_file|
+        line = data_file.uri.to_s
+        if long?
+          line += " lag=#{get_lag(data_file)}s"
+        end
+        puts line
+      end
+    end
+
+    private
+
+    def get_lag(data_file)
+      return [] unless long?
+      period_start = Time.parse("#{data_file.period} +1000")
+      last_modified = Time.parse(http_head(data_file.uri).fetch("last-modified").first)
+      (last_modified - period_start).to_i
+    end
+
+    def http_head(uri)
+      uri = URI(uri)
+      Net::HTTP.start(uri.host, uri.port) do |http|
+        http.head(uri.path).to_hash
+      end
+    end
+
+  end
+
+  subcommand "fetch", "Fetch a NEMWEB data file" do
+
+    option %w(--name -n), :flag, "display data file name"
+    option %w(--name-only -N), :flag, "display data file name, only"
+
+    parameter "SOURCE", "data file URI or path"
+
+    def execute
+      nemweb.fetch(source) do |io, name|
+        puts "file: #{name}" if name? || name_only?
+        IO.copy_stream(io, $stdout) unless name_only?
+      end
+    end
+
+  end
+
+  subcommand "parse", "Parse a NEMWEB data file" do
+
+    parameter "SOURCE", "data file URI or path"
+
+    def execute
+      nemweb.parse(source) do |record|
+        puts JSON.dump(record.to_h)
+      end
+    end
+
+  end
+
+  module DirectoryMonitoring
+
+    def self.included(target)
+      target.option ["-i", "--poll-interval"], "SECONDS", "polling interval", default: 10, &method(:Integer)
+      target.option ["-a", "--after"], "PERIOD", "last period processed"
+    end
+
+    def on_new_data_file(directories)
+      while true do
+        begin
+          directories.each do |dir|
+            files = nemweb.list_files(dir)
+            new_files = if cursors[dir].nil?
+              [files.last]
+            else
+              files.select { |data_file| data_file.period > cursors[dir] }
+            end
+            new_files.each do |data_file|
+              yield data_file
+              cursors[dir] = data_file.period
+            end
+          end
+        rescue OpenURI::HTTPError
+          puts "  HTTPError!"
+        end
+        sleep(poll_interval)
+      end
+    end
+
+    def cursors
+      @cursors ||= Hash.new(after)
+    end
+
+  end
+
+  subcommand "watch", "Watch a NEMWEB data directory" do
+
+    parameter "DIR ...", "directories to watch"
+
+    include DirectoryMonitoring
+
+    def execute
+      on_new_data_file(dir_list) do |data_file|
+        puts data_file.uri
+      end
+    end
+
+  end
+
+  private
+
+  def nemweb
+    Nemweb::Client.new
+  end
+
+end

data/lib/nemweb/client.rb

@@ -0,0 +1,81 @@
+require 'csv'
+require 'nemweb/data_file'
+require 'nemweb/data_record'
+require 'net/http'
+require 'nokogiri'
+require 'open-uri'
+require 'zip'
+
+module Nemweb
+
+  class Client
+
+    # Public: Load and parse data.
+    #
+    # source - a data file or URI
+    #
+    # Yields records, as Hash objects.
+    #
+    def parse(source)
+      fetch(source) do |stream|
+        headers = nil
+        CSV.parse(stream) do |row|
+          case row[0]
+          when "I"
+            headers = row[4..-1]
+          when "D"
+            record = DataRecord.new(
+              row[1], row[2], Integer(row[3]),
+              Hash[headers.zip(row[4..-1])]
+            )
+            yield record
+          end
+        end
+      end
+    end
+
+    CURRENT_REPORTS = URI("http://nemweb.com.au/Reports/Current/").freeze
+
+    # Public: Get the files in a NEM data directory.
+    #
+    # dir - subdirectory of http://nemweb.com.au/Reports/Current/
+    #
+    def list_files(dir)
+      dir_uri = CURRENT_REPORTS + dir
+      doc = Nokogiri::HTML.parse(dir_uri.read)
+      doc.css('a').map { |link| link["href"] }.grep(/\.zip$/).map do |href|
+        DataFile.new(dir_uri + href, self)
+      end
+    end
+
+    # Public: List available NEM data directories
+    #
+    def dirs(root_uri = CURRENT_REPORTS)
+      doc = Nokogiri::HTML.parse(root_uri.read)
+      doc.css('a').map { |link| root_uri + link["href"] }
+    end
+
+    # Open a data source, and yield a stream.
+    #
+    # source - a data file or URI
+    #
+    # If the data source is a ZIP file, extract the first entry.
+    #
+    def fetch(source)
+      source_uri = CURRENT_REPORTS + source
+      source_uri.open do |stream|
+        if source_uri.to_s =~ /zip\Z/i
+          Zip::File.open_buffer(stream) do |zipfile|
+            zipfile.entries.each do |entry|
+              yield entry.get_input_stream, entry.name
+            end
+          end
+        else
+          yield stream, source_uri
+        end
+      end
+    end
+
+  end
+
+end

data/lib/nemweb/data_file.rb

@@ -0,0 +1,24 @@
+module Nemweb
+
+  class DataFile < Struct.new(:uri)
+
+    def initialize(uri, client)
+      super(uri)
+      @client = client
+    end
+
+    def name
+      File.basename(uri.path)
+    end
+
+    def period
+      name.scan(/_(\d{4,})/).first.first
+    end
+
+    def parse(&block)
+      @client.parse(uri, &block)
+    end
+
+  end
+
+end

data/lib/nemweb/data_record.rb

@@ -0,0 +1,16 @@
+module Nemweb
+
+  class DataRecord < Struct.new(:type, :subtype, :version, :data)
+
+    def to_h
+      {
+        "type" => type,
+        "subtype" => subtype,
+        "version" => version,
+        "data" => data
+      }
+    end
+
+  end
+
+end

data/lib/nemweb/version.rb

@@ -0,0 +1,3 @@
+module Nemweb
+  VERSION = "0.2.0"
+end

data/nemweb.gemspec

@@ -0,0 +1,33 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "nemweb/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "nemweb"
+  spec.version       = Nemweb::VERSION
+  spec.authors       = ["Mike Williams"]
+  spec.email         = ["mike.williams@greensync.com.au"]
+
+  spec.summary       = %q{A Ruby client for nemweb.com.au}
+  spec.homepage      = "https://github.com/greensync/nemweb-ruby"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_runtime_dependency "clamp"
+  spec.add_runtime_dependency "nokogiri"
+  spec.add_runtime_dependency "rubyzip"
+
+end

metadata

@@ -0,0 +1,145 @@
+--- !ruby/object:Gem::Specification
+name: nemweb
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Mike Williams
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-07-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: clamp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- mike.williams@greensync.com.au
+executables:
+- nemweb
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/nemweb
+- lib/nemweb/client.rb
+- lib/nemweb/data_file.rb
+- lib/nemweb/data_record.rb
+- lib/nemweb/version.rb
+- nemweb.gemspec
+homepage: https://github.com/greensync/nemweb-ruby
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: A Ruby client for nemweb.com.au
+test_files: []