aemo 0.1.39 → 0.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/aemo.rb +4 -0
- data/lib/aemo/dispatchable.rb +2 -0
- data/lib/aemo/market.rb +17 -11
- data/lib/aemo/market/interval.rb +3 -1
- data/lib/aemo/market/node.rb +4 -2
- data/lib/aemo/meter.rb +37 -0
- data/lib/aemo/msats.rb +7 -6
- data/lib/aemo/nem12.rb +40 -40
- data/lib/aemo/nem13.rb +2 -0
- data/lib/aemo/nmi.rb +170 -244
- data/lib/aemo/region.rb +3 -1
- data/lib/aemo/register.rb +42 -0
- data/lib/aemo/version.rb +3 -1
- data/lib/data/TNI-MLF-Codes.csv +644 -564
- data/lib/data/xml_to_json.rb +27 -11
- data/spec/aemo_spec.rb +2 -0
- data/spec/lib/aemo/market/interval_spec.rb +2 -0
- data/spec/lib/aemo/market/node_spec.rb +2 -0
- data/spec/lib/aemo/market_spec.rb +3 -1
- data/spec/lib/aemo/meter_spec.rb +18 -0
- data/spec/lib/aemo/msats_spec.rb +2 -0
- data/spec/lib/aemo/nem12_spec.rb +21 -1
- data/spec/lib/aemo/nmi_spec.rb +4 -1
- data/spec/lib/aemo/region_spec.rb +2 -0
- data/spec/spec_helper.rb +10 -0
- metadata +22 -38
data/lib/data/xml_to_json.rb
CHANGED
@@ -1,10 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'nokogiri'
|
2
4
|
require 'json'
|
3
5
|
require 'csv'
|
4
6
|
require 'active_support/all'
|
5
7
|
|
6
8
|
@path = Dir.pwd
|
7
|
-
@files = Dir.entries(@path).reject { |f| %w
|
9
|
+
@files = Dir.entries(@path).reject { |f| %w[. ..].include?(f) }
|
8
10
|
|
9
11
|
@mlf_data = {}
|
10
12
|
@dlf_data = {}
|
@@ -12,26 +14,38 @@ require 'active_support/all'
|
|
12
14
|
# Let's get the CSV Data first
|
13
15
|
|
14
16
|
# TNI to MLF
|
15
|
-
file_contents = File.read(File.join(@path, 'tni-mlf-codes.csv'))
|
17
|
+
file_contents = File.read(File.join(@path, 'tni-mlf-codes.csv'))
|
18
|
+
.encode('UTF-8', 'binary', invalid: :replace,
|
19
|
+
undef: :replace, replace: '')
|
16
20
|
CSV.parse(file_contents, headers: true, converters: :numeric).each do |row|
|
17
|
-
@mlf_data[row['TNI']] ||= { location: row['Location'],
|
18
|
-
|
21
|
+
@mlf_data[row['TNI']] ||= { location: row['Location'],
|
22
|
+
voltage: row['Voltage'],
|
23
|
+
loss_factors: [] }
|
24
|
+
row.headers.select { |x| x =~ /^FY\d{2}$/ }.sort.reverse.each do |fin_year|
|
19
25
|
year = "20#{fin_year.match(/FY(\d{2})/)[1]}".to_i
|
20
|
-
@mlf_data[row['TNI']][:loss_factors] << {
|
26
|
+
@mlf_data[row['TNI']][:loss_factors] << {
|
27
|
+
start: DateTime.parse("#{year - 1}-07-01T00:00:00+1000"),
|
28
|
+
finish: DateTime.parse("#{year}-07-01T00:00:00+1000"),
|
29
|
+
value: row[fin_year]
|
30
|
+
}
|
21
31
|
end
|
22
32
|
end
|
23
33
|
|
24
34
|
# TNI to MLF
|
25
|
-
CSV.open(File.join(@path, 'aemo-dlf-dnsp.csv'),
|
35
|
+
CSV.open(File.join(@path, 'aemo-dlf-dnsp.csv'),
|
36
|
+
headers: true, converters: :numeric).each do |row|
|
26
37
|
@dlf_data[row['dlf_code']] ||= row['nsp_code']
|
27
38
|
end
|
28
39
|
|
29
40
|
# Now to create the DLF and TNI output JSON files for use
|
30
|
-
@files.select { |x| ['aemo-tni.xml', 'aemo-dlf.xml'].include?(x) }
|
41
|
+
@files.select { |x| ['aemo-tni.xml', 'aemo-dlf.xml'].include?(x) }
|
42
|
+
.each do |file|
|
31
43
|
output_file = file.gsub('.xml', '.json')
|
32
44
|
output_data = {}
|
33
45
|
open_file = File.open(File.join(@path, file))
|
34
|
-
xml = Nokogiri::XML(open_file)
|
46
|
+
xml = Nokogiri::XML(open_file) do |c|
|
47
|
+
c.options = Nokogiri::XML::ParseOptions::NOBLANKS
|
48
|
+
end
|
35
49
|
open_file.close
|
36
50
|
|
37
51
|
xml.xpath('//Row').each do |row|
|
@@ -47,10 +61,12 @@ end
|
|
47
61
|
output_data_instance[:mlf_data] = {}
|
48
62
|
unless @mlf_data[code].nil?
|
49
63
|
output_data_instance[:mlf_data] = @mlf_data[code].deep_dup
|
50
|
-
output_data_instance[:mlf_data][:loss_factors]
|
51
|
-
DateTime.parse(output_data_instance['ToDate']) < x[:start] ||
|
64
|
+
output_data_instance[:mlf_data][:loss_factors].reject! do |x|
|
65
|
+
DateTime.parse(output_data_instance['ToDate']) < x[:start] ||
|
66
|
+
DateTime.parse(output_data_instance['FromDate']) >= x[:finish]
|
52
67
|
end
|
53
|
-
puts
|
68
|
+
puts 'output_data_instance[:mlf_data][:loss_factors]: ' \
|
69
|
+
"#{output_data_instance[:mlf_data][:loss_factors].inspect}"
|
54
70
|
end
|
55
71
|
elsif file =~ /dlf/
|
56
72
|
output_data_instance[:nsp_code] = @dlf_data[code]
|
data/spec/aemo_spec.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'spec_helper'
|
2
4
|
|
3
5
|
describe AEMO::Market do
|
@@ -30,7 +32,7 @@ describe AEMO::Market do
|
|
30
32
|
|
31
33
|
describe '.historic_trading' do
|
32
34
|
it 'has an array of data' do
|
33
|
-
expect(AEMO::Market.historic_trading('NSW', 2015,
|
35
|
+
expect(AEMO::Market.historic_trading('NSW', 2015, 1).class).to eq(Array)
|
34
36
|
end
|
35
37
|
end
|
36
38
|
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'spec_helper'
|
4
|
+
|
5
|
+
describe AEMO::Meter do
|
6
|
+
describe 'instance methods' do
|
7
|
+
it 'creates a new instance' do
|
8
|
+
expect(AEMO::Meter.new).to be_a AEMO::Meter
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'can be initialized from MSATS mumbo jumbo' do
|
12
|
+
AEMO::MSATS.authorize('ER', 'ER', 'ER')
|
13
|
+
nmi_detail_query = AEMO::MSATS.nmi_detail('4001234567')
|
14
|
+
meter = nmi_detail_query['MeterRegister']['Meter'].first
|
15
|
+
expect(AEMO::Meter.from_hash(meter)).to be_a AEMO::Meter
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
data/spec/lib/aemo/msats_spec.rb
CHANGED
data/spec/lib/aemo/nem12_spec.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'spec_helper'
|
2
4
|
require 'json'
|
3
5
|
|
@@ -10,12 +12,24 @@ describe AEMO::NEM12 do
|
|
10
12
|
end
|
11
13
|
end
|
12
14
|
|
15
|
+
describe '#nmi_identifier' do
|
16
|
+
it 'returns the NMI identifier or nil' do
|
17
|
+
Dir.entries(File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'NEM12'))
|
18
|
+
.reject { |f| %w[. .. .DS_Store].include?(f) }
|
19
|
+
.each do |file|
|
20
|
+
AEMO::NEM12.parse_nem12_file(fixture(File.join('NEM12', file))).each do |nem12|
|
21
|
+
expect(nem12.nmi_identifier).to be_a String
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
13
27
|
describe '#parse_nem12' do
|
14
28
|
end
|
15
29
|
|
16
30
|
describe '.parse_nem12_file' do
|
17
31
|
it 'should parse a file' do
|
18
|
-
Dir.entries(File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'NEM12')).reject { |f| %w
|
32
|
+
Dir.entries(File.join(File.dirname(__FILE__), '..', '..', 'fixtures', 'NEM12')).reject { |f| %w[. .. .DS_Store].include?(f) }.each do |file|
|
19
33
|
expect(AEMO::NEM12.parse_nem12_file(fixture(File.join('NEM12', file))).length).not_to eq(0)
|
20
34
|
end
|
21
35
|
end
|
@@ -52,5 +66,11 @@ describe AEMO::NEM12 do
|
|
52
66
|
end
|
53
67
|
|
54
68
|
describe '#flag_to_s' do
|
69
|
+
it 'converts the flags to a string' do
|
70
|
+
flag = { quality_flag: 'S', method_flag: 11, reason_code: 53 }
|
71
|
+
nem12 = AEMO::NEM12.new('NEEE000010')
|
72
|
+
expect(nem12.flag_to_s(flag))
|
73
|
+
.to eq 'Substituted Data - Check - Bees/Wasp In Meter Box'
|
74
|
+
end
|
55
75
|
end
|
56
76
|
end
|
data/spec/lib/aemo/nmi_spec.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'spec_helper'
|
2
4
|
require 'json'
|
3
5
|
|
@@ -90,6 +92,7 @@ describe AEMO::NMI do
|
|
90
92
|
end
|
91
93
|
end
|
92
94
|
end
|
95
|
+
|
93
96
|
describe '#valid_nmi?' do
|
94
97
|
it 'should validate nmi' do
|
95
98
|
json.each do |nmi|
|
@@ -153,7 +156,7 @@ describe AEMO::NMI do
|
|
153
156
|
end
|
154
157
|
it 'should return a friendly address if the address is a nested hash' do
|
155
158
|
nmi = AEMO::NMI.new('4001234567')
|
156
|
-
nmi.address = { house: { number: '1', suffix: 'B' }, street: 'Bob', street_type: 'Street'
|
159
|
+
nmi.address = { house: { number: '1', suffix: 'B' }, street: 'Bob', street_type: 'Street' }
|
157
160
|
expect(nmi.friendly_address).to eq('1 B, Bob, Street')
|
158
161
|
end
|
159
162
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'coveralls'
|
2
4
|
require 'simplecov'
|
3
5
|
require 'webmock/rspec'
|
@@ -21,6 +23,14 @@ RSpec.configure do |config|
|
|
21
23
|
csv_headers = { 'Content-Type' => 'text/csv' }
|
22
24
|
xml_headers = { 'Content-Type' => 'text/xml' }
|
23
25
|
|
26
|
+
# Updated Market Data
|
27
|
+
stub_request(:get, 'https://aemo.com.au/aemo/data/nem/priceanddemand/PRICE_AND_DEMAND_201501_NSW1.csv')
|
28
|
+
.with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
|
29
|
+
.to_return(status: 200, body: File.new('spec/fixtures/Market/DATA201501_NSW1.csv'), headers: csv_headers)
|
30
|
+
stub_request(:get, 'https://aemo.com.au/aemo/data/nem/priceanddemand/PRICE_AND_DEMAND_201502_NSW1.csv')
|
31
|
+
.with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
|
32
|
+
.to_return(status: 200, body: File.new('spec/fixtures/Market/DATA201502_NSW1.csv'), headers: csv_headers)
|
33
|
+
|
24
34
|
# Market Data
|
25
35
|
stub_request(:get, 'http://www.nemweb.com.au/mms.GRAPHS/GRAPHS/GRAPH_5NSW1.csv')
|
26
36
|
.with(headers: { 'Accept' => '*/*', 'User-Agent' => 'Ruby' })
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aemo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.40
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Joel Courtney
|
@@ -46,26 +46,6 @@ dependencies:
|
|
46
46
|
- - ">="
|
47
47
|
- !ruby/object:Gem::Version
|
48
48
|
version: 1.6.8
|
49
|
-
- !ruby/object:Gem::Dependency
|
50
|
-
name: rubyzip
|
51
|
-
requirement: !ruby/object:Gem::Requirement
|
52
|
-
requirements:
|
53
|
-
- - "~>"
|
54
|
-
- !ruby/object:Gem::Version
|
55
|
-
version: '1.1'
|
56
|
-
- - ">="
|
57
|
-
- !ruby/object:Gem::Version
|
58
|
-
version: 1.1.7
|
59
|
-
type: :runtime
|
60
|
-
prerelease: false
|
61
|
-
version_requirements: !ruby/object:Gem::Requirement
|
62
|
-
requirements:
|
63
|
-
- - "~>"
|
64
|
-
- !ruby/object:Gem::Version
|
65
|
-
version: '1.1'
|
66
|
-
- - ">="
|
67
|
-
- !ruby/object:Gem::Version
|
68
|
-
version: 1.1.7
|
69
49
|
- !ruby/object:Gem::Dependency
|
70
50
|
name: multi_xml
|
71
51
|
requirement: !ruby/object:Gem::Requirement
|
@@ -92,40 +72,40 @@ dependencies:
|
|
92
72
|
requirements:
|
93
73
|
- - "~>"
|
94
74
|
- !ruby/object:Gem::Version
|
95
|
-
version:
|
75
|
+
version: 0.13.0
|
96
76
|
- - ">="
|
97
77
|
- !ruby/object:Gem::Version
|
98
|
-
version: 0.13.
|
78
|
+
version: 0.13.7
|
99
79
|
type: :runtime
|
100
80
|
prerelease: false
|
101
81
|
version_requirements: !ruby/object:Gem::Requirement
|
102
82
|
requirements:
|
103
83
|
- - "~>"
|
104
84
|
- !ruby/object:Gem::Version
|
105
|
-
version:
|
85
|
+
version: 0.13.0
|
106
86
|
- - ">="
|
107
87
|
- !ruby/object:Gem::Version
|
108
|
-
version: 0.13.
|
88
|
+
version: 0.13.7
|
109
89
|
- !ruby/object:Gem::Dependency
|
110
90
|
name: activesupport
|
111
91
|
requirement: !ruby/object:Gem::Requirement
|
112
92
|
requirements:
|
113
|
-
- - "
|
93
|
+
- - "~>"
|
114
94
|
- !ruby/object:Gem::Version
|
115
|
-
version: 4.
|
116
|
-
- - "
|
95
|
+
version: '4.2'
|
96
|
+
- - ">="
|
117
97
|
- !ruby/object:Gem::Version
|
118
|
-
version:
|
98
|
+
version: 4.2.6
|
119
99
|
type: :runtime
|
120
100
|
prerelease: false
|
121
101
|
version_requirements: !ruby/object:Gem::Requirement
|
122
102
|
requirements:
|
123
|
-
- - "
|
103
|
+
- - "~>"
|
124
104
|
- !ruby/object:Gem::Version
|
125
|
-
version: 4.
|
126
|
-
- - "
|
105
|
+
version: '4.2'
|
106
|
+
- - ">="
|
127
107
|
- !ruby/object:Gem::Version
|
128
|
-
version:
|
108
|
+
version: 4.2.6
|
129
109
|
- !ruby/object:Gem::Dependency
|
130
110
|
name: listen
|
131
111
|
requirement: !ruby/object:Gem::Requirement
|
@@ -372,20 +352,20 @@ dependencies:
|
|
372
352
|
requirements:
|
373
353
|
- - "~>"
|
374
354
|
- !ruby/object:Gem::Version
|
375
|
-
version:
|
355
|
+
version: 0.41.0
|
376
356
|
- - ">="
|
377
357
|
- !ruby/object:Gem::Version
|
378
|
-
version: 0.41.
|
358
|
+
version: 0.41.2
|
379
359
|
type: :development
|
380
360
|
prerelease: false
|
381
361
|
version_requirements: !ruby/object:Gem::Requirement
|
382
362
|
requirements:
|
383
363
|
- - "~>"
|
384
364
|
- !ruby/object:Gem::Version
|
385
|
-
version:
|
365
|
+
version: 0.41.0
|
386
366
|
- - ">="
|
387
367
|
- !ruby/object:Gem::Version
|
388
|
-
version: 0.41.
|
368
|
+
version: 0.41.2
|
389
369
|
- !ruby/object:Gem::Dependency
|
390
370
|
name: timecop
|
391
371
|
requirement: !ruby/object:Gem::Requirement
|
@@ -421,11 +401,13 @@ files:
|
|
421
401
|
- lib/aemo/market.rb
|
422
402
|
- lib/aemo/market/interval.rb
|
423
403
|
- lib/aemo/market/node.rb
|
404
|
+
- lib/aemo/meter.rb
|
424
405
|
- lib/aemo/msats.rb
|
425
406
|
- lib/aemo/nem12.rb
|
426
407
|
- lib/aemo/nem13.rb
|
427
408
|
- lib/aemo/nmi.rb
|
428
409
|
- lib/aemo/region.rb
|
410
|
+
- lib/aemo/register.rb
|
429
411
|
- lib/aemo/version.rb
|
430
412
|
- lib/data/TNI-MLF-Codes.csv
|
431
413
|
- lib/data/aemo-dlf-dnsp.csv
|
@@ -548,6 +530,7 @@ files:
|
|
548
530
|
- spec/lib/aemo/market/interval_spec.rb
|
549
531
|
- spec/lib/aemo/market/node_spec.rb
|
550
532
|
- spec/lib/aemo/market_spec.rb
|
533
|
+
- spec/lib/aemo/meter_spec.rb
|
551
534
|
- spec/lib/aemo/msats_spec.rb
|
552
535
|
- spec/lib/aemo/nem12_spec.rb
|
553
536
|
- spec/lib/aemo/nmi_spec.rb
|
@@ -574,7 +557,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
574
557
|
version: '0'
|
575
558
|
requirements: []
|
576
559
|
rubyforge_project:
|
577
|
-
rubygems_version: 2.6.
|
560
|
+
rubygems_version: 2.6.13
|
578
561
|
signing_key:
|
579
562
|
specification_version: 4
|
580
563
|
summary: Gem providing functionality for the Australian Energy Market Operator data
|
@@ -693,6 +676,7 @@ test_files:
|
|
693
676
|
- spec/lib/aemo/market/interval_spec.rb
|
694
677
|
- spec/lib/aemo/market/node_spec.rb
|
695
678
|
- spec/lib/aemo/market_spec.rb
|
679
|
+
- spec/lib/aemo/meter_spec.rb
|
696
680
|
- spec/lib/aemo/msats_spec.rb
|
697
681
|
- spec/lib/aemo/nem12_spec.rb
|
698
682
|
- spec/lib/aemo/nmi_spec.rb
|