nm_datafile 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/changelog.md +26 -4
- data/lib/nm_datafile/crypto.rb +4 -2
- data/lib/nm_datafile/data_loading.rb +22 -1
- data/lib/nm_datafile/debug.rb +184 -0
- data/lib/nm_datafile/file_encoding.rb +92 -0
- data/lib/nm_datafile/nm_datafile.rb +8 -281
- data/lib/nm_datafile/version.rb +1 -1
- data/lib/nm_datafile.rb +14 -6
- data/spec/data/nmd_binary_string_w_2s_2li_2a_1ubws_1ep.zip +0 -0
- data/spec/factories/sales.rb +94 -0
- data/spec/nm_datafile_spec.rb +38 -20
- data/spec/spec_helper.rb +5 -0
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b2e0ec29390c2ebe162f08ef1b29cfb539ed905d
|
4
|
+
data.tar.gz: 433fa346539ce9b8f3bdc3c191c11b747b810ff3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6f81158867206df6bd21ed0bb20f5cbbb79f0f0fded7ca4b19c48a602f801f0ebc491be55bbb27aaefe334be4f0fb5d3ac066416674adbc228fcd6e8fc2dfba0
|
7
|
+
data.tar.gz: 0f74419eac0eadf844654df09258e3c904a88020892822fcb6f2e3cc59ba3761ceedef8e063b6908c05f9b7ae074287e6fe19d7c787c704784e54cf813c45fe7
|
data/changelog.md
CHANGED
@@ -1,8 +1,32 @@
|
|
1
|
+
### TODO:
|
2
|
+
|
3
|
+
* Fix the way there's an nm_datafile class... :(
|
4
|
+
* Define the class methods more clearly...
|
5
|
+
|
6
|
+
* Handle $FrontDoorKey
|
7
|
+
- When Crypto::clean_decrypt_string is called... it should be called on an instantiation
|
8
|
+
that has a symmetric_key set
|
9
|
+
- it should also raise an exception if that instance variable isn't set
|
10
|
+
- Crypto::clean_decrypt_string is only used from within the gem!
|
11
|
+
- data_loading.rb
|
12
|
+
- nm_datafile.rb
|
13
|
+
|
14
|
+
- It's hard to tell what's an instance method and what's a class method
|
15
|
+
with all the mixin usage...
|
16
|
+
|
17
|
+
|
18
|
+
### 0.0.2
|
19
|
+
|
20
|
+
* Created some tests
|
21
|
+
- Included tests that were in model/nm_datafile_spec.rb
|
22
|
+
|
23
|
+
|
24
|
+
|
25
|
+
|
1
26
|
### 0.0.1
|
2
27
|
|
3
28
|
* Code migrated into this repo
|
4
|
-
*
|
5
|
-
* Get pull out the class methods from the class
|
29
|
+
* Pull out the class methods from the class
|
6
30
|
** 1. Pull all those methods into a module
|
7
31
|
** 2. Include them on the class where needed.
|
8
32
|
** 3. Extend them onto the parent namespace module
|
@@ -12,5 +36,3 @@
|
|
12
36
|
### 0.0.0
|
13
37
|
|
14
38
|
Skeleton
|
15
|
-
|
16
|
-
|
data/lib/nm_datafile/crypto.rb
CHANGED
@@ -76,11 +76,13 @@ module NmDatafile
|
|
76
76
|
end
|
77
77
|
|
78
78
|
|
79
|
-
|
79
|
+
# This method needs to be available on the
|
80
|
+
# NmDatafile module
|
80
81
|
def clean_decrypt_string(string)
|
81
82
|
# string = Base64.decode64 string
|
82
83
|
# decode_password_protected_string(@@unsecure_pass, string)
|
83
|
-
|
84
|
+
unsecure_pass = ::NmDatafile::FRONT_DOOR_KEY
|
85
|
+
fast_decrypt_string_with_pass(unsecure_pass, string)
|
84
86
|
end
|
85
87
|
|
86
88
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
module NmDatafile
|
2
2
|
|
3
3
|
module DataLoading
|
4
|
+
|
4
5
|
# (m) Load: loads a file into memory as an NmDatafile
|
5
6
|
# TODO: Make lowercase
|
6
7
|
def Load(file_path)
|
@@ -26,9 +27,29 @@ module NmDatafile
|
|
26
27
|
attributes_hash["file_type"].to_sym
|
27
28
|
end
|
28
29
|
|
30
|
+
|
29
31
|
def determine_password(hash)
|
30
32
|
d = YAML::load hash[:encryption]
|
31
|
-
|
33
|
+
|
34
|
+
::NmDatafile.clean_decrypt_string(d["password"])
|
35
|
+
end
|
36
|
+
|
37
|
+
# This method peers through a zip binary data blob and returns a hash consisting of
|
38
|
+
# { file_name1: file_data1, etc }
|
39
|
+
def extract_entities_from_binary_data(binary_data)
|
40
|
+
binary_data_io = StringIO.new(binary_data)
|
41
|
+
|
42
|
+
hash = {}
|
43
|
+
::Zip::InputStream.open(binary_data_io) do |io|
|
44
|
+
while (entry = io.get_next_entry)
|
45
|
+
hash[entry.name.to_sym] = io.read
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
password = self.determine_password(hash)
|
50
|
+
|
51
|
+
decrypt_encryptable_data!(password, hash)
|
52
|
+
hash
|
32
53
|
end
|
33
54
|
|
34
55
|
end
|
@@ -0,0 +1,184 @@
|
|
1
|
+
module NmDatafile
|
2
|
+
|
3
|
+
# Just a (drunken) note about all this stuff... This code, is specific to the rails app AnonymousPublications
|
4
|
+
# And is dependant on factory_girl, the factories defined in the rails app, and active record and also
|
5
|
+
# the sale model in AnonymousPubliactions. That sucks (sux), right? Yes.
|
6
|
+
# To fix it, I should not have a #create_sale method, instead I should create those sales using Factories
|
7
|
+
# in the rails app, and then send those records in... For now, this stuff is just going to sit here though
|
8
|
+
module Debug
|
9
|
+
|
10
|
+
###################
|
11
|
+
# Testing methods #
|
12
|
+
###################
|
13
|
+
|
14
|
+
# do as address_completion_file to create an rfsb that should have already existed, but didn't because a fresh db was used
|
15
|
+
# for testing
|
16
|
+
def simulate_rfsb_existance_on_webserver
|
17
|
+
rfsb = self.ready_for_shipment_batch
|
18
|
+
ReadyForShipmentBatch.create(batch_stamp: rfsb["batch_stamp"], sf_integrity_hash: rfsb["integrity_hash"])
|
19
|
+
end
|
20
|
+
|
21
|
+
# creates an address_completion file based on the contents of the current shippable_file object
|
22
|
+
def simulate_address_completion_response(n_shipped)
|
23
|
+
raise "can't make an address_completion_response unless it's a shippable_file" if @file_type != :shippable_file
|
24
|
+
setup_object_for_schema # TODu: I put this in, there was a bug in the test where it needs to be run... does it not run on init somehow?
|
25
|
+
|
26
|
+
shipped_sales = []
|
27
|
+
erroneous_addresses = []
|
28
|
+
sales.each.with_index do |sale, i|
|
29
|
+
if i < n_shipped
|
30
|
+
shipped_sales << { "id" => sale["id"] }
|
31
|
+
else
|
32
|
+
erroneous_addresses << { "id" => sale["id"] }
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
simulated_response = [shipped_sales, erroneous_addresses, ready_for_shipment_batch]
|
37
|
+
|
38
|
+
nmd_address_completion = NmDatafile.new(:address_completion_file, *simulated_response)
|
39
|
+
end
|
40
|
+
|
41
|
+
def generate_upload_params(action = "upload_shippable_file")
|
42
|
+
temp_zip = Tempfile.new('temp_zip', "#{Rails.root}/tmp")
|
43
|
+
save_to_file(temp_zip.path)
|
44
|
+
|
45
|
+
upload_shippable_params = {
|
46
|
+
"file_upload" => { "my_file" => Rack::Test::UploadedFile.new(temp_zip.path, "application/zip") },
|
47
|
+
"controller"=>"admin_pages",
|
48
|
+
"action"=>action
|
49
|
+
}
|
50
|
+
end
|
51
|
+
|
52
|
+
# Don't use push on #sales, it will use the push on the array. TODu: to fix this, make the @sales arrays collections
|
53
|
+
# and give those collections special properties when push is invoked
|
54
|
+
def add_sale(sale)
|
55
|
+
# add to collection the sale
|
56
|
+
self.sales << sale
|
57
|
+
regenerate_rfsb if should_gen_a_new_rfsb?(sale)
|
58
|
+
end
|
59
|
+
|
60
|
+
# we should NOT make a new rfsb if we're adding an old erroneous sale to the object
|
61
|
+
# to check for age... see if it already has an rfsb_id
|
62
|
+
def should_gen_a_new_rfsb?(sale)
|
63
|
+
return true if sale.ready_for_shipment_batch_id.nil?
|
64
|
+
false
|
65
|
+
end
|
66
|
+
|
67
|
+
def regenerate_rfsb
|
68
|
+
rfsb = ReadyForShipmentBatch.gen
|
69
|
+
self.ready_for_shipment_batch = rfsb
|
70
|
+
rfsb.delete
|
71
|
+
end
|
72
|
+
|
73
|
+
# create's n valid sales and e erroneous sales
|
74
|
+
# Line items are created in this process
|
75
|
+
# This is to be run form AnonymousPublications only
|
76
|
+
# until it gets 'the big refactor'
|
77
|
+
def create_sales_for_shippable_file(n, e=nil)
|
78
|
+
|
79
|
+
if @file_type == :shippable_file
|
80
|
+
s, l, a, rfsb = create_sales_and_return_data(n)
|
81
|
+
self.sales += s
|
82
|
+
self.line_items += l
|
83
|
+
self.addresses += a
|
84
|
+
self.ready_for_shipment_batch = rfsb
|
85
|
+
rfsb.delete
|
86
|
+
Sale.deep_delete_sales(s)
|
87
|
+
elsif @file_type == :address_completion_file
|
88
|
+
s, e, rfsb = create_sales_and_return_data_address_completion_file(n, e)
|
89
|
+
self.sales += s
|
90
|
+
self.erroneous_sales += e
|
91
|
+
|
92
|
+
self.ready_for_shipment_batch = rfsb
|
93
|
+
rfsb.delete
|
94
|
+
Sale.deep_delete_sales(s)
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
98
|
+
alias create_sales create_sales_for_shippable_file
|
99
|
+
|
100
|
+
|
101
|
+
# puts "Deprecated due to the test being specific to a kind of model and schema"
|
102
|
+
def create_sales_and_return_data_address_completion_file(n, e)
|
103
|
+
e = 0 if e.nil?
|
104
|
+
sales = []
|
105
|
+
errors = []
|
106
|
+
n.times { sales << FactoryGirl.create(:sale_with_1_book) }
|
107
|
+
e.times { errors << FactoryGirl.create(:sale_with_1_book) }
|
108
|
+
rfsb = ReadyForShipmentBatch.gen
|
109
|
+
|
110
|
+
sales.each {|s| s.ready_for_shipment_batch_id = rfsb.id}
|
111
|
+
errors.each {|s| s.ready_for_shipment_batch_id = rfsb.id}
|
112
|
+
|
113
|
+
[ sales, errors, rfsb ]
|
114
|
+
end
|
115
|
+
|
116
|
+
# puts "Deprecated due to the test being specific to a kind of model and schema"
|
117
|
+
def create_sales_and_return_data(n)
|
118
|
+
sales = []
|
119
|
+
n.times { sales << FactoryGirl.create(:sale_with_1_book) }
|
120
|
+
rfsb = ReadyForShipmentBatch.gen
|
121
|
+
sales.each {|s| s.ready_for_shipment_batch_id = rfsb.id}
|
122
|
+
l = capture_line_items(sales)
|
123
|
+
a = capture_addresses(sales)
|
124
|
+
|
125
|
+
[sales, l, a, rfsb]
|
126
|
+
end
|
127
|
+
|
128
|
+
def capture_line_items(sales)
|
129
|
+
l = []
|
130
|
+
sales.each do |s|
|
131
|
+
l += s.line_items
|
132
|
+
end
|
133
|
+
l
|
134
|
+
end
|
135
|
+
|
136
|
+
def capture_addresses(sales)
|
137
|
+
a = []
|
138
|
+
sales.each do |s|
|
139
|
+
a << s.address
|
140
|
+
end
|
141
|
+
a
|
142
|
+
end
|
143
|
+
|
144
|
+
|
145
|
+
|
146
|
+
#################
|
147
|
+
# Debug methods #
|
148
|
+
#################
|
149
|
+
# render a count of sales
|
150
|
+
def to_s
|
151
|
+
string = "NmDatafile: \n"
|
152
|
+
data_collection_names.each.with_index do |collection_name, i|
|
153
|
+
string << " #{collection_name}: #{@data_collections[i].count} \n"
|
154
|
+
end
|
155
|
+
|
156
|
+
data_object_names.each.with_index do |variable_name, i|
|
157
|
+
string << " #{variable_name}: #{1} \n"
|
158
|
+
end
|
159
|
+
|
160
|
+
string
|
161
|
+
end
|
162
|
+
|
163
|
+
def inspect
|
164
|
+
puts self.to_s
|
165
|
+
end
|
166
|
+
|
167
|
+
def ==(other)
|
168
|
+
return false if other.class != self.class
|
169
|
+
return true if all_data_matches?(other)
|
170
|
+
false
|
171
|
+
end
|
172
|
+
|
173
|
+
def all_data_matches?(other)
|
174
|
+
if self.integrity_hash == other.integrity_hash
|
175
|
+
if self.build_attributes == other.build_attributes
|
176
|
+
return true
|
177
|
+
end
|
178
|
+
end
|
179
|
+
false
|
180
|
+
end
|
181
|
+
|
182
|
+
end
|
183
|
+
|
184
|
+
end
|
@@ -0,0 +1,92 @@
|
|
1
|
+
#################
|
2
|
+
# File Encoding #
|
3
|
+
#################
|
4
|
+
require 'zip'
|
5
|
+
module NmDatafile
|
6
|
+
|
7
|
+
module FileEncoding
|
8
|
+
|
9
|
+
# hash consists of {file1: string_data}
|
10
|
+
# output is a binary string representing a zip file of the entries specified
|
11
|
+
def encode_datafiles_as_zip(hash_of_entries)
|
12
|
+
temp_file = Tempfile.new(file_type.to_s, get_temp_directory)
|
13
|
+
FileUtils.rm temp_file.path
|
14
|
+
|
15
|
+
stream = ::Zip::OutputStream.write_buffer do |zos|
|
16
|
+
hash_of_entries.each do |entry_name, data|
|
17
|
+
zos.put_next_entry entry_name
|
18
|
+
zos.write data
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
stream.rewind
|
23
|
+
stream.read
|
24
|
+
end
|
25
|
+
|
26
|
+
# Play:
|
27
|
+
# Below will write to stdout as long as stdout isn't the terminal (so works in irb and ruby)
|
28
|
+
# zip -P 'fp5!IZbVxgx2hWh8m*UQyc@d5nCGCrbiqPx73hh&' - file_to_add
|
29
|
+
#
|
30
|
+
# Below is attempt to read file from stdin:
|
31
|
+
# out = `echo 'hi' | zip -P 'fp5!IZbVxgx2hWh8m*UQyc@d5nCGCrbiqPx73hh&' - -`
|
32
|
+
#
|
33
|
+
#
|
34
|
+
# Zip commandline is
|
35
|
+
# zip -P 'fp5!IZbVxgx2hWh8m*UQyc@d5nCGCrbiqPx73hh&' zip_to_make.zip file_to_add
|
36
|
+
# maybe password has invalid chars
|
37
|
+
def encode_string_as_password_protected_old_zip_based(encryptable_portion, pass = nil)
|
38
|
+
pish = @password
|
39
|
+
pish = pass unless pass.nil?
|
40
|
+
|
41
|
+
supress_errors = "2>/dev/null"
|
42
|
+
# this will read that file... let's try passing it in from stdin pipes though
|
43
|
+
# alt = "zip -P '#{pish}' - #{@@clear_text_path}"
|
44
|
+
|
45
|
+
# TODu: escape single quotes or this command breaks...
|
46
|
+
raise "tried to encrypt an encryptable_portion which contained illegal character ' which would break the command being piped to zip" if encryptable_portion =~ /\'/
|
47
|
+
# passing in clear_text through pipes
|
48
|
+
alt = "echo '#{encryptable_portion}'| zip -P '#{pish}' - - #{supress_errors}"
|
49
|
+
#alt = "echo '#{encryptable_portion.gsub("\\n", "")}'| zip -P '#{pish}' - -"
|
50
|
+
|
51
|
+
binary_output = `#{alt}`
|
52
|
+
end
|
53
|
+
|
54
|
+
def encode_string_as_password_protected(encryptable_portion, pass = nil)
|
55
|
+
#require 'b_f'
|
56
|
+
|
57
|
+
pish = @password
|
58
|
+
pish = pass unless pass.nil?
|
59
|
+
raise "error, password given was too long, must be 56 or less chars" if pish.length > 56
|
60
|
+
|
61
|
+
bf = BF.new(pish, true)
|
62
|
+
|
63
|
+
encrypted = bf.encrypt(encryptable_portion)
|
64
|
+
end
|
65
|
+
|
66
|
+
|
67
|
+
def decode_password_protected_string(password, decryptable_portion)
|
68
|
+
::NmDatafile.decode_password_protected_string(password, decryptable_portion)
|
69
|
+
end
|
70
|
+
|
71
|
+
def obfuscate_file_format
|
72
|
+
# TODu: implement me
|
73
|
+
end
|
74
|
+
|
75
|
+
def deobfuscate_file_format
|
76
|
+
|
77
|
+
end
|
78
|
+
|
79
|
+
|
80
|
+
def clean_encrypt_string(string)
|
81
|
+
# Base64.encode64(encode_string_as_password_protected(string, @@unsecure_pass))
|
82
|
+
::NmDatafile.fast_encrypt_string_with_pass(::NmDatafile::FRONT_DOOR_KEY, string)
|
83
|
+
end
|
84
|
+
|
85
|
+
# This redirects to the mixin used by Crypto
|
86
|
+
# ughhh.....
|
87
|
+
def clean_decrypt_string(string)
|
88
|
+
::NmDatafile.clean_decrypt_string(string)
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
end
|
@@ -1,3 +1,6 @@
|
|
1
|
+
require 'nm_datafile/debug'
|
2
|
+
require 'nm_datafile/file_encoding'
|
3
|
+
|
1
4
|
module NmDatafile
|
2
5
|
|
3
6
|
# This class describes a way for AP to convert data into a portable format
|
@@ -89,14 +92,12 @@ module NmDatafile
|
|
89
92
|
# aNonyMousDatafile
|
90
93
|
class NmDatafile
|
91
94
|
@@schemas = ::NmDatafile::SCHEMA[:schemas] # TODO: move to initialize
|
92
|
-
@@unsecure_pass = $FrontDoorKey # ppl with root access can decrypt nmd files
|
93
95
|
@@clear_text_path = "clear_text_protected_nmd" # used for using system calls to decrypt and encrypt using a zip password
|
94
96
|
attr_reader :file_type, :password
|
95
97
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
98
|
+
# include Crypto
|
99
|
+
include Debug
|
100
|
+
include FileEncoding
|
100
101
|
|
101
102
|
###############################
|
102
103
|
# Loading and Dumping Methods #
|
@@ -175,10 +176,7 @@ module NmDatafile
|
|
175
176
|
@password = password
|
176
177
|
end
|
177
178
|
|
178
|
-
|
179
|
-
def version
|
180
|
-
"0.0.0"
|
181
|
-
end
|
179
|
+
|
182
180
|
|
183
181
|
def integrity_hash
|
184
182
|
encryptable_portion = build_encryptable_portion
|
@@ -195,7 +193,7 @@ module NmDatafile
|
|
195
193
|
|
196
194
|
def build_attributes
|
197
195
|
hash = { file_type: @file_type
|
198
|
-
# build_date: Time.zone.now, #
|
196
|
+
# build_date: Time.zone.now, # TODO: change me to the date the data was last modified...
|
199
197
|
}.to_json
|
200
198
|
end
|
201
199
|
|
@@ -281,112 +279,6 @@ module NmDatafile
|
|
281
279
|
defined?(Rails) ? "#{Rails.root}/tmp" : "/tmp"
|
282
280
|
end
|
283
281
|
|
284
|
-
###############
|
285
|
-
# zip methods #
|
286
|
-
###############
|
287
|
-
require 'zip'
|
288
|
-
|
289
|
-
# hash consists of {file1: string_data}
|
290
|
-
# output is a binary string representing a zip file of the entries specified
|
291
|
-
def encode_datafiles_as_zip(hash_of_entries)
|
292
|
-
temp_file = Tempfile.new(file_type.to_s, get_temp_directory)
|
293
|
-
FileUtils.rm temp_file.path
|
294
|
-
|
295
|
-
stream = ::Zip::OutputStream.write_buffer do |zos|
|
296
|
-
hash_of_entries.each do |entry_name, data|
|
297
|
-
zos.put_next_entry entry_name
|
298
|
-
zos.write data
|
299
|
-
end
|
300
|
-
end
|
301
|
-
|
302
|
-
stream.rewind
|
303
|
-
stream.read
|
304
|
-
end
|
305
|
-
|
306
|
-
# This method peers through a zip binary data blob and returns a hash consisting of
|
307
|
-
# { file_name1: file_data1, etc }
|
308
|
-
def self.extract_entities_from_binary_data(binary_data)
|
309
|
-
binary_data_io = StringIO.new(binary_data)
|
310
|
-
|
311
|
-
hash = {}
|
312
|
-
::Zip::InputStream.open(binary_data_io) do |io|
|
313
|
-
while (entry = io.get_next_entry)
|
314
|
-
hash[entry.name.to_sym] = io.read
|
315
|
-
end
|
316
|
-
end
|
317
|
-
|
318
|
-
password = self.determine_password(hash)
|
319
|
-
|
320
|
-
decrypt_encryptable_data!(password, hash)
|
321
|
-
hash
|
322
|
-
end
|
323
|
-
|
324
|
-
# Play:
|
325
|
-
# Below will write to stdout as long as stdout isn't the terminal (so works in irb and ruby)
|
326
|
-
# zip -P 'fp5!IZbVxgx2hWh8m*UQyc@d5nCGCrbiqPx73hh&' - file_to_add
|
327
|
-
#
|
328
|
-
# Below is attempt to read file from stdin:
|
329
|
-
# out = `echo 'hi' | zip -P 'fp5!IZbVxgx2hWh8m*UQyc@d5nCGCrbiqPx73hh&' - -`
|
330
|
-
#
|
331
|
-
#
|
332
|
-
# Zip commandline is
|
333
|
-
# zip -P 'fp5!IZbVxgx2hWh8m*UQyc@d5nCGCrbiqPx73hh&' zip_to_make.zip file_to_add
|
334
|
-
# maybe password has invalid chars
|
335
|
-
def encode_string_as_password_protected_old_zip_based(encryptable_portion, pass = nil)
|
336
|
-
pish = @password
|
337
|
-
pish = pass unless pass.nil?
|
338
|
-
|
339
|
-
supress_errors = "2>/dev/null"
|
340
|
-
# this will read that file... let's try passing it in from stdin pipes though
|
341
|
-
# alt = "zip -P '#{pish}' - #{@@clear_text_path}"
|
342
|
-
|
343
|
-
# TODu: escape single quotes or this command breaks...
|
344
|
-
raise "tried to encrypt an encryptable_portion which contained illegal character ' which would break the command being piped to zip" if encryptable_portion =~ /\'/
|
345
|
-
# passing in clear_text through pipes
|
346
|
-
alt = "echo '#{encryptable_portion}'| zip -P '#{pish}' - - #{supress_errors}"
|
347
|
-
#alt = "echo '#{encryptable_portion.gsub("\\n", "")}'| zip -P '#{pish}' - -"
|
348
|
-
|
349
|
-
binary_output = `#{alt}`
|
350
|
-
end
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
def encode_string_as_password_protected(encryptable_portion, pass = nil)
|
356
|
-
#require 'b_f'
|
357
|
-
|
358
|
-
pish = @password
|
359
|
-
pish = pass unless pass.nil?
|
360
|
-
raise "error, password given was too long, must be 56 or less chars" if pish.length > 56
|
361
|
-
|
362
|
-
bf = BF.new(pish, true)
|
363
|
-
|
364
|
-
encrypted = bf.encrypt(encryptable_portion)
|
365
|
-
end
|
366
|
-
|
367
|
-
|
368
|
-
def decode_password_protected_string(password, decryptable_portion)
|
369
|
-
NmDatafile.decode_password_protected_string(password, decryptable_portion)
|
370
|
-
end
|
371
|
-
|
372
|
-
def obfuscate_file_format
|
373
|
-
# TODu: implement me
|
374
|
-
end
|
375
|
-
|
376
|
-
def deobfuscate_file_format
|
377
|
-
|
378
|
-
end
|
379
|
-
|
380
|
-
|
381
|
-
def clean_encrypt_string(string)
|
382
|
-
# Base64.encode64(encode_string_as_password_protected(string, @@unsecure_pass))
|
383
|
-
NmDatafile.fast_encrypt_string_with_pass(@@unsecure_pass, string)
|
384
|
-
end
|
385
|
-
|
386
|
-
def clean_decrypt_string(string)
|
387
|
-
NmDatafile.clean_decrypt_string(string)
|
388
|
-
end
|
389
|
-
|
390
282
|
|
391
283
|
|
392
284
|
|
@@ -394,171 +286,6 @@ module NmDatafile
|
|
394
286
|
|
395
287
|
# `gpg -c --no-use-agent`
|
396
288
|
|
397
|
-
###################
|
398
|
-
# Testing methods #
|
399
|
-
###################
|
400
|
-
|
401
|
-
# do as address_completion_file to create an rfsb that should have already existed, but didn't because a fresh db was used
|
402
|
-
# for testing
|
403
|
-
def simulate_rfsb_existance_on_webserver
|
404
|
-
rfsb = self.ready_for_shipment_batch
|
405
|
-
ReadyForShipmentBatch.create(batch_stamp: rfsb["batch_stamp"], sf_integrity_hash: rfsb["integrity_hash"])
|
406
|
-
end
|
407
|
-
|
408
|
-
# creates an address_completion file based on the contents of the current shippable_file object
|
409
|
-
def simulate_address_completion_response(n_shipped)
|
410
|
-
raise "can't make an address_completion_response unless it's a shippable_file" if @file_type != :shippable_file
|
411
|
-
setup_object_for_schema # TODu: I put this in, there was a bug in the test where it needs to be run... does it not run on init somehow?
|
412
|
-
|
413
|
-
shipped_sales = []
|
414
|
-
erroneous_addresses = []
|
415
|
-
sales.each.with_index do |sale, i|
|
416
|
-
if i < n_shipped
|
417
|
-
shipped_sales << { "id" => sale["id"] }
|
418
|
-
else
|
419
|
-
erroneous_addresses << { "id" => sale["id"] }
|
420
|
-
end
|
421
|
-
end
|
422
|
-
|
423
|
-
simulated_response = [shipped_sales, erroneous_addresses, ready_for_shipment_batch]
|
424
|
-
|
425
|
-
nmd_address_completion = NmDatafile.new(:address_completion_file, *simulated_response)
|
426
|
-
end
|
427
|
-
|
428
|
-
def generate_upload_params(action = "upload_shippable_file")
|
429
|
-
temp_zip = Tempfile.new('temp_zip', "#{Rails.root}/tmp")
|
430
|
-
save_to_file(temp_zip.path)
|
431
|
-
|
432
|
-
upload_shippable_params = {
|
433
|
-
"file_upload" => { "my_file" => Rack::Test::UploadedFile.new(temp_zip.path, "application/zip") },
|
434
|
-
"controller"=>"admin_pages",
|
435
|
-
"action"=>action
|
436
|
-
}
|
437
|
-
end
|
438
|
-
|
439
|
-
# Don't use push on #sales, it will use the push on the array. TODu: to fix this, make the @sales arrays collections
|
440
|
-
# and give those collections special properties when push is invoked
|
441
|
-
def add_sale(sale)
|
442
|
-
# add to collection the sale
|
443
|
-
self.sales << sale
|
444
|
-
regenerate_rfsb if should_gen_a_new_rfsb?(sale)
|
445
|
-
end
|
446
|
-
|
447
|
-
# we should NOT make a new rfsb if we're adding an old erroneous sale to the object
|
448
|
-
# to check for age... see if it already has an rfsb_id
|
449
|
-
def should_gen_a_new_rfsb?(sale)
|
450
|
-
return true if sale.ready_for_shipment_batch_id.nil?
|
451
|
-
false
|
452
|
-
end
|
453
|
-
|
454
|
-
def regenerate_rfsb
|
455
|
-
rfsb = ReadyForShipmentBatch.gen
|
456
|
-
self.ready_for_shipment_batch = rfsb
|
457
|
-
rfsb.delete
|
458
|
-
end
|
459
|
-
|
460
|
-
# create's some sales, line_items and
|
461
|
-
def create_sales_for_shippable_file(n, e=nil)
|
462
|
-
|
463
|
-
if @file_type == :shippable_file
|
464
|
-
s, l, a, rfsb = create_sales_and_return_data(n)
|
465
|
-
self.sales += s
|
466
|
-
self.line_items += l
|
467
|
-
self.addresses += a
|
468
|
-
self.ready_for_shipment_batch = rfsb
|
469
|
-
rfsb.delete
|
470
|
-
Sale.deep_delete_sales(s)
|
471
|
-
elsif @file_type == :address_completion_file
|
472
|
-
s, e, rfsb = create_sales_and_return_data_address_completion_file(n, e)
|
473
|
-
self.sales += s
|
474
|
-
self.erroneous_sales += e
|
475
|
-
|
476
|
-
self.ready_for_shipment_batch = rfsb
|
477
|
-
rfsb.delete
|
478
|
-
Sale.deep_delete_sales(s)
|
479
|
-
end
|
480
|
-
|
481
|
-
end
|
482
|
-
alias create_sales create_sales_for_shippable_file
|
483
|
-
|
484
|
-
def create_sales_and_return_data_address_completion_file(n, e)
|
485
|
-
e = 0 if e.nil?
|
486
|
-
sales = []
|
487
|
-
errors = []
|
488
|
-
n.times { sales << FactoryGirl.create(:sale_with_1_book) }
|
489
|
-
e.times { errors << FactoryGirl.create(:sale_with_1_book) }
|
490
|
-
rfsb = ReadyForShipmentBatch.gen
|
491
|
-
|
492
|
-
sales.each {|s| s.ready_for_shipment_batch_id = rfsb.id}
|
493
|
-
errors.each {|s| s.ready_for_shipment_batch_id = rfsb.id}
|
494
|
-
|
495
|
-
[ sales, errors, rfsb ]
|
496
|
-
end
|
497
|
-
|
498
|
-
def create_sales_and_return_data(n)
|
499
|
-
sales = []
|
500
|
-
n.times { sales << FactoryGirl.create(:sale_with_1_book) }
|
501
|
-
rfsb = ReadyForShipmentBatch.gen
|
502
|
-
sales.each {|s| s.ready_for_shipment_batch_id = rfsb.id}
|
503
|
-
l = capture_line_items(sales)
|
504
|
-
a = capture_addresses(sales)
|
505
|
-
|
506
|
-
[sales, l, a, rfsb]
|
507
|
-
end
|
508
|
-
|
509
|
-
def capture_line_items(sales)
|
510
|
-
l = []
|
511
|
-
sales.each do |s|
|
512
|
-
l += s.line_items
|
513
|
-
end
|
514
|
-
l
|
515
|
-
end
|
516
|
-
|
517
|
-
def capture_addresses(sales)
|
518
|
-
a = []
|
519
|
-
sales.each do |s|
|
520
|
-
a << s.address
|
521
|
-
end
|
522
|
-
a
|
523
|
-
end
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
#################
|
528
|
-
# Debug methods #
|
529
|
-
#################
|
530
|
-
# render a count of sales
|
531
|
-
def to_s
|
532
|
-
string = "NmDatafile: \n"
|
533
|
-
data_collection_names.each.with_index do |collection_name, i|
|
534
|
-
string << " #{collection_name}: #{@data_collections[i].count} \n"
|
535
|
-
end
|
536
|
-
|
537
|
-
data_object_names.each.with_index do |variable_name, i|
|
538
|
-
string << " #{variable_name}: #{1} \n"
|
539
|
-
end
|
540
|
-
|
541
|
-
string
|
542
|
-
end
|
543
|
-
|
544
|
-
def inspect
|
545
|
-
puts self.to_s
|
546
|
-
end
|
547
|
-
|
548
|
-
def ==(other)
|
549
|
-
return false if other.class != self.class
|
550
|
-
return true if all_data_matches?(other)
|
551
|
-
false
|
552
|
-
end
|
553
|
-
|
554
|
-
def all_data_matches?(other)
|
555
|
-
if self.integrity_hash == other.integrity_hash
|
556
|
-
if self.build_attributes == other.build_attributes
|
557
|
-
return true
|
558
|
-
end
|
559
|
-
end
|
560
|
-
false
|
561
|
-
end
|
562
289
|
|
563
290
|
#####################################################
|
564
291
|
# batch checking, high conasence with Importable #
|
data/lib/nm_datafile/version.rb
CHANGED
data/lib/nm_datafile.rb
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
require 'json'
|
2
|
+
require 'yaml'
|
2
3
|
require 'zip'
|
3
4
|
|
5
|
+
# require 'factory_girl'
|
6
|
+
# require 'pry'; binding.pry
|
7
|
+
# require File.expand_path('../../spec/factories/sales.rb', __FILE__)
|
8
|
+
|
4
9
|
require 'nm_datafile/version'
|
5
10
|
require 'nm_datafile/schema'
|
6
11
|
require 'nm_datafile/b_f'
|
@@ -11,18 +16,21 @@ require 'nm_datafile/nm_datafile'
|
|
11
16
|
|
12
17
|
|
13
18
|
module NmDatafile
|
14
|
-
#
|
19
|
+
FRONT_DOOR_KEY = "$FrontDoorKey" # Write to NmDatafile::FRONT_DOOR_KEY to set a symetric key
|
20
|
+
@@symmetric_key = "$FrontDoorKey"
|
15
21
|
|
16
|
-
|
22
|
+
extend DataLoading
|
23
|
+
extend Crypto
|
17
24
|
|
18
25
|
def self.new(file_type, *args)
|
19
|
-
NmDatafile.new(file_type, args)
|
26
|
+
NmDatafile.new(file_type, *args)
|
20
27
|
end
|
21
|
-
|
22
|
-
def self.
|
23
|
-
|
28
|
+
|
29
|
+
def self.set_symmetric_key(val)
|
30
|
+
|
24
31
|
end
|
25
32
|
|
33
|
+
|
26
34
|
end
|
27
35
|
|
28
36
|
|
Binary file
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# Read about factories at https://github.com/thoughtbot/factory_girl
|
2
|
+
|
3
|
+
FactoryGirl.define do
|
4
|
+
factory :sale do
|
5
|
+
currency_used "BTC"
|
6
|
+
|
7
|
+
# Set these in special cases...
|
8
|
+
sequence(:receipt_confirmed) {|n| Time.zone.now }
|
9
|
+
# downloaded "2013-12-12"
|
10
|
+
# shipped "2013-12-12"
|
11
|
+
|
12
|
+
|
13
|
+
# These values are actually calculated values... optimize later
|
14
|
+
# sale_amount "9.99"
|
15
|
+
# tax_amount "9.99"
|
16
|
+
shipping_amount "0.00"
|
17
|
+
|
18
|
+
after(:build) do |sale|
|
19
|
+
sale.user = FactoryGirl.create(:user_with_1_address)
|
20
|
+
sale.address = sale.user.addresses.first
|
21
|
+
sale.checkout_wallet = FactoryGirl.create(:checkout_wallet)
|
22
|
+
ubw = UtilizedBitcoinWallet.find_by_wallet_address(ENV['SHOPPING_CART_WALLET'])
|
23
|
+
ubw = FactoryGirl.create(:utilized_bitcoin_wallet, wallet_address: ENV['SHOPPING_CART_WALLET']) if ubw.nil?
|
24
|
+
sale.utilized_bitcoin_wallet = ubw
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
28
|
+
|
29
|
+
factory :sale_with_1_book, :parent => :sale do
|
30
|
+
line_items { |items| [ items.association(:line_item_1_book)
|
31
|
+
]}
|
32
|
+
|
33
|
+
after(:build) do |sale|
|
34
|
+
@btc_payment_params = { "secret_authorization_token" => "babe" }
|
35
|
+
bp = FactoryGirl.create(:bitcoin_payment)
|
36
|
+
bp.value = WorkHard.convert_back_to_satoshi(sale.total_amount)
|
37
|
+
sale.bitcoin_payments << bp
|
38
|
+
bp.save
|
39
|
+
|
40
|
+
sale.calculate_shipping_amount
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
factory :sale_with_2_books, :parent => :sale do
|
45
|
+
line_items { |items| [ items.association(:line_item_2_books),
|
46
|
+
#items.association(:line_item_discount)
|
47
|
+
]}
|
48
|
+
|
49
|
+
after(:build) do |sale|
|
50
|
+
sale.checkout_wallet = FactoryGirl.create(:checkout_wallet)
|
51
|
+
sale.calculate_shipping_amount
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
factory :sale_with_5_books, :parent => :sale do
|
56
|
+
line_items { |items| [ items.association(:line_item_5_books),
|
57
|
+
#items.association(:line_item_discount)
|
58
|
+
]}
|
59
|
+
|
60
|
+
after(:build) do |sale|
|
61
|
+
sale.checkout_wallet = FactoryGirl.create(:checkout_wallet)
|
62
|
+
sale.calculate_shipping_amount
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
factory :sale_with_10_books, :parent => :sale do
|
67
|
+
line_items { |items| [ items.association(:line_item_10_books),
|
68
|
+
#items.association(:line_item_discount)
|
69
|
+
]}
|
70
|
+
|
71
|
+
after(:build) do |sale|
|
72
|
+
sale.checkout_wallet = FactoryGirl.create(:checkout_wallet)
|
73
|
+
sale.calculate_shipping_amount
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
factory :sale_with_1_book_unpaid, :parent => :sale do
|
78
|
+
receipt_confirmed nil
|
79
|
+
|
80
|
+
line_items { |items| [
|
81
|
+
items.association(:line_item_1_book)
|
82
|
+
]}
|
83
|
+
|
84
|
+
after(:build) do |sale|
|
85
|
+
sale.checkout_wallet = FactoryGirl.create(:checkout_wallet)
|
86
|
+
sale.calculate_shipping_amount
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
|
94
|
+
|
data/spec/nm_datafile_spec.rb
CHANGED
@@ -4,35 +4,53 @@ describe "nm_datafile" do
|
|
4
4
|
|
5
5
|
before :each do
|
6
6
|
#@sample_data = get_sample_data
|
7
|
-
@
|
8
|
-
@sample_data = [ @
|
7
|
+
@sales = [{"address_id"=>1, "created_at"=>"2015-03-08T03:54:51Z", "currency_used"=>"BTC"}]
|
8
|
+
@sample_data = [ @sales ]
|
9
|
+
@binary_nmd_path = 'spec/data/nmd_binary_string_w_2s_2li_2a_1ubws_1ep.zip'
|
9
10
|
end
|
10
11
|
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
it "should be able to load data from a binary string" do
|
13
|
+
nmd_binary_string = File.read(@binary_nmd_path)
|
14
|
+
|
15
|
+
nmd_file = NmDatafile::LoadBinaryData nmd_binary_string
|
16
|
+
|
17
|
+
sales = nmd_file.sales
|
18
|
+
line_items = nmd_file.line_items
|
19
|
+
discounts = nmd_file.discounts
|
20
|
+
addresses = nmd_file.addresses
|
21
|
+
ubws = nmd_file.ubws
|
22
|
+
encryption_pairs = nmd_file.encryption_pairs
|
23
|
+
rfsb = nmd_file.ready_for_shipment_batch
|
24
|
+
|
25
|
+
nm_data = NmDatafile.new(:shippable_file, sales, line_items, discounts, addresses, ubws, encryption_pairs, rfsb)
|
26
|
+
|
27
|
+
nm_data.sales.count.should eq 2
|
28
|
+
nm_data.ready_for_shipment_batch["batch_stamp"].should eq rfsb["batch_stamp"]
|
14
29
|
end
|
15
30
|
|
16
|
-
|
17
|
-
it "should be instantiable with data" do
|
18
|
-
#return_array = [ @sales,
|
19
|
-
# @line_items,
|
20
|
-
# @discounts,
|
21
|
-
# @addresses,
|
22
|
-
# @utilized_bitcoin_wallets,
|
23
|
-
# @encryption_pairs,
|
24
|
-
# rfsb ]
|
31
|
+
it "should be able to load data from a path to a zip" do
|
25
32
|
|
26
|
-
|
33
|
+
nmd_string_loaded = NmDatafile::LoadBinaryData File.read(@binary_nmd_path)
|
27
34
|
|
28
|
-
|
29
|
-
|
30
|
-
|
35
|
+
nmd_path_loaded = NmDatafile::Load @binary_nmd_path
|
36
|
+
|
37
|
+
nmd_string_loaded.sales.should eq nmd_path_loaded.sales
|
31
38
|
end
|
32
39
|
|
33
|
-
it "should
|
34
|
-
|
40
|
+
it "should be able to save data as a zip string" do
|
41
|
+
nmd_shippable = NmDatafile.new(:shippable_file, *@sample_data)
|
42
|
+
|
43
|
+
nmd = NmDatafile::LoadBinaryData nmd_shippable.save_to_string
|
44
|
+
|
45
|
+
nmd.sales.should eq @sample_data.first
|
35
46
|
end
|
47
|
+
|
48
|
+
it "should be able to load in the attributes" do
|
49
|
+
nmd = NmDatafile::Load @binary_nmd_path
|
50
|
+
|
51
|
+
nmd.file_type.should eq :shippable_file
|
52
|
+
end
|
53
|
+
|
36
54
|
|
37
55
|
end
|
38
56
|
|
data/spec/spec_helper.rb
CHANGED
@@ -24,6 +24,11 @@ RSpec.configure do |config|
|
|
24
24
|
# `true` in RSpec 4.
|
25
25
|
mocks.verify_partial_doubles = true
|
26
26
|
end
|
27
|
+
|
28
|
+
config.expect_with :rspec do |c|
|
29
|
+
# Enable expect and should
|
30
|
+
c.syntax = [:should, :expect]
|
31
|
+
end
|
27
32
|
|
28
33
|
# The settings below are suggested to provide a good initial experience
|
29
34
|
# with RSpec, but feel free to customize to your heart's content.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: nm_datafile
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- dsj
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-03-
|
11
|
+
date: 2015-03-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rubyzip
|
@@ -100,10 +100,14 @@ files:
|
|
100
100
|
- lib/nm_datafile/b_f.rb
|
101
101
|
- lib/nm_datafile/crypto.rb
|
102
102
|
- lib/nm_datafile/data_loading.rb
|
103
|
+
- lib/nm_datafile/debug.rb
|
104
|
+
- lib/nm_datafile/file_encoding.rb
|
103
105
|
- lib/nm_datafile/nm_datafile.rb
|
104
106
|
- lib/nm_datafile/schema.rb
|
105
107
|
- lib/nm_datafile/version.rb
|
106
108
|
- nm_datafile.gemspec
|
109
|
+
- spec/data/nmd_binary_string_w_2s_2li_2a_1ubws_1ep.zip
|
110
|
+
- spec/factories/sales.rb
|
107
111
|
- spec/nm_datafile_spec.rb
|
108
112
|
- spec/spec_helper.rb
|
109
113
|
homepage: https://github.com/AnonymousPublications/nm_datafile
|
@@ -131,5 +135,7 @@ signing_key:
|
|
131
135
|
specification_version: 4
|
132
136
|
summary: A gem that saves files into a secure encrypted file.
|
133
137
|
test_files:
|
138
|
+
- spec/data/nmd_binary_string_w_2s_2li_2a_1ubws_1ep.zip
|
139
|
+
- spec/factories/sales.rb
|
134
140
|
- spec/nm_datafile_spec.rb
|
135
141
|
- spec/spec_helper.rb
|