nm_datafile 0.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d0e2702e30b76edee854437b9198f80aa8a0e17f
+  data.tar.gz: 2d6c9faeded184a7f7b460541c386be6f6a9a03b
+SHA512:
+  metadata.gz: 599da7c695dca546ddf93b02e2020454acd3aa4a90119dc8cf9ac0e16424797c757a867736002288f9862b5496e5259a44ea811d735a2966b66184b0ecbb235a
+  data.tar.gz: bead10acc0a21dc9332ef46d87c9d0269efee22175d8728610a78cbd93a0c82d4c2867de358bbfa91b234be21c5dd91ded066c48478f7768238e4a8d9db61790

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in nm_datafile.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 antisec
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+# NmDatafile
+
+NmDatafile is a library that defines a file format that makes adding files and strings to an easy, selfencrypting file that can use either asymetric or symetric cryptography.  
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'nm_datafile'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install nm_datafile
+
+
+## Code Example
+
+    nmd = NmDatafile.new(:shippable_file)
+
+    nmd.sales = [1,2,3,4]
+    nmd.ready_for_shipment_batch = 1
+
+    nmd.save(path_to_file)
+
+    nmd_loaded = NmDatafile::Load(path_to_file)
+
+    nmd_loaded.sales #=> [1,2,3,4]
+
+
+## Usage
+
+First define a file Schema.  
+For instance, if you wanted a file type called 'shippable_file' and you wanted it to have many records of 'sales', 'line_items', etc, 
+and you wanted your file to have a string of data named 'ready_for_shipment_batch', then you would define it like so. 
+
+    $nm_datafile_schemas = { 
+      schemas: {
+        :shippable_file => {
+          data_collections: [:sales, :line_items, :discounts, :addresses, :ubws, :encryption_pairs], # name the data that is input into the NMDatafile as an array
+          data_objects: [:ready_for_shipment_batch]
+        }
+      }
+    }
+
+TODO:  Specify encryption type on file as well...
+
+Then you'd want to create the NMD file:
+
+And finally save it as shown below:
+
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/nm_datafile/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+
+

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+
+require 'rspec/core/rake_task'
+
+task :default do
+  RSpec::Core::RakeTask.new(:spec)
+  Rake::Task["spec"].execute
+end
+

data/changelog.md

@@ -0,0 +1,16 @@
+### 0.0.1
+
+* Code migrated into this repo
+* Created some tests
+* Get pull out the class methods from the class
+** 1. Pull all those methods into a module
+** 2. Include them on the class where needed.
+** 3. Extend them onto the parent namespace module
+** 4. Plug the gem into rails and see if it still works.
+
+
+### 0.0.0
+
+Skeleton
+
+

data/lib/nm_datafile/b_f.rb

@@ -0,0 +1,33 @@
+# https://gist.github.com/nono/2995118
+
+require "openssl"
+
+module NmDatafile
+
+  class BF < Struct.new(:key, :pad_with_spaces)
+    def encrypt(str)
+      cipher = OpenSSL::Cipher.new('bf-ecb').encrypt
+      if pad_with_spaces
+        str += " " until str.bytesize % 8 == 0
+        cipher.padding = 0
+      end
+      cipher.key = key
+      binary_data = cipher.update(str) << cipher.final
+      hex_encoded = binary_data.unpack('H*').first
+    end
+     
+    def decrypt(hex_encoded)
+      cipher = OpenSSL::Cipher.new('bf-ecb').decrypt
+      cipher.padding = 0 if pad_with_spaces
+      cipher.key = key
+      binary_data = [hex_encoded].pack('H*')
+      str = cipher.update(binary_data) << cipher.final
+      str.force_encoding(Encoding::UTF_8)
+      str
+    end
+  end
+
+end
+
+
+

data/lib/nm_datafile/crypto.rb

@@ -0,0 +1,146 @@
+require 'base64'
+
+module NmDatafile
+
+  module Crypto
+    # The zip implementation... should use 7z though and throw this out when it's done... or truecript or gpg...
+    def decode_protected_zip_old_zip_based(password, decryptable_portion)
+      encryptable_portion = Tempfile.new('encryptable_portion', "#{Rails.root}/tmp")
+      FileUtils.rm(encryptable_portion.path)
+      File.binwrite(encryptable_portion.path, decryptable_portion)
+      
+      supress_errors = "2>/dev/null"
+      decompress_zip_cmd = "funzip -'#{password}' '#{encryptable_portion.path}' #{supress_errors}"
+
+      clear_text_hash = `#{decompress_zip_cmd}`.chomp
+      
+      clear_text_hash = convert_newline_chars_back_to_symbols(clear_text_hash)
+      
+      FileUtils.rm(encryptable_portion.path)
+      clear_text_hash
+    end
+    
+    # TODu:  rename to decode_string_as_password_protected
+    def decode_string_as_password_protected(password, decryptable_portion)
+      #require 'b_f'
+      bf = BF.new(password, true)
+      
+      bf.decrypt(decryptable_portion)
+    end
+    
+    
+    # Takes in a password and binary data of protected archive file and outputs
+    # a string of it's first entry in clear text which should be a hash
+    # of {:file_name, "file_data"}
+    def decode_password_protected_string(password, decryptable_portion)
+      decode = decode_string_as_password_protected(password, decryptable_portion)
+    end
+    
+    
+    
+    # converts the string pairs into symbol/ string pairs
+    def symbolize_keys(decode)
+      p = {}
+      decode.each do |key_pair|
+        p.merge!( { key_pair[0].to_sym => key_pair[1] } )
+      end
+      p
+    end
+    
+    
+    
+    
+    def convert_newline_chars_back_to_symbols(clear_text_hash)
+      clear_text_hash.gsub("\n", "\\n")
+    end
+    
+    # I think 7z is considered secure and zip is considered insecure so write this
+    def decode_protected_7z(password, decryptable_portion)
+      # TODu: implement
+    end
+    
+    
+    def decrypt_encryptable_data!(password, hash)
+      return if hash[:crypt].nil?  # leave this function if there's no 'crypt' entry to decrypt
+      
+      decode = decode_string_into_NMDatafile_stores(password, hash[:crypt])
+      
+      hash.delete :crypt
+      
+      hash.merge!(decode)
+    end
+    
+    def decode_string_into_NMDatafile_stores(password, crypt_string)
+      decode = YAML::load decode_password_protected_string(password, crypt_string)
+      decode = symbolize_keys(decode)
+    end
+    
+    
+    
+    def clean_decrypt_string(string)
+    #  string = Base64.decode64 string
+    #  decode_password_protected_string(@@unsecure_pass, string)
+      NMDatafile.fast_decrypt_string_with_pass(@@unsecure_pass, string)
+    end
+    
+    
+    
+    def fast_encrypt_string_with_pass(pass, string)
+      encoded_as_base64 = Base64.encode64(string)
+      rearranged = rearrangement(encoded_as_base64)
+      obfs = obfuscated_ending(rearranged)
+      Base64.encode64(obfs)
+    end
+    
+    def fast_decrypt_string_with_pass(pass, string)
+      obfs = Base64.decode64(string)
+      rearranged = obfuscated_ending_undo(obfs)
+      encoded_as_base64 = rearrangement_undo(rearranged)
+      Base64.decode64(encoded_as_base64)
+    end
+    
+    def rearrangement(s)
+      s = the_last_three_chars(s) + the_string_minus_the_last_three_chars(s)
+    end
+    
+    def rearrangement_undo(s)
+      s = the_string_minus_the_first_three_chars(s) + the_first_three_chars(s)
+    end
+    
+    
+    def obfuscated_ending(s)
+      junk = "tlD3=\n"
+      s + junk
+    end
+    
+    def obfuscated_ending_undo(s)
+      junk = "tlD3=\n"
+      s[0...(-1*junk.length)]
+    end
+    
+    # hide these somewhere, so ugly
+    def the_last_three_chars(s)
+      s[-3..-1]
+    end
+    
+    def the_first_three_chars(s)
+      s[0..3]
+    end
+    
+    def the_string_minus_the_last_three_chars(s)
+      s[0...-3]
+    end
+    
+    def the_string_minus_the_first_three_chars(s)
+      s[2..-1]
+    end
+    
+    
+    def encrypt_using_gpg(pass, string)
+      crypto = GPGME::Crypto.new :symmetric => true, :password => "gpgme"
+      encrypted_data = crypto.encrypt "string"
+      encrypted_data.read
+    end
+  end
+
+end

data/lib/nm_datafile/nm_datafile.rb

@@ -0,0 +1,643 @@
+module NmDatafile
+
+  # This class describes a way for AP to convert data into a portable format 
+  # (zip_file.nmd).  nms == aNonyMousDatafile 
+  #
+  #   To create a new file, you first must define a schema.  As of right now, 
+  # there are two public schemas.  As defined in these schemas, NmDatafiles 
+  # consist of collections of objects, and individual objects... eg
+  # An instance might have nmd.important_numbers which is an array of number, 
+  # whereas it might also have a data_object, nmd#stupidest_number which is a
+  # single number.  
+  #
+  #   So to create an NmDatafile, do
+  # `NmDatafile.new(:shippable_file, sales, line_items, addresses, rfsb)`
+  # that will work if you have a schema that has 3 data_collections specified
+  # and 1 data_object specified in the schema named :shippable_file.  
+  #
+  #   Using NmDatafiles allows you access to utility functions such as
+  #     - #save_to_string
+  #     - #save_to_file
+  #     - tons of testing methods
+  #
+  #   The NmDatafile is just a zip, but it's a password protected zip, and 
+  # it can also be obfuscated to confuse the feds.  
+  #
+  # Creating a new APDatafile means creates, in memory, a way to:
+  #  - Render the data to the hdd as a zip file
+  #  - Render the data as a string which can be streamed as a zip file
+  #  - Reading a file
+  #  - Encrypt the data (assymetrically?) YES
+  #  - Define the standard for formating 
+  #  - Create mock data outputs easily (helpful in testing)
+  #  - Specify the version of the file
+  #  - File can non-destructively corrupt itself so it doesn't look so much like a zip file to cryptographic analists
+  #  - Object is responsive to what ever schema it's set to... so it will be able to do
+  #       shippable_file = NmDatafile.new(:shippable_file);  shippable_file.sales << Sale.new
+  #  - loads data, nmd.load_data([sales, line_items, addresses, [ready_for_shipment_batch]])
+  #  - loads variables nmd.load_data([sales, line_items, addresses, ready_for_shipment_batch])
+
+  # Encrypt the data (asymetrically?) YES
+  #   To do this, all the files should be lumped together as one huge string (a hash of file names and data strings)... split by a special split marker and file name definer such as
+  #   Data Encryption:
+  #     Meth. 1)  Once this massive string is created, a PGP key is used to encrypt the whole file.  
+  #     Meth. 2)  The data can be encrypted using a randomly generated password, and then the password get's encrypted with PGP <- better
+  #
+  #   
+
+  # (r)  file_type:  Can be 
+  #      - shippable_file, used to export shipping data from AP book shopping app
+  #      - address_completion_file, used to transfer data from shipping machine to AP book shopping app
+  #      - db_backup_file, used to download a complete backup of a webserver
+  #      - etc, etc, used for other organization objectives
+  #
+  # (w)  set_public_key:  Specify a public PGP key to encode the data with
+  #
+  # (r)  get_public_key:  Shows the email and fingerprint of the key that's been set (for debugging)
+  # 
+  # (r)  show_current_schema:  Shows the data schema for the file type
+  # 
+  # (w)  set_current_schema:   deletes all data and sets the schema
+  #
+  # (r)  save_to_string:  creates the zip file, handleing
+  #
+  # (rw) protected: bool, specifies whether the file is secured with PGP or if it's just a zip file
+  #
+  # (m)  Load: loads a file into memory as an NmDatafile
+  #
+  # (m)  load_data:  loads array of data into memory as an NmDatafile object
+
+
+  # PRIVATE
+  # (m)  encrypt_file!:  Does all encryption procedures before rendering to a string or file
+  #
+  # (m)  decrypt_file!:  reverse of encrypt_file!
+  #
+  # (m)  encrypt_string:  symetrically encrypts a string using a password
+  #
+  # (m)  decrypt_string:  reverse of encrypt_string
+  # 
+  # (m)  encrypt_symetric_key:  uses PGP to encrypt the password that encrypted the data
+  #
+  # (m)  decrypt_symetric_key:  reverse of encrypt_symetric_key.
+  #
+  # (m)  corrupt_zip:  corrupts the zip file so it doesn't look like a zip file, make it look like a jpeg
+  #
+  # (m)  uncorrupt_zip:  reverses corrupt_zip file so the zip can be processed
+  # 
+  # (m)  
+  # aNonyMousDatafile
+  class NmDatafile
+    @@schemas = ::NmDatafile::SCHEMA[:schemas]  # TODO:  move to initialize
+    @@unsecure_pass = $FrontDoorKey  # ppl with root access can decrypt nmd files
+    @@clear_text_path = "clear_text_protected_nmd" # used for using system calls to decrypt and encrypt using a zip password
+    attr_reader :file_type, :password
+    
+    extend Crypto
+    
+    
+    
+    
+    ###############################
+    # Loading and Dumping Methods #
+    ###############################
+    
+    # (m)  Load: loads a file into memory as an NmDatafile
+    def self.Load(file_path)
+      zip_data = File.read(file_path)
+      LoadBinaryData(zip_data)
+    end
+    
+    def self.LoadBinaryData(binary_data)
+      hash = extract_entities_from_binary_data(binary_data)
+      
+      file_type = determine_file_type(hash[:attributes])
+      nmd = self.new( file_type )
+      
+      nmd.load_attributes(hash[:attributes]) unless hash[:attributes].nil?
+      nmd.load_encryption(hash[:encryption])
+      
+      nmd.load_data([*hash[:data_collections], *hash[:data_objects]])
+    end
+    
+    def self.determine_file_type(attributes_hash)
+      attributes_hash = YAML::load attributes_hash
+      attributes_hash["file_type"].to_sym
+    end
+    
+    def self.determine_password(hash)
+      d = YAML::load hash[:encryption]
+      clean_decrypt_string(d["password"])
+    end
+    
+    
+    
+    
+    
+    
+    def initialize(file_type, *args)
+      set_file_type(file_type)
+      
+      load_data(args)
+      
+      setup_object_for_schema
+    end
+    
+    def load_attributes(attribute_data)
+      d = YAML::load attribute_data
+      @file_type = d["file_type"].to_sym
+      @build_date = Time.zone.parse d["build_date"] unless d["build_date"].nil?
+    end
+    
+    def load_encryption(encryption_data)
+      d = YAML::load encryption_data
+      @integrity_hash = d["integrity_hash"] unless d["integrity_hash"].nil?
+      @password = clean_decrypt_string(d["password"]) unless d["password"].nil?
+    end
+    
+    # (m)  load_data:  loads array of data into memory as an NmDatafile object
+    # clears the data arrays so it's like reinitializing the whole file
+    def load_data(*args)
+      init_data_arrays  # wipes all preexisting data in @data_collections and @data_objects
+      
+      args[0].each.with_index do |array_or_variable, i|
+        if i < data_collection_names.count
+          @data_collections[i] += array_or_variable 
+        else
+         j = i - data_collection_names.count
+         @data_objects[j] = array_or_variable #  if array_or_variable.class != Array
+        end
+      end
+      self
+    end
+    
+    
+    def save_to_string
+      set_password
+      
+      clear_text = build_encryptable_portion
+      encrypted_data = encode_string_as_password_protected(clear_text)
+      
+      hash_of_entries = { :crypt => encrypted_data,
+                          :encryption => build_encryption,
+                          :attributes => build_attributes }
+      
+      encode_datafiles_as_zip(hash_of_entries)
+    end
+    
+    def save_to_file(path)
+      File.write(path, save_to_string)
+    end
+    
+    def set_password
+      len = 41
+      
+      password = ""
+      chars = ("a".."z").to_a + ("A".."Z").to_a + ("0".."9").to_a + ['!', '@', '#', '$', '%', '^', '&', '*', '(', ')']
+      
+      1.upto(len) { |i| password << chars[rand(chars.size-1)]}
+      
+      space = rand(len-1)
+      password[space] = ("0".."9").to_a[rand(9)]  # ensure there's number and letter
+      space += 1
+      space = 0 if space == len
+      password[space] = ("a".."z").to_a[rand(22)]
+      
+      @password = password
+    end
+    
+    # TODO del me
+    def version
+      "0.0.0"
+    end
+    
+    def integrity_hash
+      encryptable_portion = build_encryptable_portion
+      Digest::SHA2.hexdigest(encryptable_portion)
+    end
+    
+    def build_data_collections
+      @data_collections.to_json
+    end
+    
+    def build_data_objects
+      @data_objects.to_json
+    end
+    
+    def build_attributes
+      hash = { file_type: @file_type
+                # build_date: Time.zone.now,           # TODu:  change me to the date the data was last modified...
+              }.to_json
+    end
+    
+    def build_encryption
+      hash = { integrity_hash: integrity_hash,
+               password: clean_encrypt_string(@password)
+             }.to_json
+    end
+    
+    def build_encryptable_portion
+      e = { :data_collections => @data_collections,
+        :data_objects => @data_objects
+      }.to_json
+    end
+    
+    
+    
+    #######################
+    # schema related junk #
+    #######################
+    
+    def schema
+      @@schemas[@file_type]
+    end
+    
+    def data_collection_names
+      schema[:data_collections]
+    end
+    
+    def data_object_names
+      schema[:data_objects]
+    end
+    
+    def var_names
+      data_collection_names + data_object_names
+    end
+    
+    # specify the schema type
+    def set_file_type(file_type)
+      @file_type = file_type
+      init_data_arrays
+    end
+    
+    def init_data_arrays
+      @data_collections = []
+      data_collection_names.count.times { @data_collections << [] }
+      @data_objects = []
+      data_object_names.count.times { @data_objects << [] }
+    end
+    
+    # This makes it so you can call file.attribute to get direct access to an attribute
+    def setup_object_for_schema
+      data_collection_names.each.with_index do |data_collection_name, i|
+        # define getter
+        self.define_singleton_method(var_names[i]) do
+          @data_collections[i]
+        end
+        # define setter
+        self.define_singleton_method((data_collection_names[i].to_s + "=").to_sym) do |val|
+          @data_collections[i] = val
+        end
+      end
+      
+      data_object_names.each.with_index do |data_object_name, i|
+        # getters
+        self.define_singleton_method(data_object_name) do
+          @data_objects[i]
+        end
+        # setters
+        self.define_singleton_method((data_object_name.to_s + "=").to_sym) do |val|
+          @data_objects[i] = val
+        end
+      end
+      
+    end
+    
+    
+    
+    
+    # This method get's the temp directory.  If it's a rails
+    # app, that would be Rails.root/tmp, else just /tmp
+    def get_temp_directory
+      defined?(Rails) ? "#{Rails.root}/tmp" : "/tmp"
+    end
+    
+    ###############
+    # zip methods #
+    ###############
+    require 'zip'
+    
+    # hash consists of {file1: string_data}
+    # output is a binary string representing a zip file of the entries specified
+    def encode_datafiles_as_zip(hash_of_entries)
+      temp_file = Tempfile.new(file_type.to_s, get_temp_directory)
+      FileUtils.rm temp_file.path
+      
+      stream = ::Zip::OutputStream.write_buffer do |zos|
+        hash_of_entries.each do |entry_name, data|
+          zos.put_next_entry entry_name
+          zos.write data
+        end
+      end
+      
+      stream.rewind
+      stream.read
+    end
+    
+    # This method peers through a zip binary data blob and returns a hash consisting of
+    # { file_name1: file_data1, etc }
+    def self.extract_entities_from_binary_data(binary_data)
+      binary_data_io = StringIO.new(binary_data)
+      
+      hash = {}
+      ::Zip::InputStream.open(binary_data_io) do |io|
+        while (entry = io.get_next_entry)
+          hash[entry.name.to_sym] = io.read
+        end
+      end
+      
+      password = self.determine_password(hash)
+      
+      decrypt_encryptable_data!(password, hash)
+      hash
+    end
+    
+    # Play:
+    # Below will write to stdout as long as stdout isn't the terminal (so works in irb and ruby)
+    # zip -P 'fp5!IZbVxgx2hWh8m*UQyc@d5nCGCrbiqPx73hh&' - file_to_add
+    #
+    # Below is attempt to read file from stdin:
+    # out = `echo 'hi' | zip -P 'fp5!IZbVxgx2hWh8m*UQyc@d5nCGCrbiqPx73hh&' - -`
+    #
+    #
+    # Zip commandline is
+    # zip -P 'fp5!IZbVxgx2hWh8m*UQyc@d5nCGCrbiqPx73hh&' zip_to_make.zip file_to_add
+    # maybe password has invalid chars
+    def encode_string_as_password_protected_old_zip_based(encryptable_portion, pass = nil)
+      pish = @password 
+      pish = pass unless pass.nil?
+      
+      supress_errors = "2>/dev/null"
+      # this will read that file... let's try passing it in from stdin pipes though
+      # alt = "zip -P '#{pish}' - #{@@clear_text_path}"
+      
+      # TODu:  escape single quotes or this command breaks...  
+      raise "tried to encrypt an encryptable_portion which contained illegal character ' which would break the command being piped to zip" if encryptable_portion =~ /\'/ 
+      # passing in clear_text through pipes
+      alt = "echo '#{encryptable_portion}'| zip -P '#{pish}' - - #{supress_errors}"
+      #alt = "echo '#{encryptable_portion.gsub("\\n", "")}'| zip -P '#{pish}' - -"
+
+      binary_output = `#{alt}`
+    end
+    
+    
+    
+    
+    def encode_string_as_password_protected(encryptable_portion, pass = nil)
+      #require 'b_f'
+      
+      pish = @password
+      pish = pass unless pass.nil?
+      raise "error, password given was too long, must be 56 or less chars" if pish.length > 56
+      
+      bf = BF.new(pish, true)
+      
+      encrypted = bf.encrypt(encryptable_portion)
+    end
+    
+    
+    def decode_password_protected_string(password, decryptable_portion)
+      NmDatafile.decode_password_protected_string(password, decryptable_portion)
+    end
+    
+    def obfuscate_file_format
+      # TODu: implement me
+    end
+    
+    def deobfuscate_file_format
+      
+    end
+    
+    
+    def clean_encrypt_string(string)
+      # Base64.encode64(encode_string_as_password_protected(string, @@unsecure_pass))
+      NmDatafile.fast_encrypt_string_with_pass(@@unsecure_pass, string)
+    end
+    
+    def clean_decrypt_string(string)
+      NmDatafile.clean_decrypt_string(string)
+    end
+    
+    
+    
+    
+    
+    
+    # `gpg -c --no-use-agent`
+    
+    ###################
+    # Testing methods #
+    ###################
+    
+    # do as address_completion_file to create an rfsb that should have already existed, but didn't because a fresh db was used
+    # for testing
+    def simulate_rfsb_existance_on_webserver
+      rfsb = self.ready_for_shipment_batch
+      ReadyForShipmentBatch.create(batch_stamp: rfsb["batch_stamp"], sf_integrity_hash: rfsb["integrity_hash"])
+    end
+    
+    # creates an address_completion file based on the contents of the current shippable_file object
+    def simulate_address_completion_response(n_shipped)
+      raise "can't make an address_completion_response unless it's a shippable_file" if @file_type != :shippable_file
+      setup_object_for_schema   # TODu:  I put this in, there was a bug in the test where it needs to be run... does it not run on init somehow?
+      
+      shipped_sales = []
+      erroneous_addresses = []
+      sales.each.with_index do |sale, i|
+        if i < n_shipped
+          shipped_sales << { "id" => sale["id"] }
+        else
+          erroneous_addresses << { "id" => sale["id"] }
+        end
+      end
+      
+      simulated_response = [shipped_sales, erroneous_addresses, ready_for_shipment_batch]
+      
+      nmd_address_completion = NmDatafile.new(:address_completion_file, *simulated_response)
+    end
+    
+    def generate_upload_params(action = "upload_shippable_file")
+      temp_zip = Tempfile.new('temp_zip', "#{Rails.root}/tmp")
+      save_to_file(temp_zip.path)
+      
+      upload_shippable_params = { 
+      "file_upload" => { "my_file" => Rack::Test::UploadedFile.new(temp_zip.path, "application/zip") },
+      "controller"=>"admin_pages",
+      "action"=>action
+    }
+    end
+    
+    # Don't use push on #sales, it will use the push on the array.  TODu: to fix this, make the @sales arrays collections
+    # and give those collections special properties when push is invoked
+    def add_sale(sale)
+      # add to collection the sale
+      self.sales << sale
+      regenerate_rfsb if should_gen_a_new_rfsb?(sale)
+    end
+    
+    # we should NOT make a new rfsb if we're adding an old erroneous sale to the object
+    # to check for age... see if it already has an rfsb_id
+    def should_gen_a_new_rfsb?(sale)
+      return true if sale.ready_for_shipment_batch_id.nil?
+      false
+    end
+    
+    def regenerate_rfsb
+      rfsb = ReadyForShipmentBatch.gen
+      self.ready_for_shipment_batch = rfsb
+      rfsb.delete
+    end
+    
+    # create's some sales, line_items and 
+    def create_sales_for_shippable_file(n, e=nil)
+      
+      if @file_type == :shippable_file
+        s, l, a, rfsb = create_sales_and_return_data(n)
+        self.sales += s
+        self.line_items += l
+        self.addresses += a
+        self.ready_for_shipment_batch = rfsb
+        rfsb.delete
+        Sale.deep_delete_sales(s)
+      elsif @file_type == :address_completion_file
+        s, e, rfsb = create_sales_and_return_data_address_completion_file(n, e)
+        self.sales += s
+        self.erroneous_sales += e
+        
+        self.ready_for_shipment_batch = rfsb
+        rfsb.delete
+        Sale.deep_delete_sales(s)
+      end
+        
+    end
+    alias create_sales create_sales_for_shippable_file
+    
+    def create_sales_and_return_data_address_completion_file(n, e)
+      e = 0 if e.nil?
+      sales = []
+      errors = []
+      n.times { sales << FactoryGirl.create(:sale_with_1_book) }
+      e.times { errors << FactoryGirl.create(:sale_with_1_book) }
+      rfsb = ReadyForShipmentBatch.gen
+      
+      sales.each {|s| s.ready_for_shipment_batch_id = rfsb.id}
+      errors.each {|s| s.ready_for_shipment_batch_id = rfsb.id}
+      
+      [ sales, errors, rfsb ]
+    end
+    
+    def create_sales_and_return_data(n)
+      sales = []
+      n.times { sales << FactoryGirl.create(:sale_with_1_book) }
+      rfsb = ReadyForShipmentBatch.gen
+      sales.each {|s| s.ready_for_shipment_batch_id = rfsb.id}
+      l = capture_line_items(sales)
+      a = capture_addresses(sales)
+      
+      [sales, l, a, rfsb]
+    end
+    
+    def capture_line_items(sales)
+      l = []
+      sales.each do |s|
+        l += s.line_items
+      end
+      l
+    end
+    
+    def capture_addresses(sales)
+      a = []
+      sales.each do |s|
+        a << s.address
+      end
+      a
+    end
+    
+    
+    
+    #################
+    # Debug methods #
+    #################
+    # render a count of sales
+    def to_s
+      string = "NmDatafile:  \n"
+      data_collection_names.each.with_index do |collection_name, i|
+        string << "  #{collection_name}: #{@data_collections[i].count}  \n"
+      end
+      
+      data_object_names.each.with_index do |variable_name, i|
+        string << "  #{variable_name}: #{1}  \n"
+      end
+      
+      string
+    end
+    
+    def inspect
+     puts self.to_s
+    end
+    
+    def ==(other)
+      return false if other.class != self.class
+      return true if all_data_matches?(other)
+      false
+    end
+    
+    def all_data_matches?(other)
+      if self.integrity_hash == other.integrity_hash
+        if self.build_attributes == other.build_attributes
+          return true
+        end
+      end
+      false
+    end
+    
+    #####################################################
+    #  batch checking, high conasence with Importable   #
+    #####################################################
+    
+    # Move to... NmDatafile
+    def duplicate_batch?(previous_batch)
+      return false if previous_batch.nil?
+      
+      previous_batch.sf_integrity_hash == integrity_hash
+    end
+    
+    
+  end
+
+
+  # Zip Entry Structure
+  # data_collections:  array of data collections... atm, contains
+  # data_objects:      array of data_objects
+  # schema:            Schema of the data...  this should be encrypted if data1 is encrypted
+  # encryption:  integrity_hash: SHA2 hash of data for fingerprinting, password: password used to encrypt the data entrys
+  # attributes: - @file_type, nmd_version, build_date, PGP key id used for encrypting the encryption entry
+
+  # Zip Entry Structure after full protection
+  # crypt (encrypted: data_collections, data_objects, schema)
+  # encryption (encrypted)
+  # attributes (clear_text)
+
+  
+  # This hack is for... some tricky bullshit, I forgot about
+  def passfunc(hook, uid_hint, passphrase_info, prev_was_bad, fd)
+    $stderr.write("Passphrase for #{uid_hint}: ")
+    $stderr.flush
+    begin
+      system('stty -echo')
+      io = IO.for_fd(fd, 'w')
+      io.puts(gets)
+      io.flush
+    ensure
+      (0 ... $_.length).each do |i| $_[i] = ?0 end if $_
+      system('stty echo')
+    end
+    $stderr.puts
+  end
+  
+  
+end
+
+
+
+

data/lib/nm_datafile/schema.rb

@@ -0,0 +1,16 @@
+# A default schema is defined here... this needs to be converted into 
+# something more generic
+module NmDatafile
+  SCHEMA = { schemas: 
+                         { :shippable_file => {
+                           data_collections: [:sales, :line_items, :discounts, :addresses, :ubws, :encryption_pairs], # name the data that is input into the NMDatafile as an array
+                           data_objects: [:ready_for_shipment_batch]
+                           },
+                         
+                         :address_completion_file => {
+                           data_collections: [:sales, :erroneous_sales],
+                           data_objects: [:ready_for_shipment_batch]
+                           }
+                         }
+                       }
+end

data/lib/nm_datafile/version.rb

@@ -0,0 +1,3 @@
+module NmDatafile
+  VERSION = "0.0.0"
+end

data/lib/nm_datafile.rb

@@ -0,0 +1,21 @@
+require 'json'
+
+require "nm_datafile/version"
+require "nm_datafile/schema"
+require 'nm_datafile/b_f'
+require 'nm_datafile/crypto'
+require "nm_datafile/nm_datafile"
+
+
+module NmDatafile
+  # Your code goes here...
+  
+  def self.new(file_type, *args)
+    NmDatafile.new(file_type, args)
+  end
+  
+  
+end
+
+
+

data/nm_datafile.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'nm_datafile/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "nm_datafile"
+  spec.version       = NmDatafile::VERSION
+  spec.authors       = ["antisec"]
+  spec.email         = ["antisec@antisec.com"]
+  spec.summary       = %q{A gem that saves files into a secure encrypted file.}
+  spec.description   = %q{A gem that creates a data file based on arrays or strings that you feed it.  When you save the file, you can choose from multiple encryption methods, asymetric, symetric, etc. etc.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "rubyzip"
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "rspec"
+end

data/spec/nm_datafile_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+describe "nm_datafile" do
+
+  before :each do
+    #@sample_data = get_sample_data
+    @sale = {"address_id"=>1, "created_at"=>"2015-03-08T03:54:51Z", "currency_used"=>"BTC"}
+    @sample_data = [ @sale ]
+  end
+  
+
+  it "should be instantiable" do
+    NmDatafile.new(:shippable_file)
+  end
+
+
+  it "should be instantiable with data" do
+    #return_array = [ @sales, 
+    #                 @line_items, 
+    #                 @discounts,
+    #                 @addresses, 
+    #                 @utilized_bitcoin_wallets,
+    #                 @encryption_pairs,
+    #                 rfsb ]
+    
+    nmd_shippable = NmDatafile.new(:shippable_file, *@sample_data)
+    
+    str = nmd_shippable.save_to_string
+
+    # TODO: actually test data and make more data in @sample_data
+    
+    
+  end
+
+end
+
+

data/spec/spec_helper.rb

@@ -0,0 +1,82 @@
+require 'nm_datafile'
+require 'pry'
+
+RSpec.configure do |config|
+  # rspec-expectations config goes here. You can use an alternate
+  # assertion/expectation library such as wrong or the stdlib/minitest
+  # assertions if you prefer.
+  config.expect_with :rspec do |expectations|
+    # This option will default to `true` in RSpec 4. It makes the `description`
+    # and `failure_message` of custom matchers include text for helper methods
+    # defined using `chain`, e.g.:
+    #     be_bigger_than(2).and_smaller_than(4).description
+    #     # => "be bigger than 2 and smaller than 4"
+    # ...rather than:
+    #     # => "be bigger than 2"
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  # rspec-mocks config goes here. You can use an alternate test double
+  # library (such as bogus or mocha) by changing the `mock_with` option here.
+  config.mock_with :rspec do |mocks|
+    # Prevents you from mocking or stubbing a method that does not exist on
+    # a real object. This is generally recommended, and will default to
+    # `true` in RSpec 4.
+    mocks.verify_partial_doubles = true
+  end
+
+# The settings below are suggested to provide a good initial experience
+# with RSpec, but feel free to customize to your heart's content.
+=begin
+  # These two settings work together to allow you to limit a spec run
+  # to individual examples or groups you care about by tagging them with
+  # `:focus` metadata. When nothing is tagged with `:focus`, all examples
+  # get run.
+  config.filter_run :focus
+  config.run_all_when_everything_filtered = true
+
+  # Limits the available syntax to the non-monkey patched syntax that is
+  # recommended. For more details, see:
+  #   - http://myronmars.to/n/dev-blog/2012/06/rspecs-new-expectation-syntax
+  #   - http://teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/
+  #   - http://myronmars.to/n/dev-blog/2014/05/notable-changes-in-rspec-3#new__config_option_to_disable_rspeccore_monkey_patching
+  config.disable_monkey_patching!
+
+  # This setting enables warnings. It's recommended, but in some cases may
+  # be too noisy due to issues in dependencies.
+  config.warnings = true
+
+  # Many RSpec users commonly either run the entire suite or an individual
+  # file, and it's useful to allow more verbose output when running an
+  # individual spec file.
+  if config.files_to_run.one?
+    # Use the documentation formatter for detailed output,
+    # unless a formatter has already been configured
+    # (e.g. via a command-line flag).
+    config.default_formatter = 'doc'
+  end
+
+  # Print the 10 slowest examples and example groups at the
+  # end of the spec run, to help surface which specs are running
+  # particularly slow.
+  config.profile_examples = 10
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = :random
+
+  # Seed global randomization in this process using the `--seed` CLI option.
+  # Setting this allows you to use `--seed` to deterministically reproduce
+  # test failures related to randomization by passing the same `--seed` value
+  # as the one that triggered the failure.
+  Kernel.srand config.seed
+=end
+end
+
+
+
+def get_sample_data
+  
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: nm_datafile
+version: !ruby/object:Gem::Version
+  version: 0.0.0
+platform: ruby
+authors:
+- antisec
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A gem that creates a data file based on arrays or strings that you feed
+  it.  When you save the file, you can choose from multiple encryption methods, asymetric,
+  symetric, etc. etc.
+email:
+- antisec@antisec.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- changelog.md
+- lib/nm_datafile.rb
+- lib/nm_datafile/b_f.rb
+- lib/nm_datafile/crypto.rb
+- lib/nm_datafile/nm_datafile.rb
+- lib/nm_datafile/schema.rb
+- lib/nm_datafile/version.rb
+- nm_datafile.gemspec
+- spec/nm_datafile_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.6
+signing_key: 
+specification_version: 4
+summary: A gem that saves files into a secure encrypted file.
+test_files:
+- spec/nm_datafile_spec.rb
+- spec/spec_helper.rb