hirsute 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+--- 
+SHA1: 
+  data.tar.gz: d700fb375ef73a088c8570f51f22ebab0f6c126e
+  metadata.gz: 6d4be2b6029c43c577cca8bcc958f42a489ec96a
+SHA512: 
+  data.tar.gz: f1fcf9ccd33fae57312302103ff9b75c9150db8ac2bee8f05b1f00f88b610f2491e67e55636ed6eb6a08bad3fd0d8448241b75e097a0b53ce6644724f6763a06
+  metadata.gz: 59189a4d7d722bb6c92b2c46d8f66a512ceb61a4a9502a76238a6af7ecaf129108e73a855304a3b58ba0ed94fbcd2281899b34282f912a73d2d6f1ccb2d3bd6b

data/MIT_LICENSE

@@ -0,0 +1,7 @@
+Copyright (c) 2012 Derrick Schneider
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,105 @@
+Hirsute
+=======
+Hirsute is a Ruby-based domain specific language for generating plausible fake data. You might need fake data for any of the following reasons:
+
+* demoing to a potential customer and providing a realistic experience
+* building a "real" database for testing (versus the often messy, inaccurate databases in dev systems)
+* building a database for load testing before a launch
+
+In Hirsute, you define a template for what an object might look like, and then generate however many copies you need. Then you can work with those collections as needed. There is <a href="https://github.com/derricks/hirsute/blob/master/manual.md">a full manual</a>, but here is a quick example to give the flavor. Say you're building a system where you have a bunch of users, and you want to build in a "friend" concept that allows each user to have 10 friends. You want to generate a random sample of data, but you think most users will only have 2-4 friends.
+
+The relevant Hirsute script might look like this:
+
+<code><pre>
+    # define a user template that has an id that is an incrementing counter starting at 1 and an email address that we're defining as testuser1@gmail.com,testuser2@yahoo.com,testuser3@aol.com, and so forth
+
+    storage :mysql
+
+    a('user') {
+        has :id => counter,
+            :email => combination(
+                      "testuser",counter,"@",one_of(['gmail','aol','yahoo']),".com")
+        is_stored_in "users"
+    }
+
+    #make 1000 users
+    users = user * 1000
+
+    # define a friendship object that maps two users together. Just define the user ids as literals so they can be defined but can be filled in later
+    a('friendship') {
+        has :user1 => 1, 
+            :user2 => 1
+        is_stored_in "friendship"
+    }
+    friendships = collection_of friendship
+
+    # for each user, pick an appropriate number of friends and create the friendship objects
+    foreach user do |cur_user|
+      # figure out a number of friends this user might have. Pass in a histogram to steer the probability the way we want
+      
+      # while you can pass in an array of probabilities, you can also define the histogram more intuitively
+      friend_dist = <<-HIST
+      num_friends
+             0 **
+             1 **********
+             2 ******************************
+             3 ******************************
+             4 ********************
+             5 *
+             6 *
+             7 *
+             8 *
+             9 **
+            10 *
+      HIST
+      # the first argument is the options to draw from, the second argument (optional) is a histogram representing distribution of probabilities
+      num_friends = pick_from([0,1,2,3,4,5,6,7,8,9,10], HIST)
+
+      # since this in Ruby, you can just write in it as needed
+      (0...num_friends).each do |idx|
+         # grab a random user that isn't this one
+         friend = any(user) {|friend| friend.id != cur_user.id}
+
+         new_friendship = friendship.make # because there's only one collection holding these, it's added automatically
+         new_friendship.user1 = friend.id
+         new_friendship.user2 = cur_user.id
+      end
+    end
+
+    # and now write them all out to files
+    finish users
+    finish friendships
+</pre></code>
+
+This will create files that have data such as this:
+<code><pre>
+    INSERT INTO users ('email','id') VALUES ('testuser1@yahoo.com',1);
+    INSERT INTO users ('email','id') VALUES ('testuser2@yahoo.com',2);
+    INSERT INTO users ('email','id') VALUES ('testuser3@aol.com',3);
+    INSERT INTO users ('email','id') VALUES ('testuser4@yahoo.com',4);
+    INSERT INTO users ('email','id') VALUES ('testuser5@yahoo.com',5);
+</pre></code>
+
+and 
+
+<code><pre>
+    INSERT INTO friendship ('user1','user2') VALUES (624,1);
+    INSERT INTO friendship ('user1','user2') VALUES (808,1);
+    INSERT INTO friendship ('user1','user2') VALUES (81,1);
+    INSERT INTO friendship ('user1','user2') VALUES (15,2);
+</pre></code>
+
+To run this script, cd to the hirsute directory and run
+<code><pre>
+  ruby lib/hirsute.rb samples/readme.hrs
+</pre></code>
+
+Roadmap
+-------
+Hirsute is still early in development, but my goal is to continue adding output formats (currently only mysql and csv are supported) and generators as well as continuing to allow for more declarative syntax that would make the data generation more flexible, terse, and intuitive.
+
+I also don't think it will yet meet one of my needs, which is to generate the data for a multimillion-user system. So far, it does everything in memory, which will obviously cause problems for large data sets.
+
+Why Hirsute?
+------------
+The name was a joke with a friend. When I wanted something like this, I asked him, since he's up on many open-source projects. He said he didn't know of something like this, so I replied, "No, no. You're supposed to say 'Look at Hirsute' or something like that."

data/bin/hirsute

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+
+cur_wd = FileUtils.pwd()
+
+# convert args to absolute paths
+ARGV.map! {|item| File.expand_path(item)}
+
+# Absolute path to this script, e.g. /home/user/bin/foo.sh
+curdir = File.dirname(__FILE__)
+FileUtils.cd(File.join([curdir,".."]))
+load(File.join([curdir,"..","lib","hirsute.rb"]))
+
+FileUtils.cd(cur_wd)

data/lib/hirsute.rb

@@ -0,0 +1,110 @@
+# defines the basic functions used by the hirsute language, allowing a hirsute file to be loaded in
+# usage:
+#   ruby hirsute.rb <filename>
+# if filename is not specified, you can use this in irb to define language items
+
+require('lib/hirsute_utils.rb')
+require('lib/hirsute_template.rb')
+require('lib/hirsute_collection.rb')
+require('lib/hirsute_fixed.rb')
+require('lib/hirsute_output.rb')
+
+# store the absolute path of the file (if present) to ensure we don't lose track during chdirs
+ABS_HRS_FILE = File::expand_path(ARGV[0]) if ARGV[0]
+
+Dir::chdir(File::dirname(__FILE__) + "/..")
+
+
+include Hirsute::Support
+
+@outputters = {:mysql => Hirsute::MySQLOutputter,
+               :csv   => Hirsute::CSVOutputter}
+
+@objToTemplates = Hash.new
+
+def storage(storage_system)
+   @storage = storage_system
+end
+
+def storage_options(options = Hash.new)
+  @storage_options = options
+end
+
+# you can use an or a as your definition
+def an(objName,&block)
+  a(objName) {block.call}
+end
+
+# This method defines a Template with an identifier of objName. It is the basic method for defining
+# dummy objects
+def a(objName, &block)
+  
+  template = make_template(objName,&block)
+
+  @objToTemplates[objName] = template
+  
+  # this allows the client to do something like user * 5
+  # define_method objName -> template
+  self.class.send(:define_method,objName.to_sym) {template}
+  
+  
+  template
+end
+
+# iterates over every object of the specified type across any collection holding that type
+# usage: foreach user {|cur_user|}
+# if you only want to iterate over one collection, use that collection's each method
+def foreach(objTemplate)
+  #find every collection that has registered for this type of object (in the call you get the template)
+  colls = Hirsute::Collection.collections_holding_object(objTemplate.templateName)
+  colls.each {|coll| coll.each {|item| yield item if block_given?}}
+end
+
+# return any object in any collection that meets the criteria passed in a block
+def any(objTemplate,&block)
+  every(objTemplate,&block).choice
+end
+
+# return every object of a given type, combined from any collections that might include that type. If a block is passed, only elements returning true from the
+# block will be returned
+def every(objTemplate)
+  results = Array.new
+  foreach(objTemplate) do |item|
+    results << item if !block_given?
+    results << item if block_given? && (yield item)
+  end
+  results  
+end
+
+# tells Hirsute to output the given collection to the given storage system (or to generate the files necessary for that)
+# if no storage symbol is passed in, this will use the default set with the storage command
+def finish(collection,storageSymbol = nil)
+  raise "No storage defined. Use 'storage <symbol>' to define a storage type" if @storage.nil? && storageSymbol.nil?
+  
+  if storageSymbol.nil?
+    @outputters[@storage].new(collection,@storage_options).output
+  else
+    @outputters[storageSymbol].new(collection,@storage_options).output
+  end
+end
+
+# returns an empty collection
+def collection(objectName)
+  Hirsute::Collection.new(objectName)
+end
+
+# returns an empty collection of the specified template type
+def collection_of(template)
+  collection template.templateName
+end
+
+# pick an item from an array based on an optional histogram
+def pick_from(options,histogram = nil)
+  random_item_with_histogram(options,histogram)
+end
+  
+
+if ARGV[0]
+  Dir::chdir(File::dirname(ABS_HRS_FILE))
+  load ABS_HRS_FILE
+end

data/lib/hirsute_collection.rb

@@ -0,0 +1,67 @@
+# defines a Collection interface for Hirsute::Fixed objects
+# why not just an array? because eventually this might need to deal with objects in a text file for memory purposes, but I want to provide a consistent interface
+# in the short-term though, just wrap an array
+require('lib/hirsute_utils.rb')
+require('lib/hirsute_template.rb')
+
+module Hirsute
+  
+   class Collection
+       include Enumerable
+       # hold a class variable that contains all the collections for specific users
+       @object_names_to_collections = Hash.new
+       
+       #class methods
+       # return a list of all collections holding objects of the specified type
+       def self.collections_holding_object(object_name)
+         @object_names_to_collections[object_name]
+       end
+       
+       def self.registerCollectionForObject(collection,objectName)
+         if @object_names_to_collections[objectName]
+           @object_names_to_collections[objectName] << collection
+         else
+           @object_names_to_collections[objectName] = [collection]
+         end
+       end
+        
+       include Enumerable
+       include Support
+       
+       attr_reader :object_name
+       
+       def initialize(objectName)
+          @object_name = objectName # defines the object type kept in this collection
+          Hirsute::Collection.registerCollectionForObject(self,objectName)
+          @collection = Array.new
+       end
+       
+       def each(&block)
+          @collection.each(&block)
+       end
+       
+       def <<(element)
+          # allows for deferred definition of type
+           if element.kind_of? Hirsute::Template
+             self.<<(element.make(false))
+             return
+           end
+           
+           if element.kind_of? Hirsute::Collection
+              element.each {|item| self << item}
+              return
+           end
+                      
+           raise "Only objects of type #{@object_name} can be stored in this collection" if element.class != class_for_name(@object_name)    
+           
+           @collection << element
+      end
+       
+       def length; @collection.length; end;
+       
+       # so that collections can be used with the one_of generator
+       def choice
+         @collection.choice
+       end
+   end
+end

data/lib/hirsute_constraint.rb

@@ -0,0 +1,17 @@
+module Hirsute
+  # defines the interface for a Constraint object that can correct a field that would otherwise break a constraint
+  class Constraint
+    def correct(field_value)
+    end
+  end
+  
+  # this just adds 
+  class UniqueConstraint < Constraint
+    @counter = 1
+    def correct(field_value)
+      ret_val = field_value.to_s + @counter.to_s
+      @counter = @counter + 1
+      ret_val
+    end
+  end
+end

data/lib/hirsute_fixed.rb

@@ -0,0 +1,17 @@
+# this represents a fixed object created from a template
+module Hirsute
+  class Fixed
+      
+      attr_accessor :fields
+      
+      # though Hirsute can use set and get within itself, set also configures require(ibute accessors
+      # for the specified field, thus allowing more user-friendly management of Hirsute objects
+      def set(field,value)
+         set_method = (field.to_s + "=").to_sym
+         m = self.method((field.to_s + "=").to_sym)
+         m.call value
+      end
+      
+      def get(field); self.method(field.to_sym).call; end      
+  end
+end

data/lib/hirsute_generator.rb

@@ -0,0 +1,140 @@
+# represents a generator used to derive a field value. An empty object, save for a generate method that is here largely for
+# documentation. Instances of this class are set up within the Template object
+
+require ('lib/hirsute_utils.rb')
+
+module Hirsute
+  class Generator
+    include Hirsute::Support
+         
+     def initialize(block)
+       @finalizer = block
+     end
+     
+     # do the actual work of generating a value. takes the fixed object being made as an argument
+     def generate(onObj)
+       result = _generate(onObj)
+
+       # if a generator returns a generator, keep going down the chain
+       while result.kind_of? Generator
+         result = result.generate(onObj)
+       end
+       
+       # if it's a range, grab the array from the range and choose one item randomly
+       if result.kind_of? Range
+         ary = Hirsute::Support.get_range_array(result)
+         result = ary.choice
+       end
+       
+       finish(result,onObj)
+     end
+     
+     def _generate(onObj)
+     end
+     
+     private
+       def finish(value,onObj)
+         # create a local copy for closure
+         if @finalizer
+           onObj.instance_exec value, &@finalizer
+         else
+           value
+         end
+       end
+  end
+  
+  #in this case, the definition is fixed, so no need for dynamic construction
+  class CompoundGenerator < Generator
+      
+      def initialize(generators,block)
+         @generators = generators
+         super(block)
+      end
+      
+      # return the joined response of each embedded generator
+      def _generate(onObj)
+         ret_val = ""
+         @generators.each {|gen| ret_val = ret_val + gen.generate(onObj).to_s}
+         ret_val
+      end
+  end
+  
+  # convenience class for literal values (especially strings and nil) 
+  class LiteralGenerator < Generator
+      def initialize(value,block)
+        @value = value
+        super(block)
+      end
+      
+      def _generate(onObj)
+        @value
+      end
+   end
+   
+   class ReadFromFileGenerator < Generator
+     
+     public
+       def initialize(file_name,algorithm,block)
+         @file_name = file_name
+         @file = File.open @file_name
+         @algorithm = algorithm
+         super(block)
+       end
+       
+       def _generate(onObj)
+         if @algorithm == :markov
+           advance_count = rand(100)
+           read_line_at(advance_count)
+         elsif @algorithm == :linear
+           read_line_at(1)
+         else
+           raise "Unknown read_from_file algorithm: " + @algorithm
+         end
+       
+       end
+      
+     private
+       def reset_file
+         @file.close
+         @file = File.open(@file_name)
+       end
+     
+       # advances line_count lines from current location (resetting the file if necessary)
+       # and returns the relevant line
+       def read_line_at(line_count)
+         line = ""
+         (0...line_count).each do |idx|
+           line = @file.gets
+           
+           while line && line.strip == ""
+             line = @file.gets
+           end
+           
+           if line.nil? # reached the end of the file
+             reset_file
+             line = read_line_at(1)
+           end
+         end
+         line.chomp
+      end
+  end
+  
+  # generators of this type are dependant on some field in the final object already being set
+  # this base class allows 
+  class DependentGenerator < Generator
+    def initialize(dependencyFields,block)
+      @dependencyFields = dependencyFields
+      super(block)
+    end
+    
+    def dependency_fields
+      if @dependencyFields.kind_of? Array
+        @dependencyFields
+      else
+        @dependencyFields = [@dependencyFields]
+        dependency_fields
+      end
+    end
+      
+  end
+end