datashift 0.0.1

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,25 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "rails", '3.1.3'
+  gem "activerecord", '3.1.3'
+  gem "activesupport", '3.1.3'
+  
+  platform :jruby do
+    gem 'jruby-openssl'
+    gem 'activerecord-jdbcsqlite3-adapter'
+  end
+  
+  gem 'spree', '0.70.3'
+  gem "rspec", ">= 0"
+  gem "shoulda", ">= 0"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.8.3"
+end
+

data/Gemfile.lock

@@ -0,0 +1,211 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    actionmailer (3.1.3)
+      actionpack (= 3.1.3)
+      mail (~> 2.3.0)
+    actionpack (3.1.3)
+      activemodel (= 3.1.3)
+      activesupport (= 3.1.3)
+      builder (~> 3.0.0)
+      erubis (~> 2.7.0)
+      i18n (~> 0.6)
+      rack (~> 1.3.5)
+      rack-cache (~> 1.1)
+      rack-mount (~> 0.8.2)
+      rack-test (~> 0.6.1)
+      sprockets (~> 2.0.3)
+    activemerchant (1.17.0)
+      activesupport (>= 2.3.11)
+      braintree (>= 2.0.0)
+      builder (>= 2.0.0)
+      json (>= 1.5.1)
+    activemodel (3.1.3)
+      activesupport (= 3.1.3)
+      builder (~> 3.0.0)
+      i18n (~> 0.6)
+    activerecord (3.1.3)
+      activemodel (= 3.1.3)
+      activesupport (= 3.1.3)
+      arel (~> 2.2.1)
+      tzinfo (~> 0.3.29)
+    activerecord-jdbc-adapter (1.2.2)
+    activerecord-jdbcsqlite3-adapter (1.2.2)
+      activerecord-jdbc-adapter (~> 1.2.2)
+      jdbc-sqlite3 (~> 3.7.2)
+    activeresource (3.1.3)
+      activemodel (= 3.1.3)
+      activesupport (= 3.1.3)
+    activesupport (3.1.3)
+      multi_json (~> 1.0)
+    acts_as_list (0.1.4)
+    arel (2.2.1)
+    bcrypt-ruby (3.0.1-java)
+    bouncy-castle-java (1.5.0146.1)
+    braintree (2.13.3)
+      builder (>= 2.0.0)
+    builder (3.0.0)
+    cancan (1.6.5)
+    cocaine (0.2.1)
+    deface (0.7.2)
+      nokogiri (~> 1.5.0)
+      rails (>= 3.0.9)
+    devise (1.4.7)
+      bcrypt-ruby (~> 3.0)
+      orm_adapter (~> 0.0.3)
+      warden (~> 1.0.3)
+    diff-lcs (1.1.3)
+    erubis (2.7.0)
+    faker (1.0.0)
+      i18n (~> 0.4)
+    git (1.2.5)
+    highline (1.6.2)
+    hike (1.2.1)
+    i18n (0.6.0)
+    jdbc-sqlite3 (3.7.2)
+    jeweler (1.8.3)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+      rdoc
+    jquery-rails (1.0.19)
+      railties (~> 3.0)
+      thor (~> 0.14)
+    jruby-openssl (0.7.5)
+      bouncy-castle-java (>= 1.5.0146.1)
+    json (1.6.5-java)
+    kaminari (0.13.0)
+      actionpack (>= 3.0.0)
+      activesupport (>= 3.0.0)
+      railties (>= 3.0.0)
+    mail (2.3.0)
+      i18n (>= 0.4.0)
+      mime-types (~> 1.16)
+      treetop (~> 1.4.8)
+    meta_search (1.1.1)
+      actionpack (~> 3.1.0)
+      activerecord (~> 3.1.0)
+      activesupport (~> 3.1.0)
+      polyamorous (~> 0.5.0)
+    mime-types (1.17.2)
+    multi_json (1.0.4)
+    nested_set (1.6.8)
+      activerecord (>= 3.0.0)
+      railties (>= 3.0.0)
+    nokogiri (1.5.0-java)
+    orm_adapter (0.0.6)
+    paperclip (2.4.1)
+      activerecord (>= 2.3.0)
+      activesupport (>= 2.3.2)
+      cocaine (>= 0.0.2)
+      mime-types
+    polyamorous (0.5.0)
+      activerecord (~> 3.0)
+    polyglot (0.3.3)
+    rack (1.3.6)
+    rack-cache (1.1)
+      rack (>= 0.4)
+    rack-mount (0.8.3)
+      rack (>= 1.0.0)
+    rack-ssl (1.3.2)
+      rack
+    rack-test (0.6.1)
+      rack (>= 1.0)
+    rails (3.1.3)
+      actionmailer (= 3.1.3)
+      actionpack (= 3.1.3)
+      activerecord (= 3.1.3)
+      activeresource (= 3.1.3)
+      activesupport (= 3.1.3)
+      bundler (~> 1.0)
+      railties (= 3.1.3)
+    railties (3.1.3)
+      actionpack (= 3.1.3)
+      activesupport (= 3.1.3)
+      rack-ssl (~> 1.3.2)
+      rake (>= 0.8.7)
+      rdoc (~> 3.4)
+      thor (~> 0.14.6)
+    rake (0.9.2.2)
+    rd_find_by_param (0.1.1)
+      activerecord (~> 3.0)
+      activesupport (~> 3.0)
+    rd_resource_controller (1.0.1)
+    rdoc (3.12)
+      json (~> 1.4)
+    rspec (2.8.0)
+      rspec-core (~> 2.8.0)
+      rspec-expectations (~> 2.8.0)
+      rspec-mocks (~> 2.8.0)
+    rspec-core (2.8.0)
+    rspec-expectations (2.8.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.8.0)
+    shoulda (2.11.3)
+    spree (0.70.3)
+      spree_api (= 0.70.3)
+      spree_auth (= 0.70.3)
+      spree_core (= 0.70.3)
+      spree_dash (= 0.70.3)
+      spree_promo (= 0.70.3)
+      spree_sample (= 0.70.3)
+    spree_api (0.70.3)
+      spree_auth (= 0.70.3)
+      spree_core (= 0.70.3)
+    spree_auth (0.70.3)
+      cancan (= 1.6.5)
+      devise (= 1.4.7)
+      spree_core (= 0.70.3)
+    spree_core (0.70.3)
+      activemerchant (= 1.17.0)
+      acts_as_list (= 0.1.4)
+      deface (>= 0.7.0)
+      faker (= 1.0.0)
+      highline (= 1.6.2)
+      jquery-rails (>= 1.0.14)
+      kaminari (>= 0.12.4)
+      meta_search (= 1.1.1)
+      nested_set (= 1.6.8)
+      paperclip (= 2.4.1)
+      rails (>= 3.1.1, <= 3.1.3)
+      rd_find_by_param (= 0.1.1)
+      rd_resource_controller
+      state_machine (= 1.0.1)
+      stringex (= 1.0.3)
+    spree_dash (0.70.3)
+      spree_core (= 0.70.3)
+    spree_promo (0.70.3)
+      spree_auth (= 0.70.3)
+      spree_core (= 0.70.3)
+    spree_sample (0.70.3)
+      spree_core (= 0.70.3)
+    sprockets (2.0.3)
+      hike (~> 1.2)
+      rack (~> 1.0)
+      tilt (~> 1.1, != 1.3.0)
+    state_machine (1.0.1)
+    stringex (1.0.3)
+    thor (0.14.6)
+    tilt (1.3.3)
+    treetop (1.4.10)
+      polyglot
+      polyglot (>= 0.3.1)
+    tzinfo (0.3.31)
+    warden (1.0.6)
+      rack (>= 1.0)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  activerecord (= 3.1.3)
+  activerecord-jdbcsqlite3-adapter
+  activesupport (= 3.1.3)
+  bundler (~> 1.0.0)
+  jeweler (~> 1.8.3)
+  jruby-openssl
+  rails (= 3.1.3)
+  rdoc (~> 3.12)
+  rspec
+  shoulda
+  spree (= 0.70.3)

data/LICENSE.txt

@@ -0,0 +1,27 @@
+== DataShift 
+
+## License
+
+Copyright:: (c) Autotelik Media Ltd 2011
+Author ::   Tom Statter
+Date ::     Sept 2011
+
+The MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,286 @@
+##  DataShift 
+
+Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
+
+### Features
+
+Map Active Record models and associations to files.
+
+Import and Export ActiveRecord based data through .xls or CSV  files.
+
+Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby.
+
+High level rake tasks provided.
+
+Spree E-Commerce specific loaders and rake tasks provided enabling import/export of all data including Products with
+complex associations such as Options/Variants and Images.
+
+Example spreadsheets, fully documented with comments for each column.
+
+
+## Installation
+
+Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
+
+    `gem install datashift`
+
+To use :
+
+    gem 'datashift'
+    require 'datashift'
+
+To pull the tasks in, add this call to your Rakefile :
+
+     DataShift ::load_tasks
+
+N.B - To use the Excel loader, OLE and Excel are NOT required, however
+JRuby is required, since it uses Java's Apache POI under the hood to process .xls files.
+
+Guards are provided, and used internally, for mixed Ruby setups. Can be used like :
+
+    if(DataShift::Guards::jruby? )
+        ..do something with Excel
+    else
+        ..do something with CSV
+    end
+
+## Active Record - Import/Export
+
+Provides wrappers and rake tasks for importing/exporting data to DBs, from various sources,
+including csv and .xls files (Excel/Open Office)
+
+Simplifies the exchange of data, between such files and any active record supported database.
+
+A core feature of DataShift is the MethodMapper, which provides features for collecting
+reflection information from ActiveRecord models (all different associations, including join tables with many-to-many relationships).
+
+A full picture of all possible operations on a class can be created very easily, for example ona  Blog model :
+
+    MethodMapper.find_operators( Blog )
+
+This then allows Import/Export to be achieved, by mapping the file's header and column data to MethodMapper's operators
+(i.e. the model's attributes and associations).
+
+Here we retrieve the method details for a column name from a file, "Blog Date"
+
+    MethodMapper.find_method_detail( Blog, "Blog Date" )
+
+Loaders can use this method lookup functionality, to find the correct association for a column heading,
+and populate AR object with row data.
+
+This means data can be mapped to any model without any further coding. Generators are also supplied to export
+a models attributes and associations to files, thus providing template spreadsheets that any user can fill out.
+
+MethodMapper also stores column type information so the raw file data can be provided as is,
+and whenever possible, under the bonnet the data will be cast to correct DB type.
+
+Here we show how a column name from a file, "Blog Date", can be mapped to Assign a stringified date, to the blog_date column, on a new Blog object :
+
+    MethodMapper.find_method_detail( Blog, "Blog Date" ).assign( Blog.new, "Sat Jul 23 2011" )
+
+Because it's based on reflection against the model, can build complex relationships between tables during import/export, 
+and extending data files with new columns need not require any additional Ruby coding.
+
+New columns can simply be added to the Excel/Open Office spreadsheet, or CSV file, setting the new
+attribute or association name in the header row. 
+
+
+The Loader attempts to handle various human read-able forms of column names.
+
+For example, given an association on the model called, product_properties, will successfully load
+from columns with headings such as 'product_properties', 'Product Properties', 'ProductProperties' 'product properties' etc
+
+For has_many associations, either multiple columns can be used,
+ or multiple values can be specified in a single column using suitable delimiters.
+
+Modular - so complex associations/mappings that require non generic lookups, can be handled by extending the loader engine.
+
+Original focus was on support for the Open Source Spree e-commerce project, so includes specific loaders and rake tasks
+for loading Spree Products, and associated data such as Product Variants, and Images.
+
+## Template Generation and Export
+
+Template generation tasks can be used to export a model's definition as column headings to CSV or .xls.
+These can be provided to developers or business users, as a template for data collection and then loading.
+
+Export tasks can be used to export of a model's definition and any existing data stored in the database.
+
+This data can be exported directly to CSV or Excel/OpenOffice spreadsheets.
+
+
+## Example Spreadsheets
+    
+  A number of example Spreadsheets with headers and comments, can be found in the spec/fixtures directory.
+
+  Extensive Spree samples - including .xls and csv versions for simple Products or complex Products with multiple
+  taxons, variants properties etc - can be found in the spec/fixtures/spree subdirectory.
+
+  Column headings contain comments with full descriptions and instructions on syntax. 
+
+
+## Features
+
+- *High level wrappers around applications including Excel and Word
+
+  Quickly and easily access common enterprise applications through Ruby
+
+  Wrapper around MS Excel File format, acheived via Apache POI under JRuby, so not restricted to Windows
+  and Excel does not need to be installed.
+
+  The required POI jars are already included.
+
+- *Direct Excel export*
+
+  Excel/OpenOffice spreadsheets are heavily used in many sectors, so direct support makes it
+  easier and quicker to migrate your client's data into a Rails/ActiveRecord project.
+
+  No need to save to CSV or map to YAML.
+  
+- *Semi-Smart Name Lookup*
+
+  Includes helper classes that find and store details of all possible associations on an AR class.
+  Given a user supplied name, attempts to find the requested association.
+
+  Example usage, load from a file or spreadsheet where the column names are only
+  an approximation of the actual associations, so given 'Product Properties' heading,
+  finds real association 'product_properties' to send or call on the AR object
+
+- *Associations*
+
+  Can handle 'belongs_to, 'has_many' and 'has_one' associations, including assignment of multiple objects
+  via either multiple columns, or via specially delimited entry in a single (column). See Details section.
+
+  Supports delegated attributes.
+
+- *Rake Tasks*
+
+  High level Rake tasks are provided, only required to supply model class, and file location :
+
+    jruby -S rake datashift:import:excel model=MusicTrack input=MyTrackListing.xls
+
+
+- *Spree Rake Tasks*
+
+  Specific Rake tasks are also provided for Spree loading - currently supports Product with associations,
+  and Image loading.
+
+    jruby -S rake datashift:spree:products input=C:\MyProducts.xls
+
+
+  **Product loading from Excel files specifically requires JRuby (But not Excel or OLE)**.
+
+
+- *Seamless Spree Image loading can be achieved by ensuring SKU or class Name features in Image filename.
+
+  Lookup is performed either via the SKU being prepended to the image name, or by the image name being equal to the **name attribute** of the klass in question.
+
+  Images can be attached to any class defined with a suitable association. The class to use can be configured in rake task via
+  parameter klass=Xyz.
+
+  In the Spree tasks, this defaults to Product, so attempts to attach Image to a Product via Product SKU or Name.
+ 
+  Image loading **does not** specifically require JRuby
+
+  A report is generated in the current working directory detailing any Images in the paths that could not be matched with a Product.
+
+  rake  datashift:spree:images input=C:\images\product_images skip_if_no_assoc=true
+
+  rake  datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
+
+## Import to Active Record
+
+### Associations
+
+To perform a lookup for an associated model, the primary column(s) must be supplied, along with required select values for those columns.
+
+A single association column can contain multiple name/value sets, in string form :
+
+  column:lookup_key_1, lookup_key_2,...
+
+So if our Project model has many Categories, we can supply a Category list, which is keyed on the column Category.reference with :
+
+  |Categories|
+
+  reference:category_001,category_002
+
+During loading, a call to find_all_by_reference will be made, picking up the 2 categories with matching references,
+ and our Project model will contain those two i.e project.categories = [category_002,category_003]
+
+## Spree Suppprt
+
+### OptionTypes & Variants
+
+When loaded with the Spree specific tasks, spree specific over rides are supported, such as direct s
+support for OptionTypes with values
+
+Any 'Option Types' columns can contain the OptionType to associate with the Product, plus a selection of
+appropriate OptionValues to go with that Type. 
+
+For example, in a single column/row we could supply 2 OptionTypes (named, size & colour), with a selection values
+(such as small, medium etc)
+
+    'Option Types'
+    size:small,medium,large|colour:red,white
+
+If no such OptionType exists, e.g size, then a new one is created with the supplied name.
+
+Next the OptionValues are also parsed, again if no such OptionValue exists, e.g small, then a new one is created with the supplied name.
+
+Lastly a Variant is created on each OptionValue, with price and availaable dates being copied from Master.
+Currently a unique SKU is created by adding an index to the master's sku.
+
+TODO - Enable a hash of attributes to be supplied in association columns to enable more control over creation of associated objects.
+
+### Properties
+
+The properties to associate with this product.
+Properties are for small snippets of text, shared across many products,
+and are for display purposes only.
+
+An optional display value can be supplied to supplement the displayed text.
+
+As for all associations can contain multiple name/value sets in default form :
+
+    Property:display_value|Property:display_value
+
+Example - No values :
+    manufacturer|standard
+
+Example - Display  values :
+    manufacturer:somebody else plc|standard:ISOBlah21
+
+## TODO
+
+  - Add direct Image loading to Spree  i.e should be able to specify just path in a column .. "/images/red-tshirt.jpg"
+
+  - Smart sorting of column processing order ....
+
+  - Does not currently ensure mandatory columns (for valid?) processed first.
+    
+## License
+
+Copyright:: (c) Autotelik Media Ltd 2011
+
+Author ::   Tom Statter
+
+Date ::     Dec 2011
+
+The MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.