datashift 0.16.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (422) hide show
  1. checksums.yaml +4 -4
  2. data/{LICENSE.txt → LICENSE} +0 -0
  3. data/Rakefile +1 -20
  4. data/datashift.thor +125 -0
  5. data/lib/applications/apache_poi_extensions.rb +21 -52
  6. data/lib/applications/excel.rb +64 -57
  7. data/lib/applications/hssf_row_extensions.rb +66 -0
  8. data/lib/applications/jexcel_file.rb +99 -95
  9. data/lib/applications/jexcel_file_extensions.rb +76 -83
  10. data/lib/applications/jruby/word.rb +36 -36
  11. data/lib/applications/ruby_poi_translations.rb +34 -32
  12. data/lib/applications/spreadsheet_extensions.rb +21 -19
  13. data/lib/datashift.rb +49 -59
  14. data/lib/datashift/binder.rb +217 -0
  15. data/lib/datashift/column_packer.rb +21 -72
  16. data/lib/datashift/configuration.rb +317 -0
  17. data/lib/datashift/context_factory.rb +88 -0
  18. data/lib/datashift/core_ext/array.rb +15 -0
  19. data/lib/datashift/core_ext/csv_ext.rb +46 -0
  20. data/lib/datashift/core_ext/string.rb +49 -0
  21. data/lib/datashift/core_ext/to_b.rb +11 -0
  22. data/lib/datashift/delimiters.rb +55 -61
  23. data/lib/datashift/doc_context.rb +137 -0
  24. data/lib/datashift/excel_base.rb +93 -81
  25. data/lib/datashift/exceptions.rb +30 -28
  26. data/lib/datashift/file_definitions.rb +44 -39
  27. data/lib/datashift/guards.rb +5 -5
  28. data/lib/datashift/header.rb +25 -0
  29. data/lib/datashift/headers.rb +94 -0
  30. data/lib/datashift/inbound_data/column.rb +44 -0
  31. data/lib/datashift/inbound_data/lookup_support.rb +33 -0
  32. data/lib/datashift/inbound_data/method_binding.rb +139 -0
  33. data/lib/datashift/load_object.rb +37 -12
  34. data/lib/datashift/logging.rb +54 -27
  35. data/lib/datashift/mandatory.rb +39 -0
  36. data/lib/datashift/mapping/data_flow_schema.rb +198 -0
  37. data/lib/datashift/{model_mapper.rb → mapping/mapper_utils.rb} +30 -10
  38. data/lib/datashift/model_methods/catalogue.rb +183 -0
  39. data/lib/datashift/model_methods/collection.rb +140 -0
  40. data/lib/datashift/model_methods/model_method.rb +162 -0
  41. data/lib/datashift/model_methods/model_methods_manager.rb +76 -0
  42. data/lib/datashift/model_methods/operator.rb +62 -0
  43. data/lib/datashift/node_collection.rb +26 -0
  44. data/lib/datashift/node_context.rb +68 -0
  45. data/lib/datashift/populator.rb +308 -282
  46. data/lib/datashift/progress_monitor.rb +91 -0
  47. data/lib/datashift/querying.rb +110 -52
  48. data/lib/datashift/templates/import_export_config.erb +55 -0
  49. data/lib/datashift/transformation/factory.rb +219 -0
  50. data/lib/datashift/transformation/remove.rb +44 -0
  51. data/lib/datashift/version.rb +3 -0
  52. data/lib/exporters/configuration.rb +84 -0
  53. data/lib/exporters/csv_exporter.rb +54 -52
  54. data/lib/exporters/excel_exporter.rb +80 -61
  55. data/lib/exporters/exporter_base.rb +8 -8
  56. data/lib/generators/config_generator.rb +218 -0
  57. data/lib/generators/csv_generator.rb +13 -70
  58. data/lib/generators/excel_generator.rb +23 -111
  59. data/lib/generators/generator_base.rb +15 -70
  60. data/lib/loaders/configuration.rb +90 -0
  61. data/lib/loaders/csv_loader.rb +63 -101
  62. data/lib/loaders/excel_loader.rb +71 -156
  63. data/lib/loaders/failure_data.rb +40 -0
  64. data/lib/loaders/file_loader.rb +16 -0
  65. data/lib/loaders/loader_base.rb +82 -410
  66. data/lib/loaders/loader_factory.rb +42 -0
  67. data/lib/loaders/paperclip/attachment_loader.rb +157 -140
  68. data/lib/loaders/paperclip/datashift_paperclip.rb +18 -35
  69. data/lib/loaders/paperclip/image_loading.rb +40 -35
  70. data/lib/loaders/reporters/basic_stdout_reporter.rb +40 -0
  71. data/lib/loaders/reporters/reporter.rb +26 -0
  72. data/lib/tasks/config.thor +65 -0
  73. data/{tasks → lib/tasks}/config/seed_fu_product_template.erb +0 -0
  74. data/{tasks → lib/tasks}/config/tidy_config.txt +0 -0
  75. data/lib/tasks/export.thor +192 -0
  76. data/lib/tasks/generate.thor +190 -0
  77. data/lib/tasks/import.thor +142 -0
  78. data/lib/{thor → tasks}/paperclip.thor +69 -69
  79. data/{tasks → lib/tasks/to_convert_to_thor}/db_tasks.rake +20 -20
  80. data/lib/tasks/tools.thor +109 -0
  81. data/spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg +0 -0
  82. data/spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg +0 -0
  83. data/spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg +0 -0
  84. data/spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg +0 -0
  85. data/spec/datashift/binder_spec.rb +266 -0
  86. data/spec/datashift/config_generator_spec.rb +186 -0
  87. data/spec/datashift/configuration.rb +66 -0
  88. data/spec/datashift/context_factory_spec.rb +63 -0
  89. data/spec/datashift/data_flow_schema_spec.rb +150 -0
  90. data/spec/datashift/datashift_spec.rb +52 -0
  91. data/spec/datashift/excel_base_spec.rb +57 -0
  92. data/spec/datashift/excel_spec.rb +188 -0
  93. data/spec/datashift/failure_data_spec.rb +27 -0
  94. data/spec/{file_definitions.rb → datashift/file_definitions.rb} +9 -10
  95. data/spec/datashift/headers_spec.rb +56 -0
  96. data/spec/datashift/inbound_data_spec.rb +47 -0
  97. data/spec/datashift/mapper_utils_spec.rb +38 -0
  98. data/spec/datashift/method_binding_spec.rb +60 -0
  99. data/spec/datashift/model_method_spec.rb +109 -0
  100. data/spec/datashift/model_methods_catalogue.rb +111 -0
  101. data/spec/datashift/model_methods_collection_spec.rb +138 -0
  102. data/spec/datashift/model_methods_manager_spec.rb +329 -0
  103. data/spec/datashift/populator_spec.rb +117 -0
  104. data/spec/datashift/thor_spec.rb +314 -0
  105. data/spec/datashift/transformation/factory_spec.rb +195 -0
  106. data/spec/datashift/transformation/transformer_remove_spec.rb +43 -0
  107. data/spec/dummy/Gemfile +53 -0
  108. data/spec/dummy/Gemfile.lock +197 -0
  109. data/spec/dummy/README.rdoc +28 -0
  110. data/spec/dummy/Rakefile +6 -0
  111. data/spec/dummy/app/assets/javascripts/application.js +16 -0
  112. data/spec/dummy/app/assets/javascripts/categories.js +2 -0
  113. data/spec/dummy/app/assets/javascripts/digitals.js +2 -0
  114. data/spec/dummy/app/assets/javascripts/empties.js +2 -0
  115. data/spec/dummy/app/assets/javascripts/loader_releases.js +2 -0
  116. data/spec/dummy/app/assets/javascripts/long_and_complex_table_linked_to_versions.js +2 -0
  117. data/spec/dummy/app/assets/javascripts/milestones.js +2 -0
  118. data/spec/dummy/app/assets/javascripts/owners.js +2 -0
  119. data/spec/dummy/app/assets/javascripts/projects.js +2 -0
  120. data/spec/dummy/app/assets/javascripts/users.js +2 -0
  121. data/spec/dummy/app/assets/javascripts/versions.js +2 -0
  122. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  123. data/spec/dummy/app/assets/stylesheets/categories.css +4 -0
  124. data/spec/dummy/app/assets/stylesheets/digitals.css +4 -0
  125. data/spec/dummy/app/assets/stylesheets/empties.css +4 -0
  126. data/spec/dummy/app/assets/stylesheets/loader_releases.css +4 -0
  127. data/spec/dummy/app/assets/stylesheets/long_and_complex_table_linked_to_versions.css +4 -0
  128. data/spec/dummy/app/assets/stylesheets/milestones.css +4 -0
  129. data/spec/dummy/app/assets/stylesheets/owners.css +4 -0
  130. data/spec/dummy/app/assets/stylesheets/projects.css +4 -0
  131. data/spec/dummy/app/assets/stylesheets/scaffold.css +56 -0
  132. data/spec/dummy/app/assets/stylesheets/users.css +4 -0
  133. data/spec/dummy/app/assets/stylesheets/versions.css +4 -0
  134. data/spec/dummy/app/controllers/application_controller.rb +5 -0
  135. data/spec/dummy/app/controllers/categories_controller.rb +58 -0
  136. data/spec/dummy/app/controllers/digitals_controller.rb +58 -0
  137. data/spec/dummy/app/controllers/empties_controller.rb +58 -0
  138. data/spec/dummy/app/controllers/loader_releases_controller.rb +58 -0
  139. data/spec/dummy/app/controllers/long_and_complex_table_linked_to_versions_controller.rb +58 -0
  140. data/spec/dummy/app/controllers/milestones_controller.rb +58 -0
  141. data/spec/dummy/app/controllers/owners_controller.rb +58 -0
  142. data/spec/dummy/app/controllers/projects_controller.rb +58 -0
  143. data/spec/dummy/app/controllers/users_controller.rb +58 -0
  144. data/spec/dummy/app/controllers/versions_controller.rb +58 -0
  145. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  146. data/spec/dummy/app/helpers/categories_helper.rb +2 -0
  147. data/spec/dummy/app/helpers/digitals_helper.rb +2 -0
  148. data/spec/dummy/app/helpers/empties_helper.rb +2 -0
  149. data/spec/dummy/app/helpers/loader_releases_helper.rb +2 -0
  150. data/spec/dummy/app/helpers/long_and_complex_table_linked_to_versions_helper.rb +2 -0
  151. data/spec/dummy/app/helpers/milestones_helper.rb +2 -0
  152. data/spec/dummy/app/helpers/owners_helper.rb +2 -0
  153. data/spec/dummy/app/helpers/projects_helper.rb +2 -0
  154. data/spec/dummy/app/helpers/users_helper.rb +2 -0
  155. data/spec/dummy/app/helpers/versions_helper.rb +2 -0
  156. data/spec/dummy/app/models/category.rb +6 -0
  157. data/spec/dummy/app/models/digital.rb +22 -0
  158. data/spec/dummy/app/models/empty.rb +2 -0
  159. data/spec/dummy/app/models/loader_release.rb +10 -0
  160. data/spec/dummy/app/models/long_and_complex_table_linked_to_version.rb +6 -0
  161. data/spec/dummy/app/models/milestone.rb +15 -0
  162. data/spec/dummy/app/models/owner.rb +13 -0
  163. data/spec/dummy/app/models/project.rb +53 -0
  164. data/spec/dummy/app/models/user.rb +5 -0
  165. data/spec/dummy/app/models/version.rb +7 -0
  166. data/spec/dummy/app/views/categories/_form.html.erb +17 -0
  167. data/spec/dummy/app/views/categories/edit.html.erb +6 -0
  168. data/spec/dummy/app/views/categories/index.html.erb +25 -0
  169. data/spec/dummy/app/views/categories/new.html.erb +5 -0
  170. data/spec/dummy/app/views/categories/show.html.erb +4 -0
  171. data/spec/dummy/app/views/digitals/_form.html.erb +17 -0
  172. data/spec/dummy/app/views/digitals/edit.html.erb +6 -0
  173. data/spec/dummy/app/views/digitals/index.html.erb +25 -0
  174. data/spec/dummy/app/views/digitals/new.html.erb +5 -0
  175. data/spec/dummy/app/views/digitals/show.html.erb +4 -0
  176. data/spec/dummy/app/views/empties/_form.html.erb +17 -0
  177. data/spec/dummy/app/views/empties/edit.html.erb +6 -0
  178. data/spec/dummy/app/views/empties/index.html.erb +25 -0
  179. data/spec/dummy/app/views/empties/new.html.erb +5 -0
  180. data/spec/dummy/app/views/empties/show.html.erb +4 -0
  181. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  182. data/spec/dummy/app/views/loader_releases/_form.html.erb +17 -0
  183. data/spec/dummy/app/views/loader_releases/edit.html.erb +6 -0
  184. data/spec/dummy/app/views/loader_releases/index.html.erb +25 -0
  185. data/spec/dummy/app/views/loader_releases/new.html.erb +5 -0
  186. data/spec/dummy/app/views/loader_releases/show.html.erb +4 -0
  187. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/_form.html.erb +17 -0
  188. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/edit.html.erb +6 -0
  189. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/index.html.erb +25 -0
  190. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/new.html.erb +5 -0
  191. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/show.html.erb +4 -0
  192. data/spec/dummy/app/views/milestones/_form.html.erb +17 -0
  193. data/spec/dummy/app/views/milestones/edit.html.erb +6 -0
  194. data/spec/dummy/app/views/milestones/index.html.erb +25 -0
  195. data/spec/dummy/app/views/milestones/new.html.erb +5 -0
  196. data/spec/dummy/app/views/milestones/show.html.erb +4 -0
  197. data/spec/dummy/app/views/owners/_form.html.erb +17 -0
  198. data/spec/dummy/app/views/owners/edit.html.erb +6 -0
  199. data/spec/dummy/app/views/owners/index.html.erb +25 -0
  200. data/spec/dummy/app/views/owners/new.html.erb +5 -0
  201. data/spec/dummy/app/views/owners/show.html.erb +4 -0
  202. data/spec/dummy/app/views/projects/_form.html.erb +17 -0
  203. data/spec/dummy/app/views/projects/edit.html.erb +6 -0
  204. data/spec/dummy/app/views/projects/index.html.erb +25 -0
  205. data/spec/dummy/app/views/projects/new.html.erb +5 -0
  206. data/spec/dummy/app/views/projects/show.html.erb +4 -0
  207. data/spec/dummy/app/views/users/_form.html.erb +17 -0
  208. data/spec/dummy/app/views/users/edit.html.erb +6 -0
  209. data/spec/dummy/app/views/users/index.html.erb +25 -0
  210. data/spec/dummy/app/views/users/new.html.erb +5 -0
  211. data/spec/dummy/app/views/users/show.html.erb +4 -0
  212. data/spec/dummy/app/views/versions/_form.html.erb +17 -0
  213. data/spec/dummy/app/views/versions/edit.html.erb +6 -0
  214. data/spec/dummy/app/views/versions/index.html.erb +25 -0
  215. data/spec/dummy/app/views/versions/new.html.erb +5 -0
  216. data/spec/dummy/app/views/versions/show.html.erb +4 -0
  217. data/spec/dummy/bin/bundle +3 -0
  218. data/spec/dummy/bin/rails +9 -0
  219. data/spec/dummy/bin/rake +9 -0
  220. data/spec/dummy/bin/setup +29 -0
  221. data/spec/dummy/bin/spring +16 -0
  222. data/spec/dummy/config.ru +4 -0
  223. data/spec/dummy/config/application.rb +26 -0
  224. data/spec/dummy/config/boot.rb +3 -0
  225. data/spec/dummy/config/database.yml +25 -0
  226. data/spec/dummy/config/environment.rb +5 -0
  227. data/spec/dummy/config/environments/development.rb +41 -0
  228. data/spec/dummy/config/environments/production.rb +79 -0
  229. data/spec/dummy/config/environments/test.rb +42 -0
  230. data/spec/dummy/config/initializers/assets.rb +11 -0
  231. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  232. data/spec/dummy/config/initializers/cookies_serializer.rb +3 -0
  233. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  234. data/spec/dummy/config/initializers/inflections.rb +16 -0
  235. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  236. data/spec/dummy/config/initializers/session_store.rb +3 -0
  237. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  238. data/spec/dummy/config/locales/en.yml +23 -0
  239. data/spec/dummy/config/routes.rb +76 -0
  240. data/spec/dummy/config/secrets.yml +22 -0
  241. data/spec/dummy/db/development.sqlite3 +0 -0
  242. data/spec/dummy/db/migrate/20110803201325_create_test_bed.rb +98 -0
  243. data/spec/dummy/db/migrate/20121009161700_add_digitals.rb +24 -0
  244. data/spec/dummy/db/migrate/20161005123106_create_digitals.rb +8 -0
  245. data/spec/dummy/db/migrate/20161005123106_create_long_and_complex_table_linked_to_versions.rb +8 -0
  246. data/spec/dummy/db/migrate/20161005123107_create_loader_releases.rb +8 -0
  247. data/spec/dummy/db/migrate/20161005123108_create_owners.rb +8 -0
  248. data/spec/dummy/db/migrate/20161005123109_create_empties.rb +8 -0
  249. data/spec/dummy/db/migrate/20161005123110_create_projects.rb +8 -0
  250. data/spec/dummy/db/migrate/20161005123111_create_users.rb +8 -0
  251. data/spec/dummy/db/migrate/20161005123111_create_versions.rb +8 -0
  252. data/spec/dummy/db/migrate/20161005123112_create_milestones.rb +8 -0
  253. data/spec/dummy/db/migrate/20161005123113_create_categories.rb +8 -0
  254. data/spec/dummy/db/schema.rb +93 -0
  255. data/spec/dummy/db/seeds.rb +9 -0
  256. data/spec/dummy/db/test.sqlite3 +0 -0
  257. data/spec/dummy/log/test.log +69 -0
  258. data/spec/dummy/public/404.html +67 -0
  259. data/spec/dummy/public/422.html +67 -0
  260. data/spec/dummy/public/500.html +66 -0
  261. data/spec/dummy/public/favicon.ico +0 -0
  262. data/spec/dummy/public/robots.txt +5 -0
  263. data/spec/dummy/sandbox_example.thor +4 -0
  264. data/spec/dummy/test/controllers/categories_controller_test.rb +49 -0
  265. data/spec/dummy/test/controllers/digitals_controller_test.rb +49 -0
  266. data/spec/dummy/test/controllers/empties_controller_test.rb +49 -0
  267. data/spec/dummy/test/controllers/loader_releases_controller_test.rb +49 -0
  268. data/spec/dummy/test/controllers/long_and_complex_table_linked_to_versions_controller_test.rb +49 -0
  269. data/spec/dummy/test/controllers/milestones_controller_test.rb +49 -0
  270. data/spec/dummy/test/controllers/owners_controller_test.rb +49 -0
  271. data/spec/dummy/test/controllers/projects_controller_test.rb +49 -0
  272. data/spec/dummy/test/controllers/users_controller_test.rb +49 -0
  273. data/spec/dummy/test/controllers/versions_controller_test.rb +49 -0
  274. data/spec/dummy/test/factories/categories.rb +5 -0
  275. data/spec/dummy/test/factories/digitals.rb +5 -0
  276. data/spec/dummy/test/factories/empties.rb +5 -0
  277. data/spec/dummy/test/factories/loader_releases.rb +5 -0
  278. data/spec/dummy/test/factories/long_and_complex_table_linked_to_versions.rb +5 -0
  279. data/spec/dummy/test/factories/milestones.rb +5 -0
  280. data/spec/dummy/test/factories/owners.rb +5 -0
  281. data/spec/dummy/test/factories/projects.rb +5 -0
  282. data/spec/dummy/test/factories/users.rb +5 -0
  283. data/spec/dummy/test/factories/versions.rb +5 -0
  284. data/spec/dummy/test/models/category_test.rb +7 -0
  285. data/spec/dummy/test/models/digital_test.rb +7 -0
  286. data/spec/dummy/test/models/empty_test.rb +7 -0
  287. data/spec/dummy/test/models/loader_release_test.rb +7 -0
  288. data/spec/dummy/test/models/long_and_complex_table_linked_to_version_test.rb +7 -0
  289. data/spec/dummy/test/models/milestone_test.rb +7 -0
  290. data/spec/dummy/test/models/owner_test.rb +7 -0
  291. data/spec/dummy/test/models/project_test.rb +7 -0
  292. data/spec/dummy/test/models/user_test.rb +7 -0
  293. data/spec/dummy/test/models/version_test.rb +7 -0
  294. data/spec/dummy/test/test_helper.rb +10 -0
  295. data/spec/exporters/csv_exporter_spec.rb +240 -0
  296. data/spec/exporters/csv_generator_spec.rb +139 -0
  297. data/spec/exporters/excel_exporter_spec.rb +193 -0
  298. data/spec/exporters/excel_generator_spec.rb +181 -0
  299. data/spec/exporters/generator_base_spec.rb +45 -0
  300. data/spec/factories/categories.rb +7 -0
  301. data/spec/factories/factories.rb +18 -0
  302. data/spec/factories/milestone.rb +16 -0
  303. data/spec/factories/projects.rb +41 -0
  304. data/spec/fixtures/BadAssociationName.xls +0 -0
  305. data/spec/fixtures/DemoNegativeTesting.xls +0 -0
  306. data/spec/fixtures/ProjectConfiguration.yml +18 -0
  307. data/spec/fixtures/ProjectsMultiCategories.xls +0 -0
  308. data/spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls +0 -0
  309. data/spec/fixtures/ProjectsSingleCategories.xls +0 -0
  310. data/spec/fixtures/ProjectsSingleCategories.xlsx +0 -0
  311. data/spec/fixtures/SimpleProjects.xls +0 -0
  312. data/spec/fixtures/config/database.yml +28 -0
  313. data/spec/fixtures/csv/BadAssociationName.csv +6 -0
  314. data/spec/fixtures/csv/DemoNegativeTesting.csv +6 -0
  315. data/spec/fixtures/csv/ProjectsMultiCategories.csv +5 -0
  316. data/spec/fixtures/csv/ProjectsMultiCategoriesHeaderLookup.csv +5 -0
  317. data/spec/fixtures/csv/ProjectsSingleCategories.csv +5 -0
  318. data/spec/fixtures/csv/SimpleProjects.csv +4 -0
  319. data/spec/fixtures/db/migrate/20110803201325_create_test_bed.rb +98 -0
  320. data/spec/fixtures/db/migrate/20121009161700_add_digitals.rb +24 -0
  321. data/spec/fixtures/db/seeds.rb +9 -0
  322. data/spec/fixtures/images/DEMO_001_ror_bag.jpeg +0 -0
  323. data/spec/fixtures/images/DEMO_002_Powerstation.jpeg +0 -0
  324. data/spec/fixtures/images/DEMO_003_ror_mug.jpeg +0 -0
  325. data/spec/fixtures/images/DEMO_004_ror_ringer.jpeg +0 -0
  326. data/spec/fixtures/load_datashift.thor +3 -0
  327. data/spec/fixtures/models/category.rb +6 -0
  328. data/spec/fixtures/models/digital.rb +22 -0
  329. data/spec/fixtures/models/empty.rb +2 -0
  330. data/spec/fixtures/models/loader_release.rb +10 -0
  331. data/spec/fixtures/models/long_and_complex_table_linked_to_version.rb +6 -0
  332. data/spec/fixtures/models/milestone.rb +15 -0
  333. data/spec/fixtures/models/owner.rb +13 -0
  334. data/spec/fixtures/models/project.rb +53 -0
  335. data/spec/fixtures/models/user.rb +5 -0
  336. data/spec/fixtures/models/version.rb +7 -0
  337. data/spec/fixtures/results/exp_project_assoc_headers.xls +0 -0
  338. data/spec/fixtures/results/exp_project_collection_spec.csv +2 -0
  339. data/spec/fixtures/results/exp_project_export.xls +0 -0
  340. data/spec/fixtures/results/exp_project_first_export.xls +0 -0
  341. data/spec/fixtures/results/exp_project_plus_assoc.xls +0 -0
  342. data/spec/fixtures/results/exp_project_plus_assoc_export_spec.csv +9 -0
  343. data/spec/fixtures/results/gen_project_plus_assoc_template.xls +0 -0
  344. data/spec/fixtures/results/gen_project_plus_some_assoc_template.xls +0 -0
  345. data/spec/fixtures/results/gen_project_template.xls +0 -0
  346. data/spec/fixtures/results/project_and_assoc_in_hash_export.xls +0 -0
  347. data/spec/fixtures/results/project_and_assoc_in_json_export.csv +9 -0
  348. data/spec/fixtures/results/project_and_assoc_in_json_export.xls +0 -0
  349. data/spec/fixtures/results/project_export_spec_with_custom_delim_,.csv +2 -0
  350. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/243.csv +2 -0
  351. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/247.csv +2 -0
  352. data/spec/fixtures/results/project_plus_assoc_template.csv +1 -0
  353. data/spec/fixtures/results/project_plus_some_assoc_template.csv +1 -0
  354. data/spec/fixtures/results/project_remove_export_spec.csv +2 -0
  355. data/spec/fixtures/results/project_template.csv +1 -0
  356. data/spec/fixtures/results/project_with_methods_export_spec.csv +2 -0
  357. data/spec/fixtures/results/thor_spec_gen_project.csv +1 -0
  358. data/spec/fixtures/sandbox_example.thor +4 -0
  359. data/spec/fixtures/simple_export_spec.xls +0 -0
  360. data/spec/fixtures/simple_template_spec.xls +0 -0
  361. data/spec/fixtures/test_model_defs.rb +7 -0
  362. data/spec/loaders/csv_loader_spec.rb +206 -0
  363. data/spec/loaders/data_flow_excel_loader_spec.rb +290 -0
  364. data/spec/loaders/excel_loader_failures_spec.rb +67 -0
  365. data/spec/loaders/excel_loader_spec.rb +294 -0
  366. data/spec/loaders/loader_base_spec.rb +29 -0
  367. data/spec/loaders/paperclip_loader_spec.rb +106 -0
  368. data/spec/log/datashift.log +14930 -0
  369. data/spec/private/digitals/1/DEMO_003_ror_mug.jpeg +0 -0
  370. data/spec/private/digitals/2/DEMO_002_Powerstation.jpeg +0 -0
  371. data/spec/private/digitals/3/DEMO_004_ror_ringer.jpeg +0 -0
  372. data/spec/private/digitals/4/DEMO_001_ror_bag.jpeg +0 -0
  373. data/spec/spec_helper.rb +26 -230
  374. data/spec/support/clear_and_manage_contexts.rb +25 -0
  375. data/spec/support/database_cleaner.rb +32 -0
  376. data/spec/support/datashift_test_helpers.rb +153 -0
  377. data/spec/support/files_paths_helper.rb +13 -0
  378. data/spec/support/fixtures/results/mapping_template.yaml +15 -0
  379. data/spec/support/sandbox.rb +136 -0
  380. metadata +804 -85
  381. data/README.markdown +0 -274
  382. data/README.rdoc +0 -19
  383. data/VERSION +0 -1
  384. data/datashift.gemspec +0 -48
  385. data/lib/applications/jruby/old_pre_proxy_jexcel_file.rb +0 -437
  386. data/lib/datashift/data_transforms.rb +0 -83
  387. data/lib/datashift/mapping_file_definitions.rb +0 -88
  388. data/lib/datashift/mapping_service.rb +0 -91
  389. data/lib/datashift/method_detail.rb +0 -165
  390. data/lib/datashift/method_details_manager.rb +0 -95
  391. data/lib/datashift/method_dictionary.rb +0 -281
  392. data/lib/datashift/method_mapper.rb +0 -174
  393. data/lib/datashift/thor_base.rb +0 -38
  394. data/lib/generators/mapping_generator.rb +0 -112
  395. data/lib/helpers/core_ext/csv_file.rb +0 -33
  396. data/lib/helpers/core_ext/to_b.rb +0 -24
  397. data/lib/loaders/reporter.rb +0 -58
  398. data/lib/thor/export.thor +0 -175
  399. data/lib/thor/generate.thor +0 -191
  400. data/lib/thor/import.thor +0 -110
  401. data/lib/thor/mapping.thor +0 -65
  402. data/lib/thor/tools.thor +0 -84
  403. data/spec/Gemfile +0 -31
  404. data/spec/Gemfile.lock +0 -134
  405. data/spec/csv_exporter_spec.rb +0 -144
  406. data/spec/csv_generator_spec.rb +0 -159
  407. data/spec/csv_loader_spec.rb +0 -212
  408. data/spec/datashift_spec.rb +0 -55
  409. data/spec/excel_exporter_spec.rb +0 -199
  410. data/spec/excel_generator_spec.rb +0 -203
  411. data/spec/excel_loader_spec.rb +0 -237
  412. data/spec/excel_spec.rb +0 -203
  413. data/spec/loader_base_spec.rb +0 -166
  414. data/spec/mapping_spec.rb +0 -117
  415. data/spec/method_dictionary_spec.rb +0 -300
  416. data/spec/method_mapper_spec.rb +0 -100
  417. data/spec/model_mapper_spec.rb +0 -41
  418. data/spec/paperclip_loader_spec.rb +0 -92
  419. data/spec/populator_spec.rb +0 -128
  420. data/spec/thor_spec.rb +0 -90
  421. data/tasks/file_tasks.rake +0 -37
  422. data/tasks/word_to_seedfu.rake +0 -167
@@ -0,0 +1,11 @@
1
+ # Be sure to restart your server when you modify this file.
2
+
3
+ # Version of your assets, change this if you want to expire all your assets.
4
+ Rails.application.config.assets.version = '1.0'
5
+
6
+ # Add additional assets to the asset load path
7
+ # Rails.application.config.assets.paths << Emoji.images_path
8
+
9
+ # Precompile additional assets.
10
+ # application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
11
+ # Rails.application.config.assets.precompile += %w( search.js )
@@ -0,0 +1,7 @@
1
+ # Be sure to restart your server when you modify this file.
2
+
3
+ # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces.
4
+ # Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ }
5
+
6
+ # You can also remove all the silencers if you're trying to debug a problem that might stem from framework code.
7
+ # Rails.backtrace_cleaner.remove_silencers!
@@ -0,0 +1,3 @@
1
+ # Be sure to restart your server when you modify this file.
2
+
3
+ Rails.application.config.action_dispatch.cookies_serializer = :json
@@ -0,0 +1,4 @@
1
+ # Be sure to restart your server when you modify this file.
2
+
3
+ # Configure sensitive parameters which will be filtered from the log file.
4
+ Rails.application.config.filter_parameters += [:password]
@@ -0,0 +1,16 @@
1
+ # Be sure to restart your server when you modify this file.
2
+
3
+ # Add new inflection rules using the following format. Inflections
4
+ # are locale specific, and you may define rules for as many different
5
+ # locales as you wish. All of these examples are active by default:
6
+ # ActiveSupport::Inflector.inflections(:en) do |inflect|
7
+ # inflect.plural /^(ox)$/i, '\1en'
8
+ # inflect.singular /^(ox)en/i, '\1'
9
+ # inflect.irregular 'person', 'people'
10
+ # inflect.uncountable %w( fish sheep )
11
+ # end
12
+
13
+ # These inflection rules are supported but not enabled by default:
14
+ # ActiveSupport::Inflector.inflections(:en) do |inflect|
15
+ # inflect.acronym 'RESTful'
16
+ # end
@@ -0,0 +1,4 @@
1
+ # Be sure to restart your server when you modify this file.
2
+
3
+ # Add new mime types for use in respond_to blocks:
4
+ # Mime::Type.register "text/richtext", :rtf
@@ -0,0 +1,3 @@
1
+ # Be sure to restart your server when you modify this file.
2
+
3
+ Rails.application.config.session_store :cookie_store, key: '_dummy_session'
@@ -0,0 +1,14 @@
1
+ # Be sure to restart your server when you modify this file.
2
+
3
+ # This file contains settings for ActionController::ParamsWrapper which
4
+ # is enabled by default.
5
+
6
+ # Enable parameter wrapping for JSON. You can disable this by setting :format to an empty array.
7
+ ActiveSupport.on_load(:action_controller) do
8
+ wrap_parameters format: [:json] if respond_to?(:wrap_parameters)
9
+ end
10
+
11
+ # To enable root element in JSON for ActiveRecord objects.
12
+ # ActiveSupport.on_load(:active_record) do
13
+ # self.include_root_in_json = true
14
+ # end
@@ -0,0 +1,23 @@
1
+ # Files in the config/locales directory are used for internationalization
2
+ # and are automatically loaded by Rails. If you want to use locales other
3
+ # than English, add the necessary files in this directory.
4
+ #
5
+ # To use the locales, use `I18n.t`:
6
+ #
7
+ # I18n.t 'hello'
8
+ #
9
+ # In views, this is aliased to just `t`:
10
+ #
11
+ # <%= t('hello') %>
12
+ #
13
+ # To use a different locale, set it with `I18n.locale`:
14
+ #
15
+ # I18n.locale = :es
16
+ #
17
+ # This would use the information in config/locales/es.yml.
18
+ #
19
+ # To learn more, please read the Rails Internationalization guide
20
+ # available at http://guides.rubyonrails.org/i18n.html.
21
+
22
+ en:
23
+ hello: "Hello world"
@@ -0,0 +1,76 @@
1
+ Rails.application.routes.draw do
2
+ resources :categories
3
+ resources :milestones
4
+ resources :users
5
+ resources :versions
6
+ resources :projects
7
+ resources :empties
8
+ resources :owners
9
+ resources :loader_releases
10
+ resources :users
11
+ resources :owners
12
+ resources :categories
13
+ resources :digitals
14
+ resources :long_and_complex_table_linked_to_versions
15
+ resources :milestones
16
+ resources :digitals
17
+ resources :projects
18
+ resources :empties
19
+ resources :loader_releases
20
+ resources :versions
21
+ resources :long_and_complex_table_linked_to_versions
22
+ # The priority is based upon order of creation: first created -> highest priority.
23
+ # See how all your routes lay out with "rake routes".
24
+
25
+ # You can have the root of your site routed with "root"
26
+ # root 'welcome#index'
27
+
28
+ # Example of regular route:
29
+ # get 'products/:id' => 'catalog#view'
30
+
31
+ # Example of named route that can be invoked with purchase_url(id: product.id)
32
+ # get 'products/:id/purchase' => 'catalog#purchase', as: :purchase
33
+
34
+ # Example resource route (maps HTTP verbs to controller actions automatically):
35
+ # resources :products
36
+
37
+ # Example resource route with options:
38
+ # resources :products do
39
+ # member do
40
+ # get 'short'
41
+ # post 'toggle'
42
+ # end
43
+ #
44
+ # collection do
45
+ # get 'sold'
46
+ # end
47
+ # end
48
+
49
+ # Example resource route with sub-resources:
50
+ # resources :products do
51
+ # resources :comments, :sales
52
+ # resource :seller
53
+ # end
54
+
55
+ # Example resource route with more complex sub-resources:
56
+ # resources :products do
57
+ # resources :comments
58
+ # resources :sales do
59
+ # get 'recent', on: :collection
60
+ # end
61
+ # end
62
+
63
+ # Example resource route with concerns:
64
+ # concern :toggleable do
65
+ # post 'toggle'
66
+ # end
67
+ # resources :posts, concerns: :toggleable
68
+ # resources :photos, concerns: :toggleable
69
+
70
+ # Example resource route within a namespace:
71
+ # namespace :admin do
72
+ # # Directs /admin/products/* to Admin::ProductsController
73
+ # # (app/controllers/admin/products_controller.rb)
74
+ # resources :products
75
+ # end
76
+ end
@@ -0,0 +1,22 @@
1
+ # Be sure to restart your server when you modify this file.
2
+
3
+ # Your secret key is used for verifying the integrity of signed cookies.
4
+ # If you change this key, all old signed cookies will become invalid!
5
+
6
+ # Make sure the secret is at least 30 characters and all random,
7
+ # no regular words or you'll be exposed to dictionary attacks.
8
+ # You can use `rake secret` to generate a secure secret key.
9
+
10
+ # Make sure the secrets in this file are kept private
11
+ # if you're sharing your code publicly.
12
+
13
+ development:
14
+ secret_key_base: c1b92ba58af673d0e47f06c9cc4226186aaa9f044bf686b52d19353459e5cbc60180943459c4defb6c9c5a8660b76885d04c66098ab1af96c92b7a2096e08bf7
15
+
16
+ test:
17
+ secret_key_base: f3e155207a7a09721f034045b607c2c8363f01639b041a39721c4f8730c4e110d1255495792e3cc46fdd6f1d2b36835e7502cccd6c188dcff698db73ee87f4c7
18
+
19
+ # Do not keep production secrets in the repository,
20
+ # instead read values from the environment.
21
+ production:
22
+ secret_key_base: <%= ENV["SECRET_KEY_BASE"] %>
@@ -0,0 +1,98 @@
1
+ # Author :: Tom Statter
2
+ # Date :: Aug 2011
3
+ # License:: MIT
4
+ #
5
+ # Details:: Migration to create a test database that exercises all aspects of Active Record models
6
+ #
7
+
8
+ class CreateTestBed < ActiveRecord::Migration
9
+
10
+ def self.up
11
+
12
+ # has many :projects
13
+ create_table :users do |t|
14
+ t.string :title
15
+ t.string :first_name
16
+ end
17
+
18
+ # belongs_to :user
19
+ # has many :milestones
20
+ #
21
+ create_table :projects do |t|
22
+ t.string :title
23
+ t.string :value_as_string
24
+ t.text :value_as_text
25
+ t.boolean :value_as_boolean, default: false
26
+ t.datetime :value_as_datetime, default: nil
27
+ t.integer :value_as_integer, default: 0
28
+ t.decimal :value_as_double, precision: 10, scale: 4, default: 0.0
29
+ t.references :user
30
+ t.timestamps null: false
31
+ end
32
+
33
+ # belongs_to :project
34
+ # @project => has_many :milestones
35
+
36
+ create_table :milestones do |t|
37
+ t.string :name
38
+ t.datetime :datetime, default: nil
39
+ t.decimal :cost, precision: 8, scale: 2, default: 0.0
40
+ t.references :project
41
+ t.timestamps null: false
42
+ end
43
+
44
+ # belongs_to :project, project => has_one
45
+ create_table :owners do |t|
46
+ t.string :name
47
+ t.decimal :budget
48
+ t.references :project
49
+ t.timestamps null: false
50
+ end
51
+
52
+ # has_belongs_to_many :project
53
+ create_table :categories do |t|
54
+ t.string :reference
55
+ t.timestamps null: false
56
+ end
57
+
58
+ # testing has_belongs_to_many (hence no id)
59
+ create_table :categories_projects, id: false do |t|
60
+ t.references :category
61
+ t.references :project
62
+ end
63
+
64
+ create_table :versions do |t|
65
+ t.string :name
66
+ t.timestamps null: false
67
+ end
68
+
69
+ # testing project has_many release + versions :through
70
+ create_table :loader_releases do |t|
71
+ t.string :name
72
+ t.references :project
73
+ t.references :version
74
+ t.timestamps null: false
75
+ end
76
+
77
+ create_table :long_and_complex_table_linked_to_versions do |t|
78
+ t.decimal :price
79
+ t.references :version
80
+ end
81
+
82
+ create_table :empties do |_t|
83
+ end
84
+
85
+ end
86
+
87
+ def self.down
88
+ drop_table :users
89
+ drop_table :projects
90
+ drop_table :categories
91
+ drop_table :loader_releases
92
+ drop_table :versions
93
+ drop_table :categories_projectss
94
+ drop_table :milestones
95
+ drop_table :long_and_complex_table_linked_to_versions
96
+ drop_table :empties
97
+ end
98
+ end
@@ -0,0 +1,24 @@
1
+ # Author :: Tom Statter
2
+ # Date :: Oct 2012
3
+ # License:: MIT
4
+ #
5
+ # Details:: Migration for paperclip specs
6
+
7
+ class AddDigitals < ActiveRecord::Migration
8
+
9
+ def self.up
10
+
11
+ create_table :digitals do |t|
12
+ t.integer :owner_id
13
+ t.string :attachment_file_name
14
+ t.string :attachment_content_type
15
+ t.integer :attachment_file_size
16
+ t.timestamps null: false
17
+ end
18
+
19
+ end
20
+
21
+ def self.down
22
+ drop_table :digitals
23
+ end
24
+ end
@@ -0,0 +1,8 @@
1
+ class CreateDigitals < ActiveRecord::Migration
2
+ def change
3
+ create_table :digitals do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ class CreateLongAndComplexTableLinkedToVersions < ActiveRecord::Migration
2
+ def change
3
+ create_table :long_and_complex_table_linked_to_versions do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ class CreateLoaderReleases < ActiveRecord::Migration
2
+ def change
3
+ create_table :loader_releases do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ class CreateOwners < ActiveRecord::Migration
2
+ def change
3
+ create_table :owners do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ class CreateEmpties < ActiveRecord::Migration
2
+ def change
3
+ create_table :empties do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ class CreateProjects < ActiveRecord::Migration
2
+ def change
3
+ create_table :projects do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ class CreateUsers < ActiveRecord::Migration
2
+ def change
3
+ create_table :users do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ class CreateVersions < ActiveRecord::Migration
2
+ def change
3
+ create_table :versions do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ class CreateMilestones < ActiveRecord::Migration
2
+ def change
3
+ create_table :milestones do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,8 @@
1
+ class CreateCategories < ActiveRecord::Migration
2
+ def change
3
+ create_table :categories do |t|
4
+
5
+ t.timestamps null: false
6
+ end
7
+ end
8
+ end
@@ -0,0 +1,93 @@
1
+ # encoding: UTF-8
2
+ # This file is auto-generated from the current state of the database. Instead
3
+ # of editing this file, please use the migrations feature of Active Record to
4
+ # incrementally modify your database, and then regenerate this schema definition.
5
+ #
6
+ # Note that this schema.rb definition is the authoritative source for your
7
+ # database schema. If you need to create the application database on another
8
+ # system, you should be using db:schema:load, not running all the migrations
9
+ # from scratch. The latter is a flawed and unsustainable approach (the more migrations
10
+ # you'll amass, the slower it'll run and the greater likelihood for issues).
11
+ #
12
+ # It's strongly recommended that you check this file into your version control system.
13
+
14
+ ActiveRecord::Schema.define(version: 20121009161700) do
15
+
16
+ create_table "categories", force: :cascade do |t|
17
+ t.string "reference"
18
+ t.datetime "created_at", null: false
19
+ t.datetime "updated_at", null: false
20
+ end
21
+
22
+ create_table "categories_projects", id: false, force: :cascade do |t|
23
+ t.integer "category_id"
24
+ t.integer "project_id"
25
+ end
26
+
27
+ create_table "digitals", force: :cascade do |t|
28
+ t.integer "owner_id"
29
+ t.string "attachment_file_name"
30
+ t.string "attachment_content_type"
31
+ t.integer "attachment_file_size"
32
+ t.datetime "created_at", null: false
33
+ t.datetime "updated_at", null: false
34
+ end
35
+
36
+ create_table "empties", force: :cascade do |t|
37
+ end
38
+
39
+ create_table "loader_releases", force: :cascade do |t|
40
+ t.string "name"
41
+ t.integer "project_id"
42
+ t.integer "version_id"
43
+ t.datetime "created_at", null: false
44
+ t.datetime "updated_at", null: false
45
+ end
46
+
47
+ create_table "long_and_complex_table_linked_to_versions", force: :cascade do |t|
48
+ t.decimal "price"
49
+ t.integer "version_id"
50
+ end
51
+
52
+ create_table "milestones", force: :cascade do |t|
53
+ t.string "name"
54
+ t.datetime "datetime"
55
+ t.decimal "cost", precision: 8, scale: 2, default: 0.0
56
+ t.integer "project_id"
57
+ t.datetime "created_at", null: false
58
+ t.datetime "updated_at", null: false
59
+ end
60
+
61
+ create_table "owners", force: :cascade do |t|
62
+ t.string "name"
63
+ t.decimal "budget"
64
+ t.integer "project_id"
65
+ t.datetime "created_at", null: false
66
+ t.datetime "updated_at", null: false
67
+ end
68
+
69
+ create_table "projects", force: :cascade do |t|
70
+ t.string "title"
71
+ t.string "value_as_string"
72
+ t.text "value_as_text"
73
+ t.boolean "value_as_boolean", default: false
74
+ t.datetime "value_as_datetime"
75
+ t.integer "value_as_integer", default: 0
76
+ t.decimal "value_as_double", precision: 10, scale: 4, default: 0.0
77
+ t.integer "user_id"
78
+ t.datetime "created_at", null: false
79
+ t.datetime "updated_at", null: false
80
+ end
81
+
82
+ create_table "users", force: :cascade do |t|
83
+ t.string "title"
84
+ t.string "first_name"
85
+ end
86
+
87
+ create_table "versions", force: :cascade do |t|
88
+ t.string "name"
89
+ t.datetime "created_at", null: false
90
+ t.datetime "updated_at", null: false
91
+ end
92
+
93
+ end