datashift 0.16.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (422) hide show
  1. checksums.yaml +4 -4
  2. data/{LICENSE.txt → LICENSE} +0 -0
  3. data/Rakefile +1 -20
  4. data/datashift.thor +125 -0
  5. data/lib/applications/apache_poi_extensions.rb +21 -52
  6. data/lib/applications/excel.rb +64 -57
  7. data/lib/applications/hssf_row_extensions.rb +66 -0
  8. data/lib/applications/jexcel_file.rb +99 -95
  9. data/lib/applications/jexcel_file_extensions.rb +76 -83
  10. data/lib/applications/jruby/word.rb +36 -36
  11. data/lib/applications/ruby_poi_translations.rb +34 -32
  12. data/lib/applications/spreadsheet_extensions.rb +21 -19
  13. data/lib/datashift.rb +49 -59
  14. data/lib/datashift/binder.rb +217 -0
  15. data/lib/datashift/column_packer.rb +21 -72
  16. data/lib/datashift/configuration.rb +317 -0
  17. data/lib/datashift/context_factory.rb +88 -0
  18. data/lib/datashift/core_ext/array.rb +15 -0
  19. data/lib/datashift/core_ext/csv_ext.rb +46 -0
  20. data/lib/datashift/core_ext/string.rb +49 -0
  21. data/lib/datashift/core_ext/to_b.rb +11 -0
  22. data/lib/datashift/delimiters.rb +55 -61
  23. data/lib/datashift/doc_context.rb +137 -0
  24. data/lib/datashift/excel_base.rb +93 -81
  25. data/lib/datashift/exceptions.rb +30 -28
  26. data/lib/datashift/file_definitions.rb +44 -39
  27. data/lib/datashift/guards.rb +5 -5
  28. data/lib/datashift/header.rb +25 -0
  29. data/lib/datashift/headers.rb +94 -0
  30. data/lib/datashift/inbound_data/column.rb +44 -0
  31. data/lib/datashift/inbound_data/lookup_support.rb +33 -0
  32. data/lib/datashift/inbound_data/method_binding.rb +139 -0
  33. data/lib/datashift/load_object.rb +37 -12
  34. data/lib/datashift/logging.rb +54 -27
  35. data/lib/datashift/mandatory.rb +39 -0
  36. data/lib/datashift/mapping/data_flow_schema.rb +198 -0
  37. data/lib/datashift/{model_mapper.rb → mapping/mapper_utils.rb} +30 -10
  38. data/lib/datashift/model_methods/catalogue.rb +183 -0
  39. data/lib/datashift/model_methods/collection.rb +140 -0
  40. data/lib/datashift/model_methods/model_method.rb +162 -0
  41. data/lib/datashift/model_methods/model_methods_manager.rb +76 -0
  42. data/lib/datashift/model_methods/operator.rb +62 -0
  43. data/lib/datashift/node_collection.rb +26 -0
  44. data/lib/datashift/node_context.rb +68 -0
  45. data/lib/datashift/populator.rb +308 -282
  46. data/lib/datashift/progress_monitor.rb +91 -0
  47. data/lib/datashift/querying.rb +110 -52
  48. data/lib/datashift/templates/import_export_config.erb +55 -0
  49. data/lib/datashift/transformation/factory.rb +219 -0
  50. data/lib/datashift/transformation/remove.rb +44 -0
  51. data/lib/datashift/version.rb +3 -0
  52. data/lib/exporters/configuration.rb +84 -0
  53. data/lib/exporters/csv_exporter.rb +54 -52
  54. data/lib/exporters/excel_exporter.rb +80 -61
  55. data/lib/exporters/exporter_base.rb +8 -8
  56. data/lib/generators/config_generator.rb +218 -0
  57. data/lib/generators/csv_generator.rb +13 -70
  58. data/lib/generators/excel_generator.rb +23 -111
  59. data/lib/generators/generator_base.rb +15 -70
  60. data/lib/loaders/configuration.rb +90 -0
  61. data/lib/loaders/csv_loader.rb +63 -101
  62. data/lib/loaders/excel_loader.rb +71 -156
  63. data/lib/loaders/failure_data.rb +40 -0
  64. data/lib/loaders/file_loader.rb +16 -0
  65. data/lib/loaders/loader_base.rb +82 -410
  66. data/lib/loaders/loader_factory.rb +42 -0
  67. data/lib/loaders/paperclip/attachment_loader.rb +157 -140
  68. data/lib/loaders/paperclip/datashift_paperclip.rb +18 -35
  69. data/lib/loaders/paperclip/image_loading.rb +40 -35
  70. data/lib/loaders/reporters/basic_stdout_reporter.rb +40 -0
  71. data/lib/loaders/reporters/reporter.rb +26 -0
  72. data/lib/tasks/config.thor +65 -0
  73. data/{tasks → lib/tasks}/config/seed_fu_product_template.erb +0 -0
  74. data/{tasks → lib/tasks}/config/tidy_config.txt +0 -0
  75. data/lib/tasks/export.thor +192 -0
  76. data/lib/tasks/generate.thor +190 -0
  77. data/lib/tasks/import.thor +142 -0
  78. data/lib/{thor → tasks}/paperclip.thor +69 -69
  79. data/{tasks → lib/tasks/to_convert_to_thor}/db_tasks.rake +20 -20
  80. data/lib/tasks/tools.thor +109 -0
  81. data/spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg +0 -0
  82. data/spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg +0 -0
  83. data/spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg +0 -0
  84. data/spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg +0 -0
  85. data/spec/datashift/binder_spec.rb +266 -0
  86. data/spec/datashift/config_generator_spec.rb +186 -0
  87. data/spec/datashift/configuration.rb +66 -0
  88. data/spec/datashift/context_factory_spec.rb +63 -0
  89. data/spec/datashift/data_flow_schema_spec.rb +150 -0
  90. data/spec/datashift/datashift_spec.rb +52 -0
  91. data/spec/datashift/excel_base_spec.rb +57 -0
  92. data/spec/datashift/excel_spec.rb +188 -0
  93. data/spec/datashift/failure_data_spec.rb +27 -0
  94. data/spec/{file_definitions.rb → datashift/file_definitions.rb} +9 -10
  95. data/spec/datashift/headers_spec.rb +56 -0
  96. data/spec/datashift/inbound_data_spec.rb +47 -0
  97. data/spec/datashift/mapper_utils_spec.rb +38 -0
  98. data/spec/datashift/method_binding_spec.rb +60 -0
  99. data/spec/datashift/model_method_spec.rb +109 -0
  100. data/spec/datashift/model_methods_catalogue.rb +111 -0
  101. data/spec/datashift/model_methods_collection_spec.rb +138 -0
  102. data/spec/datashift/model_methods_manager_spec.rb +329 -0
  103. data/spec/datashift/populator_spec.rb +117 -0
  104. data/spec/datashift/thor_spec.rb +314 -0
  105. data/spec/datashift/transformation/factory_spec.rb +195 -0
  106. data/spec/datashift/transformation/transformer_remove_spec.rb +43 -0
  107. data/spec/dummy/Gemfile +53 -0
  108. data/spec/dummy/Gemfile.lock +197 -0
  109. data/spec/dummy/README.rdoc +28 -0
  110. data/spec/dummy/Rakefile +6 -0
  111. data/spec/dummy/app/assets/javascripts/application.js +16 -0
  112. data/spec/dummy/app/assets/javascripts/categories.js +2 -0
  113. data/spec/dummy/app/assets/javascripts/digitals.js +2 -0
  114. data/spec/dummy/app/assets/javascripts/empties.js +2 -0
  115. data/spec/dummy/app/assets/javascripts/loader_releases.js +2 -0
  116. data/spec/dummy/app/assets/javascripts/long_and_complex_table_linked_to_versions.js +2 -0
  117. data/spec/dummy/app/assets/javascripts/milestones.js +2 -0
  118. data/spec/dummy/app/assets/javascripts/owners.js +2 -0
  119. data/spec/dummy/app/assets/javascripts/projects.js +2 -0
  120. data/spec/dummy/app/assets/javascripts/users.js +2 -0
  121. data/spec/dummy/app/assets/javascripts/versions.js +2 -0
  122. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  123. data/spec/dummy/app/assets/stylesheets/categories.css +4 -0
  124. data/spec/dummy/app/assets/stylesheets/digitals.css +4 -0
  125. data/spec/dummy/app/assets/stylesheets/empties.css +4 -0
  126. data/spec/dummy/app/assets/stylesheets/loader_releases.css +4 -0
  127. data/spec/dummy/app/assets/stylesheets/long_and_complex_table_linked_to_versions.css +4 -0
  128. data/spec/dummy/app/assets/stylesheets/milestones.css +4 -0
  129. data/spec/dummy/app/assets/stylesheets/owners.css +4 -0
  130. data/spec/dummy/app/assets/stylesheets/projects.css +4 -0
  131. data/spec/dummy/app/assets/stylesheets/scaffold.css +56 -0
  132. data/spec/dummy/app/assets/stylesheets/users.css +4 -0
  133. data/spec/dummy/app/assets/stylesheets/versions.css +4 -0
  134. data/spec/dummy/app/controllers/application_controller.rb +5 -0
  135. data/spec/dummy/app/controllers/categories_controller.rb +58 -0
  136. data/spec/dummy/app/controllers/digitals_controller.rb +58 -0
  137. data/spec/dummy/app/controllers/empties_controller.rb +58 -0
  138. data/spec/dummy/app/controllers/loader_releases_controller.rb +58 -0
  139. data/spec/dummy/app/controllers/long_and_complex_table_linked_to_versions_controller.rb +58 -0
  140. data/spec/dummy/app/controllers/milestones_controller.rb +58 -0
  141. data/spec/dummy/app/controllers/owners_controller.rb +58 -0
  142. data/spec/dummy/app/controllers/projects_controller.rb +58 -0
  143. data/spec/dummy/app/controllers/users_controller.rb +58 -0
  144. data/spec/dummy/app/controllers/versions_controller.rb +58 -0
  145. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  146. data/spec/dummy/app/helpers/categories_helper.rb +2 -0
  147. data/spec/dummy/app/helpers/digitals_helper.rb +2 -0
  148. data/spec/dummy/app/helpers/empties_helper.rb +2 -0
  149. data/spec/dummy/app/helpers/loader_releases_helper.rb +2 -0
  150. data/spec/dummy/app/helpers/long_and_complex_table_linked_to_versions_helper.rb +2 -0
  151. data/spec/dummy/app/helpers/milestones_helper.rb +2 -0
  152. data/spec/dummy/app/helpers/owners_helper.rb +2 -0
  153. data/spec/dummy/app/helpers/projects_helper.rb +2 -0
  154. data/spec/dummy/app/helpers/users_helper.rb +2 -0
  155. data/spec/dummy/app/helpers/versions_helper.rb +2 -0
  156. data/spec/dummy/app/models/category.rb +6 -0
  157. data/spec/dummy/app/models/digital.rb +22 -0
  158. data/spec/dummy/app/models/empty.rb +2 -0
  159. data/spec/dummy/app/models/loader_release.rb +10 -0
  160. data/spec/dummy/app/models/long_and_complex_table_linked_to_version.rb +6 -0
  161. data/spec/dummy/app/models/milestone.rb +15 -0
  162. data/spec/dummy/app/models/owner.rb +13 -0
  163. data/spec/dummy/app/models/project.rb +53 -0
  164. data/spec/dummy/app/models/user.rb +5 -0
  165. data/spec/dummy/app/models/version.rb +7 -0
  166. data/spec/dummy/app/views/categories/_form.html.erb +17 -0
  167. data/spec/dummy/app/views/categories/edit.html.erb +6 -0
  168. data/spec/dummy/app/views/categories/index.html.erb +25 -0
  169. data/spec/dummy/app/views/categories/new.html.erb +5 -0
  170. data/spec/dummy/app/views/categories/show.html.erb +4 -0
  171. data/spec/dummy/app/views/digitals/_form.html.erb +17 -0
  172. data/spec/dummy/app/views/digitals/edit.html.erb +6 -0
  173. data/spec/dummy/app/views/digitals/index.html.erb +25 -0
  174. data/spec/dummy/app/views/digitals/new.html.erb +5 -0
  175. data/spec/dummy/app/views/digitals/show.html.erb +4 -0
  176. data/spec/dummy/app/views/empties/_form.html.erb +17 -0
  177. data/spec/dummy/app/views/empties/edit.html.erb +6 -0
  178. data/spec/dummy/app/views/empties/index.html.erb +25 -0
  179. data/spec/dummy/app/views/empties/new.html.erb +5 -0
  180. data/spec/dummy/app/views/empties/show.html.erb +4 -0
  181. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  182. data/spec/dummy/app/views/loader_releases/_form.html.erb +17 -0
  183. data/spec/dummy/app/views/loader_releases/edit.html.erb +6 -0
  184. data/spec/dummy/app/views/loader_releases/index.html.erb +25 -0
  185. data/spec/dummy/app/views/loader_releases/new.html.erb +5 -0
  186. data/spec/dummy/app/views/loader_releases/show.html.erb +4 -0
  187. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/_form.html.erb +17 -0
  188. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/edit.html.erb +6 -0
  189. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/index.html.erb +25 -0
  190. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/new.html.erb +5 -0
  191. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/show.html.erb +4 -0
  192. data/spec/dummy/app/views/milestones/_form.html.erb +17 -0
  193. data/spec/dummy/app/views/milestones/edit.html.erb +6 -0
  194. data/spec/dummy/app/views/milestones/index.html.erb +25 -0
  195. data/spec/dummy/app/views/milestones/new.html.erb +5 -0
  196. data/spec/dummy/app/views/milestones/show.html.erb +4 -0
  197. data/spec/dummy/app/views/owners/_form.html.erb +17 -0
  198. data/spec/dummy/app/views/owners/edit.html.erb +6 -0
  199. data/spec/dummy/app/views/owners/index.html.erb +25 -0
  200. data/spec/dummy/app/views/owners/new.html.erb +5 -0
  201. data/spec/dummy/app/views/owners/show.html.erb +4 -0
  202. data/spec/dummy/app/views/projects/_form.html.erb +17 -0
  203. data/spec/dummy/app/views/projects/edit.html.erb +6 -0
  204. data/spec/dummy/app/views/projects/index.html.erb +25 -0
  205. data/spec/dummy/app/views/projects/new.html.erb +5 -0
  206. data/spec/dummy/app/views/projects/show.html.erb +4 -0
  207. data/spec/dummy/app/views/users/_form.html.erb +17 -0
  208. data/spec/dummy/app/views/users/edit.html.erb +6 -0
  209. data/spec/dummy/app/views/users/index.html.erb +25 -0
  210. data/spec/dummy/app/views/users/new.html.erb +5 -0
  211. data/spec/dummy/app/views/users/show.html.erb +4 -0
  212. data/spec/dummy/app/views/versions/_form.html.erb +17 -0
  213. data/spec/dummy/app/views/versions/edit.html.erb +6 -0
  214. data/spec/dummy/app/views/versions/index.html.erb +25 -0
  215. data/spec/dummy/app/views/versions/new.html.erb +5 -0
  216. data/spec/dummy/app/views/versions/show.html.erb +4 -0
  217. data/spec/dummy/bin/bundle +3 -0
  218. data/spec/dummy/bin/rails +9 -0
  219. data/spec/dummy/bin/rake +9 -0
  220. data/spec/dummy/bin/setup +29 -0
  221. data/spec/dummy/bin/spring +16 -0
  222. data/spec/dummy/config.ru +4 -0
  223. data/spec/dummy/config/application.rb +26 -0
  224. data/spec/dummy/config/boot.rb +3 -0
  225. data/spec/dummy/config/database.yml +25 -0
  226. data/spec/dummy/config/environment.rb +5 -0
  227. data/spec/dummy/config/environments/development.rb +41 -0
  228. data/spec/dummy/config/environments/production.rb +79 -0
  229. data/spec/dummy/config/environments/test.rb +42 -0
  230. data/spec/dummy/config/initializers/assets.rb +11 -0
  231. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  232. data/spec/dummy/config/initializers/cookies_serializer.rb +3 -0
  233. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  234. data/spec/dummy/config/initializers/inflections.rb +16 -0
  235. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  236. data/spec/dummy/config/initializers/session_store.rb +3 -0
  237. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  238. data/spec/dummy/config/locales/en.yml +23 -0
  239. data/spec/dummy/config/routes.rb +76 -0
  240. data/spec/dummy/config/secrets.yml +22 -0
  241. data/spec/dummy/db/development.sqlite3 +0 -0
  242. data/spec/dummy/db/migrate/20110803201325_create_test_bed.rb +98 -0
  243. data/spec/dummy/db/migrate/20121009161700_add_digitals.rb +24 -0
  244. data/spec/dummy/db/migrate/20161005123106_create_digitals.rb +8 -0
  245. data/spec/dummy/db/migrate/20161005123106_create_long_and_complex_table_linked_to_versions.rb +8 -0
  246. data/spec/dummy/db/migrate/20161005123107_create_loader_releases.rb +8 -0
  247. data/spec/dummy/db/migrate/20161005123108_create_owners.rb +8 -0
  248. data/spec/dummy/db/migrate/20161005123109_create_empties.rb +8 -0
  249. data/spec/dummy/db/migrate/20161005123110_create_projects.rb +8 -0
  250. data/spec/dummy/db/migrate/20161005123111_create_users.rb +8 -0
  251. data/spec/dummy/db/migrate/20161005123111_create_versions.rb +8 -0
  252. data/spec/dummy/db/migrate/20161005123112_create_milestones.rb +8 -0
  253. data/spec/dummy/db/migrate/20161005123113_create_categories.rb +8 -0
  254. data/spec/dummy/db/schema.rb +93 -0
  255. data/spec/dummy/db/seeds.rb +9 -0
  256. data/spec/dummy/db/test.sqlite3 +0 -0
  257. data/spec/dummy/log/test.log +69 -0
  258. data/spec/dummy/public/404.html +67 -0
  259. data/spec/dummy/public/422.html +67 -0
  260. data/spec/dummy/public/500.html +66 -0
  261. data/spec/dummy/public/favicon.ico +0 -0
  262. data/spec/dummy/public/robots.txt +5 -0
  263. data/spec/dummy/sandbox_example.thor +4 -0
  264. data/spec/dummy/test/controllers/categories_controller_test.rb +49 -0
  265. data/spec/dummy/test/controllers/digitals_controller_test.rb +49 -0
  266. data/spec/dummy/test/controllers/empties_controller_test.rb +49 -0
  267. data/spec/dummy/test/controllers/loader_releases_controller_test.rb +49 -0
  268. data/spec/dummy/test/controllers/long_and_complex_table_linked_to_versions_controller_test.rb +49 -0
  269. data/spec/dummy/test/controllers/milestones_controller_test.rb +49 -0
  270. data/spec/dummy/test/controllers/owners_controller_test.rb +49 -0
  271. data/spec/dummy/test/controllers/projects_controller_test.rb +49 -0
  272. data/spec/dummy/test/controllers/users_controller_test.rb +49 -0
  273. data/spec/dummy/test/controllers/versions_controller_test.rb +49 -0
  274. data/spec/dummy/test/factories/categories.rb +5 -0
  275. data/spec/dummy/test/factories/digitals.rb +5 -0
  276. data/spec/dummy/test/factories/empties.rb +5 -0
  277. data/spec/dummy/test/factories/loader_releases.rb +5 -0
  278. data/spec/dummy/test/factories/long_and_complex_table_linked_to_versions.rb +5 -0
  279. data/spec/dummy/test/factories/milestones.rb +5 -0
  280. data/spec/dummy/test/factories/owners.rb +5 -0
  281. data/spec/dummy/test/factories/projects.rb +5 -0
  282. data/spec/dummy/test/factories/users.rb +5 -0
  283. data/spec/dummy/test/factories/versions.rb +5 -0
  284. data/spec/dummy/test/models/category_test.rb +7 -0
  285. data/spec/dummy/test/models/digital_test.rb +7 -0
  286. data/spec/dummy/test/models/empty_test.rb +7 -0
  287. data/spec/dummy/test/models/loader_release_test.rb +7 -0
  288. data/spec/dummy/test/models/long_and_complex_table_linked_to_version_test.rb +7 -0
  289. data/spec/dummy/test/models/milestone_test.rb +7 -0
  290. data/spec/dummy/test/models/owner_test.rb +7 -0
  291. data/spec/dummy/test/models/project_test.rb +7 -0
  292. data/spec/dummy/test/models/user_test.rb +7 -0
  293. data/spec/dummy/test/models/version_test.rb +7 -0
  294. data/spec/dummy/test/test_helper.rb +10 -0
  295. data/spec/exporters/csv_exporter_spec.rb +240 -0
  296. data/spec/exporters/csv_generator_spec.rb +139 -0
  297. data/spec/exporters/excel_exporter_spec.rb +193 -0
  298. data/spec/exporters/excel_generator_spec.rb +181 -0
  299. data/spec/exporters/generator_base_spec.rb +45 -0
  300. data/spec/factories/categories.rb +7 -0
  301. data/spec/factories/factories.rb +18 -0
  302. data/spec/factories/milestone.rb +16 -0
  303. data/spec/factories/projects.rb +41 -0
  304. data/spec/fixtures/BadAssociationName.xls +0 -0
  305. data/spec/fixtures/DemoNegativeTesting.xls +0 -0
  306. data/spec/fixtures/ProjectConfiguration.yml +18 -0
  307. data/spec/fixtures/ProjectsMultiCategories.xls +0 -0
  308. data/spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls +0 -0
  309. data/spec/fixtures/ProjectsSingleCategories.xls +0 -0
  310. data/spec/fixtures/ProjectsSingleCategories.xlsx +0 -0
  311. data/spec/fixtures/SimpleProjects.xls +0 -0
  312. data/spec/fixtures/config/database.yml +28 -0
  313. data/spec/fixtures/csv/BadAssociationName.csv +6 -0
  314. data/spec/fixtures/csv/DemoNegativeTesting.csv +6 -0
  315. data/spec/fixtures/csv/ProjectsMultiCategories.csv +5 -0
  316. data/spec/fixtures/csv/ProjectsMultiCategoriesHeaderLookup.csv +5 -0
  317. data/spec/fixtures/csv/ProjectsSingleCategories.csv +5 -0
  318. data/spec/fixtures/csv/SimpleProjects.csv +4 -0
  319. data/spec/fixtures/db/migrate/20110803201325_create_test_bed.rb +98 -0
  320. data/spec/fixtures/db/migrate/20121009161700_add_digitals.rb +24 -0
  321. data/spec/fixtures/db/seeds.rb +9 -0
  322. data/spec/fixtures/images/DEMO_001_ror_bag.jpeg +0 -0
  323. data/spec/fixtures/images/DEMO_002_Powerstation.jpeg +0 -0
  324. data/spec/fixtures/images/DEMO_003_ror_mug.jpeg +0 -0
  325. data/spec/fixtures/images/DEMO_004_ror_ringer.jpeg +0 -0
  326. data/spec/fixtures/load_datashift.thor +3 -0
  327. data/spec/fixtures/models/category.rb +6 -0
  328. data/spec/fixtures/models/digital.rb +22 -0
  329. data/spec/fixtures/models/empty.rb +2 -0
  330. data/spec/fixtures/models/loader_release.rb +10 -0
  331. data/spec/fixtures/models/long_and_complex_table_linked_to_version.rb +6 -0
  332. data/spec/fixtures/models/milestone.rb +15 -0
  333. data/spec/fixtures/models/owner.rb +13 -0
  334. data/spec/fixtures/models/project.rb +53 -0
  335. data/spec/fixtures/models/user.rb +5 -0
  336. data/spec/fixtures/models/version.rb +7 -0
  337. data/spec/fixtures/results/exp_project_assoc_headers.xls +0 -0
  338. data/spec/fixtures/results/exp_project_collection_spec.csv +2 -0
  339. data/spec/fixtures/results/exp_project_export.xls +0 -0
  340. data/spec/fixtures/results/exp_project_first_export.xls +0 -0
  341. data/spec/fixtures/results/exp_project_plus_assoc.xls +0 -0
  342. data/spec/fixtures/results/exp_project_plus_assoc_export_spec.csv +9 -0
  343. data/spec/fixtures/results/gen_project_plus_assoc_template.xls +0 -0
  344. data/spec/fixtures/results/gen_project_plus_some_assoc_template.xls +0 -0
  345. data/spec/fixtures/results/gen_project_template.xls +0 -0
  346. data/spec/fixtures/results/project_and_assoc_in_hash_export.xls +0 -0
  347. data/spec/fixtures/results/project_and_assoc_in_json_export.csv +9 -0
  348. data/spec/fixtures/results/project_and_assoc_in_json_export.xls +0 -0
  349. data/spec/fixtures/results/project_export_spec_with_custom_delim_,.csv +2 -0
  350. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/243.csv +2 -0
  351. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/247.csv +2 -0
  352. data/spec/fixtures/results/project_plus_assoc_template.csv +1 -0
  353. data/spec/fixtures/results/project_plus_some_assoc_template.csv +1 -0
  354. data/spec/fixtures/results/project_remove_export_spec.csv +2 -0
  355. data/spec/fixtures/results/project_template.csv +1 -0
  356. data/spec/fixtures/results/project_with_methods_export_spec.csv +2 -0
  357. data/spec/fixtures/results/thor_spec_gen_project.csv +1 -0
  358. data/spec/fixtures/sandbox_example.thor +4 -0
  359. data/spec/fixtures/simple_export_spec.xls +0 -0
  360. data/spec/fixtures/simple_template_spec.xls +0 -0
  361. data/spec/fixtures/test_model_defs.rb +7 -0
  362. data/spec/loaders/csv_loader_spec.rb +206 -0
  363. data/spec/loaders/data_flow_excel_loader_spec.rb +290 -0
  364. data/spec/loaders/excel_loader_failures_spec.rb +67 -0
  365. data/spec/loaders/excel_loader_spec.rb +294 -0
  366. data/spec/loaders/loader_base_spec.rb +29 -0
  367. data/spec/loaders/paperclip_loader_spec.rb +106 -0
  368. data/spec/log/datashift.log +14930 -0
  369. data/spec/private/digitals/1/DEMO_003_ror_mug.jpeg +0 -0
  370. data/spec/private/digitals/2/DEMO_002_Powerstation.jpeg +0 -0
  371. data/spec/private/digitals/3/DEMO_004_ror_ringer.jpeg +0 -0
  372. data/spec/private/digitals/4/DEMO_001_ror_bag.jpeg +0 -0
  373. data/spec/spec_helper.rb +26 -230
  374. data/spec/support/clear_and_manage_contexts.rb +25 -0
  375. data/spec/support/database_cleaner.rb +32 -0
  376. data/spec/support/datashift_test_helpers.rb +153 -0
  377. data/spec/support/files_paths_helper.rb +13 -0
  378. data/spec/support/fixtures/results/mapping_template.yaml +15 -0
  379. data/spec/support/sandbox.rb +136 -0
  380. metadata +804 -85
  381. data/README.markdown +0 -274
  382. data/README.rdoc +0 -19
  383. data/VERSION +0 -1
  384. data/datashift.gemspec +0 -48
  385. data/lib/applications/jruby/old_pre_proxy_jexcel_file.rb +0 -437
  386. data/lib/datashift/data_transforms.rb +0 -83
  387. data/lib/datashift/mapping_file_definitions.rb +0 -88
  388. data/lib/datashift/mapping_service.rb +0 -91
  389. data/lib/datashift/method_detail.rb +0 -165
  390. data/lib/datashift/method_details_manager.rb +0 -95
  391. data/lib/datashift/method_dictionary.rb +0 -281
  392. data/lib/datashift/method_mapper.rb +0 -174
  393. data/lib/datashift/thor_base.rb +0 -38
  394. data/lib/generators/mapping_generator.rb +0 -112
  395. data/lib/helpers/core_ext/csv_file.rb +0 -33
  396. data/lib/helpers/core_ext/to_b.rb +0 -24
  397. data/lib/loaders/reporter.rb +0 -58
  398. data/lib/thor/export.thor +0 -175
  399. data/lib/thor/generate.thor +0 -191
  400. data/lib/thor/import.thor +0 -110
  401. data/lib/thor/mapping.thor +0 -65
  402. data/lib/thor/tools.thor +0 -84
  403. data/spec/Gemfile +0 -31
  404. data/spec/Gemfile.lock +0 -134
  405. data/spec/csv_exporter_spec.rb +0 -144
  406. data/spec/csv_generator_spec.rb +0 -159
  407. data/spec/csv_loader_spec.rb +0 -212
  408. data/spec/datashift_spec.rb +0 -55
  409. data/spec/excel_exporter_spec.rb +0 -199
  410. data/spec/excel_generator_spec.rb +0 -203
  411. data/spec/excel_loader_spec.rb +0 -237
  412. data/spec/excel_spec.rb +0 -203
  413. data/spec/loader_base_spec.rb +0 -166
  414. data/spec/mapping_spec.rb +0 -117
  415. data/spec/method_dictionary_spec.rb +0 -300
  416. data/spec/method_mapper_spec.rb +0 -100
  417. data/spec/model_mapper_spec.rb +0 -41
  418. data/spec/paperclip_loader_spec.rb +0 -92
  419. data/spec/populator_spec.rb +0 -128
  420. data/spec/thor_spec.rb +0 -90
  421. data/tasks/file_tasks.rake +0 -37
  422. data/tasks/word_to_seedfu.rake +0 -167
@@ -0,0 +1,41 @@
1
+
2
+ FactoryGirl.define do
3
+
4
+ factory :project do
5
+
6
+ sequence(:title) { |n| "Title #{n}" }
7
+
8
+ value_as_text "some text\n<storing>nonsense</storing>"
9
+ value_as_string "this is a string"
10
+ value_as_boolean true
11
+ value_as_double 2.356
12
+ value_as_datetime Time.now
13
+ value_as_integer 23
14
+
15
+ association :user, factory: :user
16
+
17
+ factory :project_with_user do
18
+ user
19
+ owner
20
+
21
+ factory :project_with_milestones do
22
+
23
+ # milestone_count is declared as a transient attribute and available in
24
+ # attributes on the factory, as well as the callback via the evaluator
25
+ transient do
26
+ milestones_count 5
27
+ end
28
+
29
+ # the after(:create) yields two values; the project instance itself and the
30
+ # evaluator, which stores all values from the factory, including transient
31
+ # attributes; `create_list`'s second argument is the number of records
32
+ # to create and we make sure the project is associated properly to the milestone
33
+ after(:create) do |project, evaluator|
34
+ create_list(:milestone_with_project, evaluator.milestones_count, project: project)
35
+ end
36
+ end
37
+ end
38
+
39
+ end
40
+
41
+ end
@@ -0,0 +1,18 @@
1
+ Project:
2
+ defaults:
3
+ value_as_string: "Default Project Value"
4
+ category: reference:category_002
5
+ value_as_datetime: <%= Time.now.to_s(:db) %>
6
+ overrides:
7
+ value_as_double: 99.23546
8
+ value_as_integer: 3
9
+ substitutions:
10
+ value_as_text:
11
+ - 'change me'
12
+ - 'i only gone and got meself changed by datashift'
13
+ prefixes:
14
+ value_as_string: "prefix me every-time "
15
+ postfixes:
16
+ value_as_string: " postfix me every-time"
17
+ value_as_text: " postfix for value_as_text"
18
+
@@ -0,0 +1,28 @@
1
+ # Used by spec helper to build test databases
2
+ # These entires can be used in call to db_connect ( see spec_helper )
3
+
4
+ <% adapter = 'sqlite3' %>
5
+ <% adapter = 'jdbcsqlite3' if(DataShift::Guards::jruby? ) %>
6
+
7
+ test_mysql:
8
+ adapter: jdbcmysql
9
+ database: datashift_test_models_db
10
+ username: test
11
+ password: test
12
+ host: localhost
13
+
14
+ test_memory:
15
+ adapter: <%= adapter %>
16
+ database: :memory
17
+
18
+ test_file:
19
+ adapter: <%= adapter %>
20
+ database: <%= File.join(fixtures_path(), 'db', 'datashift_test_models_db.sqlite') %>
21
+ encoding: utf8
22
+ host: localhost
23
+
24
+ test_spree_standalone:
25
+ adapter: <%= adapter %>
26
+ database: <%= File.join(fixtures_path(), 'db', 'datashift_Spree_db.sqlite') %>
27
+ encoding: utf8
28
+ host: localhost
@@ -0,0 +1,6 @@
1
+ TITLE,value_as_string,Value as Text,value as datetime,value_as_boolean,value_as_double,test_association_models,test_association_bad
2
+ BAD001,Demo string,blah blah,2011-02-14,1.00,320.00,"name:test_acc_001,no_such_acc",name:test_acc_003
3
+ BAD002,Loads via Jruby,,2011-03-23,0.00,30.00,name:test_acc_003,
4
+ BAD003,Can load mulitple assoc columns with same name,,2011-05-19,TRUE,520.00,,name:missing_assoc
5
+ BAD004,,more blah,,FALSE,-345.00,name:,name:test_acc_002
6
+ ,,,23-09-09,oh dear,not a number,nam:,
@@ -0,0 +1,6 @@
1
+ TITLE,value_as_string,Value as Text,value as datetime,value_as_boolean,value_as_double,test_association_models,test_association_bad
2
+ BAD001,Demo string,blah blah,2011-02-14,1.00,320.00,"name:test_acc_001,no_such_acc",name:test_acc_003
3
+ BAD002,Loads via Jruby,,2011-03-23,0.00,30.00,name:test_acc_003,
4
+ BAD003,Can load mulitple assoc columns with same name,,2011-05-19,TRUE,520.00,,name:missing_assoc
5
+ BAD004,,more blah,,FALSE,-345.00,name:,name:test_acc_002
6
+ ,,,23-09-09,oh dear,not a number,nam:,
@@ -0,0 +1,5 @@
1
+ TITLE,value_as_string,Value as Text,value as datetime,value_as_boolean,value_as_double,Categories,Categories
2
+ 004,Demo string,blah blah,2011-02-14,1.00,320.00,"reference:category_001,category_002",reference:category_003
3
+ 005,Loads via Jruby,,2011-03-23,0.00,30.00,reference:category_003,
4
+ 006,Can load mulitple assoc columns with same name,,2011-05-19,TRUE,520.00,,
5
+ 007,,more blah,,FALSE,-345.00,,reference:category_002
@@ -0,0 +1,5 @@
1
+ TITLE,value_as_string,Value as Text,value as datetime,value_as_boolean,value_as_double,Categories:reference,Categories,Categories:reference
2
+ 004,Demo string,blah blah,2011-02-14,1.00,320.00,"category_001,category_002",reference:category_003,category_004
3
+ 005,Loads via Jruby,,2011-03-23,0.00,30.00,category_005,,
4
+ 006,Can load mulitple assoc columns with same name,,2011-05-19,TRUE,520.00,,,
5
+ 007,,more blah,,FALSE,-345.00,,,category_002
@@ -0,0 +1,5 @@
1
+ title,value_as_string,Value as Text,value as datetime,value_as_boolean,value_as_double,category,
2
+ 001,Demo string,blah blah,2011-02-14,1.00,320.00,"reference:category_001,category_002",
3
+ 002,Loads via Jruby,,2011-03-23,0.00,30.00,reference:category_003,
4
+ 003,003 Can handle different column naming styles,,2011-05-19,TRUE,520.00,"reference:category_001,category_002,category_003",
5
+ 099,,,,,,,
@@ -0,0 +1,4 @@
1
+ value_as_string,Value as Text,value as datetime,value_as_boolean,value_as_double
2
+ 001 Demo string,blah blah,2011-02-14,1.00,320.00
3
+ 002 Loads via Jruby,,2011-03-23,0.00,30.00
4
+ 003 Can handle different column naming styles,,2011-05-19,TRUE,520.00
@@ -0,0 +1,98 @@
1
+ # Author :: Tom Statter
2
+ # Date :: Aug 2011
3
+ # License:: MIT
4
+ #
5
+ # Details:: Migration to create a test database that exercises all aspects of Active Record models
6
+ #
7
+
8
+ class CreateTestBed < ActiveRecord::Migration
9
+
10
+ def self.up
11
+
12
+ # has many :projects
13
+ create_table :users do |t|
14
+ t.string :title
15
+ t.string :first_name
16
+ end
17
+
18
+ # belongs_to :user
19
+ # has many :milestones
20
+ #
21
+ create_table :projects do |t|
22
+ t.string :title
23
+ t.string :value_as_string
24
+ t.text :value_as_text
25
+ t.boolean :value_as_boolean, default: false
26
+ t.datetime :value_as_datetime, default: nil
27
+ t.integer :value_as_integer, default: 0
28
+ t.decimal :value_as_double, precision: 10, scale: 4, default: 0.0
29
+ t.references :user
30
+ t.timestamps null: false
31
+ end
32
+
33
+ # belongs_to :project
34
+ # @project => has_many :milestones
35
+
36
+ create_table :milestones do |t|
37
+ t.string :name
38
+ t.datetime :datetime, default: nil
39
+ t.decimal :cost, precision: 8, scale: 2, default: 0.0
40
+ t.references :project
41
+ t.timestamps null: false
42
+ end
43
+
44
+ # belongs_to :project, project => has_one
45
+ create_table :owners do |t|
46
+ t.string :name
47
+ t.decimal :budget
48
+ t.references :project
49
+ t.timestamps null: false
50
+ end
51
+
52
+ # has_belongs_to_many :project
53
+ create_table :categories do |t|
54
+ t.string :reference
55
+ t.timestamps null: false
56
+ end
57
+
58
+ # testing has_belongs_to_many (hence no id)
59
+ create_table :categories_projects, id: false do |t|
60
+ t.references :category
61
+ t.references :project
62
+ end
63
+
64
+ create_table :versions do |t|
65
+ t.string :name
66
+ t.timestamps null: false
67
+ end
68
+
69
+ # testing project has_many release + versions :through
70
+ create_table :loader_releases do |t|
71
+ t.string :name
72
+ t.references :project
73
+ t.references :version
74
+ t.timestamps null: false
75
+ end
76
+
77
+ create_table :long_and_complex_table_linked_to_versions do |t|
78
+ t.decimal :price
79
+ t.references :version
80
+ end
81
+
82
+ create_table :empties do |_t|
83
+ end
84
+
85
+ end
86
+
87
+ def self.down
88
+ drop_table :users
89
+ drop_table :projects
90
+ drop_table :categories
91
+ drop_table :loader_releases
92
+ drop_table :versions
93
+ drop_table :categories_projectss
94
+ drop_table :milestones
95
+ drop_table :long_and_complex_table_linked_to_versions
96
+ drop_table :empties
97
+ end
98
+ end
@@ -0,0 +1,24 @@
1
+ # Author :: Tom Statter
2
+ # Date :: Oct 2012
3
+ # License:: MIT
4
+ #
5
+ # Details:: Migration for paperclip specs
6
+
7
+ class AddDigitals < ActiveRecord::Migration
8
+
9
+ def self.up
10
+
11
+ create_table :digitals do |t|
12
+ t.integer :owner_id
13
+ t.string :attachment_file_name
14
+ t.string :attachment_content_type
15
+ t.integer :attachment_file_size
16
+ t.timestamps null: false
17
+ end
18
+
19
+ end
20
+
21
+ def self.down
22
+ drop_table :digitals
23
+ end
24
+ end
@@ -0,0 +1,9 @@
1
+ # This file will be copied into the dummy app so we can seed that app with
2
+ # some basic testable data
3
+
4
+ require "factory_girl"
5
+
6
+ FactoryGirl.factories.collect(&:name).inspect
7
+
8
+ FactoryGirl.create_list(:project_with_milestones, 8)
9
+
@@ -0,0 +1,3 @@
1
+ require 'datashift'
2
+
3
+ DataShift::load_commands
@@ -0,0 +1,6 @@
1
+
2
+
3
+ # had_and_belongs to join table
4
+ class Category < ActiveRecord::Base
5
+ has_and_belongs_to_many :projects
6
+ end
@@ -0,0 +1,22 @@
1
+ # To change this template, choose Tools | Templates
2
+ # and open the template in the editor.
3
+
4
+ require 'paperclip'
5
+
6
+ class Digital < ActiveRecord::Base
7
+
8
+ include Paperclip::Glue
9
+
10
+ # Rails 4 move to Controller
11
+ # attr_accessible :attachment
12
+
13
+ has_attached_file :attachment,
14
+ styles: { medium: '300x300>', thumb: '100x100>' },
15
+ path: ':rails_root/private/digitals/:id/:basename.:extension'
16
+
17
+ # Paperclip version 4.0 : all attachments are required to include a content_type validation,
18
+ # a file_name validation, or to explicitly state that they're not going to have either.
19
+ # Paperclip raises MissingRequiredValidatorError error if you do not do this.
20
+ validates_attachment_content_type :attachment,
21
+ content_type: ['image/jpg', 'image/jpeg', 'image/png', 'image/gif']
22
+ end
@@ -0,0 +1,2 @@
1
+ class Empty < ActiveRecord::Base
2
+ end
@@ -0,0 +1,10 @@
1
+
2
+
3
+ # Join Table with additional columns
4
+ class LoaderRelease < ActiveRecord::Base
5
+
6
+ belongs_to :project
7
+ belongs_to :version
8
+
9
+ # validate the name
10
+ end
@@ -0,0 +1,6 @@
1
+
2
+ # Join Table with additional columns
3
+ class LongAndComplexTableLinkedToVersion < ActiveRecord::Base
4
+
5
+ belongs_to :version
6
+ end
@@ -0,0 +1,15 @@
1
+
2
+
3
+ class Milestone < ActiveRecord::Base
4
+ belongs_to :project
5
+ # validate the name, cost
6
+
7
+ delegate :title, :title=, to: :project
8
+
9
+ attr_reader :x
10
+
11
+ def milestone_setter=( x)
12
+ @x = x
13
+ end
14
+
15
+ end
@@ -0,0 +1,13 @@
1
+ =begin
2
+ t.string :name
3
+ t.decimal :budget
4
+ t.references :project
5
+ =end
6
+
7
+ class Owner < ActiveRecord::Base
8
+
9
+ belongs_to :project, dependent: :destroy
10
+
11
+ has_many :digitals, dependent: :destroy
12
+
13
+ end
@@ -0,0 +1,53 @@
1
+ # A set of models and associations we can use in our specs to test
2
+ # basic database columns and also relationships
3
+
4
+ # See Companion migration spec/db/migrate for latest def
5
+
6
+ # create_table :projects do |t|
7
+ # t.string :title
8
+ # t.string :value_as_string
9
+ # t.text :value_as_text
10
+ # t.boolean :value_as_boolean, :default => false
11
+ # t.datetime :value_as_datetime, :default => nil
12
+ # t.integer :value_as_integer, :default => 0
13
+ #
14
+ # :precision - Specifies the precision for a :decimal column.
15
+ # :scale - Specifies the scale for a :decimal column.
16
+ #
17
+ # For example, the number 123.45 has a precision of 5 and a scale of 2.
18
+ # A decimal with a precision of 5 and a scale of 2 can range from -999.99 to 999.99.
19
+ #
20
+ # t.decimal :value_as_double, :precision => 8, :scale => 4, :default => 0.0
21
+ #
22
+ # t.references :user
23
+ # t.timestamps
24
+ # end
25
+
26
+ class Project < ActiveRecord::Base
27
+
28
+ belongs_to :user
29
+
30
+ has_one :owner
31
+
32
+ has_many :milestones
33
+
34
+ has_many :loader_releases
35
+ has_many :versions, through: :loader_releases
36
+
37
+ # has_many :release_versions, :through => :loader_releases, :source => :versions
38
+
39
+ has_and_belongs_to_many :categories
40
+
41
+ # Rails 4 move to Controller
42
+ # attr_accessible :value_as_string, :value_as_boolean, :value_as_double
43
+
44
+ def multiply
45
+ 10 * value_as_double
46
+ end
47
+
48
+ end
49
+
50
+ module DataShift
51
+ class AClassInAModule
52
+ end
53
+ end
@@ -0,0 +1,5 @@
1
+ class User < ActiveRecord::Base
2
+
3
+ has_many :projects
4
+
5
+ end
@@ -0,0 +1,7 @@
1
+
2
+
3
+ class Version < ActiveRecord::Base
4
+ has_many :releases
5
+
6
+ has_one :long_and_complex_table_linked_to_version
7
+ end