datashift 0.16.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (422) hide show
  1. checksums.yaml +4 -4
  2. data/{LICENSE.txt → LICENSE} +0 -0
  3. data/Rakefile +1 -20
  4. data/datashift.thor +125 -0
  5. data/lib/applications/apache_poi_extensions.rb +21 -52
  6. data/lib/applications/excel.rb +64 -57
  7. data/lib/applications/hssf_row_extensions.rb +66 -0
  8. data/lib/applications/jexcel_file.rb +99 -95
  9. data/lib/applications/jexcel_file_extensions.rb +76 -83
  10. data/lib/applications/jruby/word.rb +36 -36
  11. data/lib/applications/ruby_poi_translations.rb +34 -32
  12. data/lib/applications/spreadsheet_extensions.rb +21 -19
  13. data/lib/datashift.rb +49 -59
  14. data/lib/datashift/binder.rb +217 -0
  15. data/lib/datashift/column_packer.rb +21 -72
  16. data/lib/datashift/configuration.rb +317 -0
  17. data/lib/datashift/context_factory.rb +88 -0
  18. data/lib/datashift/core_ext/array.rb +15 -0
  19. data/lib/datashift/core_ext/csv_ext.rb +46 -0
  20. data/lib/datashift/core_ext/string.rb +49 -0
  21. data/lib/datashift/core_ext/to_b.rb +11 -0
  22. data/lib/datashift/delimiters.rb +55 -61
  23. data/lib/datashift/doc_context.rb +137 -0
  24. data/lib/datashift/excel_base.rb +93 -81
  25. data/lib/datashift/exceptions.rb +30 -28
  26. data/lib/datashift/file_definitions.rb +44 -39
  27. data/lib/datashift/guards.rb +5 -5
  28. data/lib/datashift/header.rb +25 -0
  29. data/lib/datashift/headers.rb +94 -0
  30. data/lib/datashift/inbound_data/column.rb +44 -0
  31. data/lib/datashift/inbound_data/lookup_support.rb +33 -0
  32. data/lib/datashift/inbound_data/method_binding.rb +139 -0
  33. data/lib/datashift/load_object.rb +37 -12
  34. data/lib/datashift/logging.rb +54 -27
  35. data/lib/datashift/mandatory.rb +39 -0
  36. data/lib/datashift/mapping/data_flow_schema.rb +198 -0
  37. data/lib/datashift/{model_mapper.rb → mapping/mapper_utils.rb} +30 -10
  38. data/lib/datashift/model_methods/catalogue.rb +183 -0
  39. data/lib/datashift/model_methods/collection.rb +140 -0
  40. data/lib/datashift/model_methods/model_method.rb +162 -0
  41. data/lib/datashift/model_methods/model_methods_manager.rb +76 -0
  42. data/lib/datashift/model_methods/operator.rb +62 -0
  43. data/lib/datashift/node_collection.rb +26 -0
  44. data/lib/datashift/node_context.rb +68 -0
  45. data/lib/datashift/populator.rb +308 -282
  46. data/lib/datashift/progress_monitor.rb +91 -0
  47. data/lib/datashift/querying.rb +110 -52
  48. data/lib/datashift/templates/import_export_config.erb +55 -0
  49. data/lib/datashift/transformation/factory.rb +219 -0
  50. data/lib/datashift/transformation/remove.rb +44 -0
  51. data/lib/datashift/version.rb +3 -0
  52. data/lib/exporters/configuration.rb +84 -0
  53. data/lib/exporters/csv_exporter.rb +54 -52
  54. data/lib/exporters/excel_exporter.rb +80 -61
  55. data/lib/exporters/exporter_base.rb +8 -8
  56. data/lib/generators/config_generator.rb +218 -0
  57. data/lib/generators/csv_generator.rb +13 -70
  58. data/lib/generators/excel_generator.rb +23 -111
  59. data/lib/generators/generator_base.rb +15 -70
  60. data/lib/loaders/configuration.rb +90 -0
  61. data/lib/loaders/csv_loader.rb +63 -101
  62. data/lib/loaders/excel_loader.rb +71 -156
  63. data/lib/loaders/failure_data.rb +40 -0
  64. data/lib/loaders/file_loader.rb +16 -0
  65. data/lib/loaders/loader_base.rb +82 -410
  66. data/lib/loaders/loader_factory.rb +42 -0
  67. data/lib/loaders/paperclip/attachment_loader.rb +157 -140
  68. data/lib/loaders/paperclip/datashift_paperclip.rb +18 -35
  69. data/lib/loaders/paperclip/image_loading.rb +40 -35
  70. data/lib/loaders/reporters/basic_stdout_reporter.rb +40 -0
  71. data/lib/loaders/reporters/reporter.rb +26 -0
  72. data/lib/tasks/config.thor +65 -0
  73. data/{tasks → lib/tasks}/config/seed_fu_product_template.erb +0 -0
  74. data/{tasks → lib/tasks}/config/tidy_config.txt +0 -0
  75. data/lib/tasks/export.thor +192 -0
  76. data/lib/tasks/generate.thor +190 -0
  77. data/lib/tasks/import.thor +142 -0
  78. data/lib/{thor → tasks}/paperclip.thor +69 -69
  79. data/{tasks → lib/tasks/to_convert_to_thor}/db_tasks.rake +20 -20
  80. data/lib/tasks/tools.thor +109 -0
  81. data/spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg +0 -0
  82. data/spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg +0 -0
  83. data/spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg +0 -0
  84. data/spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg +0 -0
  85. data/spec/datashift/binder_spec.rb +266 -0
  86. data/spec/datashift/config_generator_spec.rb +186 -0
  87. data/spec/datashift/configuration.rb +66 -0
  88. data/spec/datashift/context_factory_spec.rb +63 -0
  89. data/spec/datashift/data_flow_schema_spec.rb +150 -0
  90. data/spec/datashift/datashift_spec.rb +52 -0
  91. data/spec/datashift/excel_base_spec.rb +57 -0
  92. data/spec/datashift/excel_spec.rb +188 -0
  93. data/spec/datashift/failure_data_spec.rb +27 -0
  94. data/spec/{file_definitions.rb → datashift/file_definitions.rb} +9 -10
  95. data/spec/datashift/headers_spec.rb +56 -0
  96. data/spec/datashift/inbound_data_spec.rb +47 -0
  97. data/spec/datashift/mapper_utils_spec.rb +38 -0
  98. data/spec/datashift/method_binding_spec.rb +60 -0
  99. data/spec/datashift/model_method_spec.rb +109 -0
  100. data/spec/datashift/model_methods_catalogue.rb +111 -0
  101. data/spec/datashift/model_methods_collection_spec.rb +138 -0
  102. data/spec/datashift/model_methods_manager_spec.rb +329 -0
  103. data/spec/datashift/populator_spec.rb +117 -0
  104. data/spec/datashift/thor_spec.rb +314 -0
  105. data/spec/datashift/transformation/factory_spec.rb +195 -0
  106. data/spec/datashift/transformation/transformer_remove_spec.rb +43 -0
  107. data/spec/dummy/Gemfile +53 -0
  108. data/spec/dummy/Gemfile.lock +197 -0
  109. data/spec/dummy/README.rdoc +28 -0
  110. data/spec/dummy/Rakefile +6 -0
  111. data/spec/dummy/app/assets/javascripts/application.js +16 -0
  112. data/spec/dummy/app/assets/javascripts/categories.js +2 -0
  113. data/spec/dummy/app/assets/javascripts/digitals.js +2 -0
  114. data/spec/dummy/app/assets/javascripts/empties.js +2 -0
  115. data/spec/dummy/app/assets/javascripts/loader_releases.js +2 -0
  116. data/spec/dummy/app/assets/javascripts/long_and_complex_table_linked_to_versions.js +2 -0
  117. data/spec/dummy/app/assets/javascripts/milestones.js +2 -0
  118. data/spec/dummy/app/assets/javascripts/owners.js +2 -0
  119. data/spec/dummy/app/assets/javascripts/projects.js +2 -0
  120. data/spec/dummy/app/assets/javascripts/users.js +2 -0
  121. data/spec/dummy/app/assets/javascripts/versions.js +2 -0
  122. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  123. data/spec/dummy/app/assets/stylesheets/categories.css +4 -0
  124. data/spec/dummy/app/assets/stylesheets/digitals.css +4 -0
  125. data/spec/dummy/app/assets/stylesheets/empties.css +4 -0
  126. data/spec/dummy/app/assets/stylesheets/loader_releases.css +4 -0
  127. data/spec/dummy/app/assets/stylesheets/long_and_complex_table_linked_to_versions.css +4 -0
  128. data/spec/dummy/app/assets/stylesheets/milestones.css +4 -0
  129. data/spec/dummy/app/assets/stylesheets/owners.css +4 -0
  130. data/spec/dummy/app/assets/stylesheets/projects.css +4 -0
  131. data/spec/dummy/app/assets/stylesheets/scaffold.css +56 -0
  132. data/spec/dummy/app/assets/stylesheets/users.css +4 -0
  133. data/spec/dummy/app/assets/stylesheets/versions.css +4 -0
  134. data/spec/dummy/app/controllers/application_controller.rb +5 -0
  135. data/spec/dummy/app/controllers/categories_controller.rb +58 -0
  136. data/spec/dummy/app/controllers/digitals_controller.rb +58 -0
  137. data/spec/dummy/app/controllers/empties_controller.rb +58 -0
  138. data/spec/dummy/app/controllers/loader_releases_controller.rb +58 -0
  139. data/spec/dummy/app/controllers/long_and_complex_table_linked_to_versions_controller.rb +58 -0
  140. data/spec/dummy/app/controllers/milestones_controller.rb +58 -0
  141. data/spec/dummy/app/controllers/owners_controller.rb +58 -0
  142. data/spec/dummy/app/controllers/projects_controller.rb +58 -0
  143. data/spec/dummy/app/controllers/users_controller.rb +58 -0
  144. data/spec/dummy/app/controllers/versions_controller.rb +58 -0
  145. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  146. data/spec/dummy/app/helpers/categories_helper.rb +2 -0
  147. data/spec/dummy/app/helpers/digitals_helper.rb +2 -0
  148. data/spec/dummy/app/helpers/empties_helper.rb +2 -0
  149. data/spec/dummy/app/helpers/loader_releases_helper.rb +2 -0
  150. data/spec/dummy/app/helpers/long_and_complex_table_linked_to_versions_helper.rb +2 -0
  151. data/spec/dummy/app/helpers/milestones_helper.rb +2 -0
  152. data/spec/dummy/app/helpers/owners_helper.rb +2 -0
  153. data/spec/dummy/app/helpers/projects_helper.rb +2 -0
  154. data/spec/dummy/app/helpers/users_helper.rb +2 -0
  155. data/spec/dummy/app/helpers/versions_helper.rb +2 -0
  156. data/spec/dummy/app/models/category.rb +6 -0
  157. data/spec/dummy/app/models/digital.rb +22 -0
  158. data/spec/dummy/app/models/empty.rb +2 -0
  159. data/spec/dummy/app/models/loader_release.rb +10 -0
  160. data/spec/dummy/app/models/long_and_complex_table_linked_to_version.rb +6 -0
  161. data/spec/dummy/app/models/milestone.rb +15 -0
  162. data/spec/dummy/app/models/owner.rb +13 -0
  163. data/spec/dummy/app/models/project.rb +53 -0
  164. data/spec/dummy/app/models/user.rb +5 -0
  165. data/spec/dummy/app/models/version.rb +7 -0
  166. data/spec/dummy/app/views/categories/_form.html.erb +17 -0
  167. data/spec/dummy/app/views/categories/edit.html.erb +6 -0
  168. data/spec/dummy/app/views/categories/index.html.erb +25 -0
  169. data/spec/dummy/app/views/categories/new.html.erb +5 -0
  170. data/spec/dummy/app/views/categories/show.html.erb +4 -0
  171. data/spec/dummy/app/views/digitals/_form.html.erb +17 -0
  172. data/spec/dummy/app/views/digitals/edit.html.erb +6 -0
  173. data/spec/dummy/app/views/digitals/index.html.erb +25 -0
  174. data/spec/dummy/app/views/digitals/new.html.erb +5 -0
  175. data/spec/dummy/app/views/digitals/show.html.erb +4 -0
  176. data/spec/dummy/app/views/empties/_form.html.erb +17 -0
  177. data/spec/dummy/app/views/empties/edit.html.erb +6 -0
  178. data/spec/dummy/app/views/empties/index.html.erb +25 -0
  179. data/spec/dummy/app/views/empties/new.html.erb +5 -0
  180. data/spec/dummy/app/views/empties/show.html.erb +4 -0
  181. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  182. data/spec/dummy/app/views/loader_releases/_form.html.erb +17 -0
  183. data/spec/dummy/app/views/loader_releases/edit.html.erb +6 -0
  184. data/spec/dummy/app/views/loader_releases/index.html.erb +25 -0
  185. data/spec/dummy/app/views/loader_releases/new.html.erb +5 -0
  186. data/spec/dummy/app/views/loader_releases/show.html.erb +4 -0
  187. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/_form.html.erb +17 -0
  188. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/edit.html.erb +6 -0
  189. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/index.html.erb +25 -0
  190. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/new.html.erb +5 -0
  191. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/show.html.erb +4 -0
  192. data/spec/dummy/app/views/milestones/_form.html.erb +17 -0
  193. data/spec/dummy/app/views/milestones/edit.html.erb +6 -0
  194. data/spec/dummy/app/views/milestones/index.html.erb +25 -0
  195. data/spec/dummy/app/views/milestones/new.html.erb +5 -0
  196. data/spec/dummy/app/views/milestones/show.html.erb +4 -0
  197. data/spec/dummy/app/views/owners/_form.html.erb +17 -0
  198. data/spec/dummy/app/views/owners/edit.html.erb +6 -0
  199. data/spec/dummy/app/views/owners/index.html.erb +25 -0
  200. data/spec/dummy/app/views/owners/new.html.erb +5 -0
  201. data/spec/dummy/app/views/owners/show.html.erb +4 -0
  202. data/spec/dummy/app/views/projects/_form.html.erb +17 -0
  203. data/spec/dummy/app/views/projects/edit.html.erb +6 -0
  204. data/spec/dummy/app/views/projects/index.html.erb +25 -0
  205. data/spec/dummy/app/views/projects/new.html.erb +5 -0
  206. data/spec/dummy/app/views/projects/show.html.erb +4 -0
  207. data/spec/dummy/app/views/users/_form.html.erb +17 -0
  208. data/spec/dummy/app/views/users/edit.html.erb +6 -0
  209. data/spec/dummy/app/views/users/index.html.erb +25 -0
  210. data/spec/dummy/app/views/users/new.html.erb +5 -0
  211. data/spec/dummy/app/views/users/show.html.erb +4 -0
  212. data/spec/dummy/app/views/versions/_form.html.erb +17 -0
  213. data/spec/dummy/app/views/versions/edit.html.erb +6 -0
  214. data/spec/dummy/app/views/versions/index.html.erb +25 -0
  215. data/spec/dummy/app/views/versions/new.html.erb +5 -0
  216. data/spec/dummy/app/views/versions/show.html.erb +4 -0
  217. data/spec/dummy/bin/bundle +3 -0
  218. data/spec/dummy/bin/rails +9 -0
  219. data/spec/dummy/bin/rake +9 -0
  220. data/spec/dummy/bin/setup +29 -0
  221. data/spec/dummy/bin/spring +16 -0
  222. data/spec/dummy/config.ru +4 -0
  223. data/spec/dummy/config/application.rb +26 -0
  224. data/spec/dummy/config/boot.rb +3 -0
  225. data/spec/dummy/config/database.yml +25 -0
  226. data/spec/dummy/config/environment.rb +5 -0
  227. data/spec/dummy/config/environments/development.rb +41 -0
  228. data/spec/dummy/config/environments/production.rb +79 -0
  229. data/spec/dummy/config/environments/test.rb +42 -0
  230. data/spec/dummy/config/initializers/assets.rb +11 -0
  231. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  232. data/spec/dummy/config/initializers/cookies_serializer.rb +3 -0
  233. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  234. data/spec/dummy/config/initializers/inflections.rb +16 -0
  235. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  236. data/spec/dummy/config/initializers/session_store.rb +3 -0
  237. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  238. data/spec/dummy/config/locales/en.yml +23 -0
  239. data/spec/dummy/config/routes.rb +76 -0
  240. data/spec/dummy/config/secrets.yml +22 -0
  241. data/spec/dummy/db/development.sqlite3 +0 -0
  242. data/spec/dummy/db/migrate/20110803201325_create_test_bed.rb +98 -0
  243. data/spec/dummy/db/migrate/20121009161700_add_digitals.rb +24 -0
  244. data/spec/dummy/db/migrate/20161005123106_create_digitals.rb +8 -0
  245. data/spec/dummy/db/migrate/20161005123106_create_long_and_complex_table_linked_to_versions.rb +8 -0
  246. data/spec/dummy/db/migrate/20161005123107_create_loader_releases.rb +8 -0
  247. data/spec/dummy/db/migrate/20161005123108_create_owners.rb +8 -0
  248. data/spec/dummy/db/migrate/20161005123109_create_empties.rb +8 -0
  249. data/spec/dummy/db/migrate/20161005123110_create_projects.rb +8 -0
  250. data/spec/dummy/db/migrate/20161005123111_create_users.rb +8 -0
  251. data/spec/dummy/db/migrate/20161005123111_create_versions.rb +8 -0
  252. data/spec/dummy/db/migrate/20161005123112_create_milestones.rb +8 -0
  253. data/spec/dummy/db/migrate/20161005123113_create_categories.rb +8 -0
  254. data/spec/dummy/db/schema.rb +93 -0
  255. data/spec/dummy/db/seeds.rb +9 -0
  256. data/spec/dummy/db/test.sqlite3 +0 -0
  257. data/spec/dummy/log/test.log +69 -0
  258. data/spec/dummy/public/404.html +67 -0
  259. data/spec/dummy/public/422.html +67 -0
  260. data/spec/dummy/public/500.html +66 -0
  261. data/spec/dummy/public/favicon.ico +0 -0
  262. data/spec/dummy/public/robots.txt +5 -0
  263. data/spec/dummy/sandbox_example.thor +4 -0
  264. data/spec/dummy/test/controllers/categories_controller_test.rb +49 -0
  265. data/spec/dummy/test/controllers/digitals_controller_test.rb +49 -0
  266. data/spec/dummy/test/controllers/empties_controller_test.rb +49 -0
  267. data/spec/dummy/test/controllers/loader_releases_controller_test.rb +49 -0
  268. data/spec/dummy/test/controllers/long_and_complex_table_linked_to_versions_controller_test.rb +49 -0
  269. data/spec/dummy/test/controllers/milestones_controller_test.rb +49 -0
  270. data/spec/dummy/test/controllers/owners_controller_test.rb +49 -0
  271. data/spec/dummy/test/controllers/projects_controller_test.rb +49 -0
  272. data/spec/dummy/test/controllers/users_controller_test.rb +49 -0
  273. data/spec/dummy/test/controllers/versions_controller_test.rb +49 -0
  274. data/spec/dummy/test/factories/categories.rb +5 -0
  275. data/spec/dummy/test/factories/digitals.rb +5 -0
  276. data/spec/dummy/test/factories/empties.rb +5 -0
  277. data/spec/dummy/test/factories/loader_releases.rb +5 -0
  278. data/spec/dummy/test/factories/long_and_complex_table_linked_to_versions.rb +5 -0
  279. data/spec/dummy/test/factories/milestones.rb +5 -0
  280. data/spec/dummy/test/factories/owners.rb +5 -0
  281. data/spec/dummy/test/factories/projects.rb +5 -0
  282. data/spec/dummy/test/factories/users.rb +5 -0
  283. data/spec/dummy/test/factories/versions.rb +5 -0
  284. data/spec/dummy/test/models/category_test.rb +7 -0
  285. data/spec/dummy/test/models/digital_test.rb +7 -0
  286. data/spec/dummy/test/models/empty_test.rb +7 -0
  287. data/spec/dummy/test/models/loader_release_test.rb +7 -0
  288. data/spec/dummy/test/models/long_and_complex_table_linked_to_version_test.rb +7 -0
  289. data/spec/dummy/test/models/milestone_test.rb +7 -0
  290. data/spec/dummy/test/models/owner_test.rb +7 -0
  291. data/spec/dummy/test/models/project_test.rb +7 -0
  292. data/spec/dummy/test/models/user_test.rb +7 -0
  293. data/spec/dummy/test/models/version_test.rb +7 -0
  294. data/spec/dummy/test/test_helper.rb +10 -0
  295. data/spec/exporters/csv_exporter_spec.rb +240 -0
  296. data/spec/exporters/csv_generator_spec.rb +139 -0
  297. data/spec/exporters/excel_exporter_spec.rb +193 -0
  298. data/spec/exporters/excel_generator_spec.rb +181 -0
  299. data/spec/exporters/generator_base_spec.rb +45 -0
  300. data/spec/factories/categories.rb +7 -0
  301. data/spec/factories/factories.rb +18 -0
  302. data/spec/factories/milestone.rb +16 -0
  303. data/spec/factories/projects.rb +41 -0
  304. data/spec/fixtures/BadAssociationName.xls +0 -0
  305. data/spec/fixtures/DemoNegativeTesting.xls +0 -0
  306. data/spec/fixtures/ProjectConfiguration.yml +18 -0
  307. data/spec/fixtures/ProjectsMultiCategories.xls +0 -0
  308. data/spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls +0 -0
  309. data/spec/fixtures/ProjectsSingleCategories.xls +0 -0
  310. data/spec/fixtures/ProjectsSingleCategories.xlsx +0 -0
  311. data/spec/fixtures/SimpleProjects.xls +0 -0
  312. data/spec/fixtures/config/database.yml +28 -0
  313. data/spec/fixtures/csv/BadAssociationName.csv +6 -0
  314. data/spec/fixtures/csv/DemoNegativeTesting.csv +6 -0
  315. data/spec/fixtures/csv/ProjectsMultiCategories.csv +5 -0
  316. data/spec/fixtures/csv/ProjectsMultiCategoriesHeaderLookup.csv +5 -0
  317. data/spec/fixtures/csv/ProjectsSingleCategories.csv +5 -0
  318. data/spec/fixtures/csv/SimpleProjects.csv +4 -0
  319. data/spec/fixtures/db/migrate/20110803201325_create_test_bed.rb +98 -0
  320. data/spec/fixtures/db/migrate/20121009161700_add_digitals.rb +24 -0
  321. data/spec/fixtures/db/seeds.rb +9 -0
  322. data/spec/fixtures/images/DEMO_001_ror_bag.jpeg +0 -0
  323. data/spec/fixtures/images/DEMO_002_Powerstation.jpeg +0 -0
  324. data/spec/fixtures/images/DEMO_003_ror_mug.jpeg +0 -0
  325. data/spec/fixtures/images/DEMO_004_ror_ringer.jpeg +0 -0
  326. data/spec/fixtures/load_datashift.thor +3 -0
  327. data/spec/fixtures/models/category.rb +6 -0
  328. data/spec/fixtures/models/digital.rb +22 -0
  329. data/spec/fixtures/models/empty.rb +2 -0
  330. data/spec/fixtures/models/loader_release.rb +10 -0
  331. data/spec/fixtures/models/long_and_complex_table_linked_to_version.rb +6 -0
  332. data/spec/fixtures/models/milestone.rb +15 -0
  333. data/spec/fixtures/models/owner.rb +13 -0
  334. data/spec/fixtures/models/project.rb +53 -0
  335. data/spec/fixtures/models/user.rb +5 -0
  336. data/spec/fixtures/models/version.rb +7 -0
  337. data/spec/fixtures/results/exp_project_assoc_headers.xls +0 -0
  338. data/spec/fixtures/results/exp_project_collection_spec.csv +2 -0
  339. data/spec/fixtures/results/exp_project_export.xls +0 -0
  340. data/spec/fixtures/results/exp_project_first_export.xls +0 -0
  341. data/spec/fixtures/results/exp_project_plus_assoc.xls +0 -0
  342. data/spec/fixtures/results/exp_project_plus_assoc_export_spec.csv +9 -0
  343. data/spec/fixtures/results/gen_project_plus_assoc_template.xls +0 -0
  344. data/spec/fixtures/results/gen_project_plus_some_assoc_template.xls +0 -0
  345. data/spec/fixtures/results/gen_project_template.xls +0 -0
  346. data/spec/fixtures/results/project_and_assoc_in_hash_export.xls +0 -0
  347. data/spec/fixtures/results/project_and_assoc_in_json_export.csv +9 -0
  348. data/spec/fixtures/results/project_and_assoc_in_json_export.xls +0 -0
  349. data/spec/fixtures/results/project_export_spec_with_custom_delim_,.csv +2 -0
  350. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/243.csv +2 -0
  351. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/247.csv +2 -0
  352. data/spec/fixtures/results/project_plus_assoc_template.csv +1 -0
  353. data/spec/fixtures/results/project_plus_some_assoc_template.csv +1 -0
  354. data/spec/fixtures/results/project_remove_export_spec.csv +2 -0
  355. data/spec/fixtures/results/project_template.csv +1 -0
  356. data/spec/fixtures/results/project_with_methods_export_spec.csv +2 -0
  357. data/spec/fixtures/results/thor_spec_gen_project.csv +1 -0
  358. data/spec/fixtures/sandbox_example.thor +4 -0
  359. data/spec/fixtures/simple_export_spec.xls +0 -0
  360. data/spec/fixtures/simple_template_spec.xls +0 -0
  361. data/spec/fixtures/test_model_defs.rb +7 -0
  362. data/spec/loaders/csv_loader_spec.rb +206 -0
  363. data/spec/loaders/data_flow_excel_loader_spec.rb +290 -0
  364. data/spec/loaders/excel_loader_failures_spec.rb +67 -0
  365. data/spec/loaders/excel_loader_spec.rb +294 -0
  366. data/spec/loaders/loader_base_spec.rb +29 -0
  367. data/spec/loaders/paperclip_loader_spec.rb +106 -0
  368. data/spec/log/datashift.log +14930 -0
  369. data/spec/private/digitals/1/DEMO_003_ror_mug.jpeg +0 -0
  370. data/spec/private/digitals/2/DEMO_002_Powerstation.jpeg +0 -0
  371. data/spec/private/digitals/3/DEMO_004_ror_ringer.jpeg +0 -0
  372. data/spec/private/digitals/4/DEMO_001_ror_bag.jpeg +0 -0
  373. data/spec/spec_helper.rb +26 -230
  374. data/spec/support/clear_and_manage_contexts.rb +25 -0
  375. data/spec/support/database_cleaner.rb +32 -0
  376. data/spec/support/datashift_test_helpers.rb +153 -0
  377. data/spec/support/files_paths_helper.rb +13 -0
  378. data/spec/support/fixtures/results/mapping_template.yaml +15 -0
  379. data/spec/support/sandbox.rb +136 -0
  380. metadata +804 -85
  381. data/README.markdown +0 -274
  382. data/README.rdoc +0 -19
  383. data/VERSION +0 -1
  384. data/datashift.gemspec +0 -48
  385. data/lib/applications/jruby/old_pre_proxy_jexcel_file.rb +0 -437
  386. data/lib/datashift/data_transforms.rb +0 -83
  387. data/lib/datashift/mapping_file_definitions.rb +0 -88
  388. data/lib/datashift/mapping_service.rb +0 -91
  389. data/lib/datashift/method_detail.rb +0 -165
  390. data/lib/datashift/method_details_manager.rb +0 -95
  391. data/lib/datashift/method_dictionary.rb +0 -281
  392. data/lib/datashift/method_mapper.rb +0 -174
  393. data/lib/datashift/thor_base.rb +0 -38
  394. data/lib/generators/mapping_generator.rb +0 -112
  395. data/lib/helpers/core_ext/csv_file.rb +0 -33
  396. data/lib/helpers/core_ext/to_b.rb +0 -24
  397. data/lib/loaders/reporter.rb +0 -58
  398. data/lib/thor/export.thor +0 -175
  399. data/lib/thor/generate.thor +0 -191
  400. data/lib/thor/import.thor +0 -110
  401. data/lib/thor/mapping.thor +0 -65
  402. data/lib/thor/tools.thor +0 -84
  403. data/spec/Gemfile +0 -31
  404. data/spec/Gemfile.lock +0 -134
  405. data/spec/csv_exporter_spec.rb +0 -144
  406. data/spec/csv_generator_spec.rb +0 -159
  407. data/spec/csv_loader_spec.rb +0 -212
  408. data/spec/datashift_spec.rb +0 -55
  409. data/spec/excel_exporter_spec.rb +0 -199
  410. data/spec/excel_generator_spec.rb +0 -203
  411. data/spec/excel_loader_spec.rb +0 -237
  412. data/spec/excel_spec.rb +0 -203
  413. data/spec/loader_base_spec.rb +0 -166
  414. data/spec/mapping_spec.rb +0 -117
  415. data/spec/method_dictionary_spec.rb +0 -300
  416. data/spec/method_mapper_spec.rb +0 -100
  417. data/spec/model_mapper_spec.rb +0 -41
  418. data/spec/paperclip_loader_spec.rb +0 -92
  419. data/spec/populator_spec.rb +0 -128
  420. data/spec/thor_spec.rb +0 -90
  421. data/tasks/file_tasks.rake +0 -37
  422. data/tasks/word_to_seedfu.rake +0 -167
@@ -0,0 +1,138 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2011
2
+ # Author :: Tom Statter
3
+ # Date :: Mar 2015
4
+ # License:: MIT
5
+ #
6
+ # Details:: Specs for ModelMapperManager and associated DataShift::ModelMethodsMgrs::Dictionary
7
+ #
8
+ require File.join(File.dirname(__FILE__), '/../spec_helper')
9
+
10
+ module DataShift
11
+
12
+ module ModelMethods
13
+
14
+ describe 'Collection' do
15
+
16
+ before(:all) do
17
+ DataShift::ModelMethods::Catalogue.clear
18
+ DataShift::ModelMethods::Manager.clear
19
+ end
20
+
21
+ context 'Creation' do
22
+ it 'should be' do
23
+ expect(Collection.new( Project )).to be
24
+ end
25
+
26
+ it 'should manage method collection at class level' do
27
+ expect(Collection.new( Project ).managed_class).to eq Project
28
+ end
29
+
30
+ end
31
+
32
+ context 'Empty' do
33
+ let(:collection) { Collection.new(Project) }
34
+
35
+ it 'should report name of collected class' do
36
+ c = Collection.new( Project )
37
+ expect(c.managed_class).to eq Project
38
+ end
39
+
40
+ it 'should provide access to a set of model methods by type' do
41
+ expect(collection.by_optype).to be_a Hash
42
+ end
43
+
44
+ it 'should return nil when no such optype and_operator' do
45
+ expect(collection.find_by_name_and_type(:no_such_op, :belongs_to)).to be_nil
46
+ end
47
+ end
48
+
49
+ context 'Build a Collection' do
50
+
51
+ before(:all) do
52
+ Catalogue.clear
53
+ Manager.clear
54
+
55
+ Manager.catalog_class( Project )
56
+ end
57
+
58
+ let(:assignments) { Catalogue.assignments_for(Project) }
59
+
60
+ let(:name) { assignments.first }
61
+
62
+ let(:model_method) { DataShift::ModelMethod.new(Project, name, :assignment) }
63
+
64
+ let(:collection) { Collection.new(Project) }
65
+
66
+ before(:each) do
67
+ collection.add( model_method )
68
+ end
69
+
70
+ it 'should provide access to complete set of model methods for a class' do
71
+ expect(collection.model_method_list).to be_a Array
72
+ end
73
+
74
+ it 'should provide access to iterate through the complete set of model methods' do
75
+ expect(collection.respond_to? :each).to eq true
76
+ end
77
+
78
+ it 'should enable adding model methods to create manager' do
79
+ expect(collection.model_method_list.size).to be 1
80
+ expect(collection.by_optype.size).to be 1
81
+ end
82
+
83
+ it 'should enable extraction of details from manager' do
84
+
85
+ collection.add( DataShift::ModelMethod.new(Project, assignments[1], :assignment) )
86
+
87
+ expect(collection.for_type(:assignment).size).to eq 2
88
+ expect(collection.for_type(:assignment).first).to be_a ModelMethod
89
+
90
+ expect(collection.by_optype[:assignment]).to be_a Array
91
+ expect(collection.by_optype[:assignment].size).to eq 2
92
+
93
+ expect(collection.by_optype[:assignment].include?(model_method)).to eq true
94
+ expect(collection.by_optype[:assignment].first.operator).to eq name
95
+ end
96
+
97
+ it 'should enable finding specific operators' do
98
+ expect( collection.find_by_name_and_type(name, :assignment ) ).to eq model_method
99
+ end
100
+ end
101
+
102
+ context 'Full Collection from Class and operations' do
103
+
104
+ let(:collection) { ModelMethods::Manager.catalog_class(Project) }
105
+
106
+ it 'should be sortable based on operator and op type' do
107
+
108
+ expect(collection).to be_a DataShift::ModelMethods::Collection
109
+
110
+ collection.unshift( DataShift::ModelMethod.new(Project, 'aaa', :has_many) )
111
+ collection.add( DataShift::ModelMethod.new(Project, 'aaa', :assignment) )
112
+ collection.unshift( DataShift::ModelMethod.new(Project, 'zzz', :assignment) )
113
+
114
+ expect(collection.first.operator).to eq 'zzz'
115
+ expect(collection.last.operator_type? :assignment).to eq true
116
+
117
+ collection.sort!
118
+
119
+ expect(collection.first.operator).to eq 'aaa'
120
+ expect(collection.last.operator_type? :has_many).to eq true
121
+ end
122
+
123
+
124
+ it 'should be Searchable across op types' do
125
+ names = Project.reflect_on_all_associations(:belongs_to).map { |i| i.name.to_s }
126
+
127
+ mm = collection.search( names.last )
128
+
129
+ expect(mm.operator).to eq names.last
130
+ expect(mm.operator_type? :belongs_to).to eq true
131
+ end
132
+
133
+ end
134
+ end
135
+
136
+ end
137
+
138
+ end
@@ -0,0 +1,329 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2015
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2015
4
+ # License:: MIT
5
+ #
6
+ #
7
+ require File.join(File.dirname(__FILE__), '/../spec_helper')
8
+
9
+ module DataShift
10
+
11
+ module ModelMethods
12
+
13
+ describe ':ModelMethods ManagerDictionary' do
14
+ include_context 'ClearAllCatalogues'
15
+
16
+ it 'should provide a dictionary of class => manager' do
17
+ expect(Manager.collections).to be_a Hash
18
+ expect(Manager.collections.empty?).to eq true
19
+ end
20
+
21
+ it 'should provide a dictionary of class => manager' do
22
+ expect(Manager.collections[Milestone]).to be_nil
23
+ end
24
+
25
+ it 'should provide facilities to check a class been collected' do
26
+ expect(Manager.for?(Milestone)).to eq false
27
+ end
28
+
29
+ it 'should return nil as an empty collection' do
30
+ expect(Manager.for(Milestone)).to be nil
31
+ end
32
+
33
+ it 'should enable a class to be collected and return new collection' do
34
+ expect(Manager.catalog_class(Milestone)).to be_a Collection
35
+ end
36
+
37
+ context 'Cataloged' do
38
+ before(:each) do
39
+ Manager.catalog_class(Milestone)
40
+ end
41
+
42
+ it 'should report when a class has been collected' do
43
+ expect(Manager.for?(Milestone)).to eq true
44
+ end
45
+
46
+ it 'should provide facilities to return a populated collection' do
47
+ expect(Manager.for(Milestone)).to be_a Collection
48
+ end
49
+
50
+ it 'should provide access to a collection' do
51
+ expect(Manager.collections[Milestone]).to be_a Collection
52
+
53
+ expect(Manager.collections[Milestone]).to eq Manager.for(Milestone)
54
+ end
55
+
56
+ it 'should populate assignment operators for method details for different forms of a column name' do
57
+ Manager.catalog_class(Project)
58
+ Manager.catalog_class(Milestone)
59
+
60
+ collection = Manager.for(Project)
61
+
62
+ expect(collection.managed_class).to eq Project
63
+
64
+ model_method = collection.find_by_name_and_type('value_as_string', :assignment)
65
+
66
+ expect(model_method).to be_a DataShift::ModelMethod
67
+
68
+ expect(model_method.operator).to eq 'value_as_string'
69
+ expect(model_method.operator_for(:assignment)).to eq 'value_as_string'
70
+
71
+ expect(model_method.operator?('value_as_string')).to eq true
72
+ expect(model_method.operator?('blah_as_string')).to eq false
73
+
74
+ expect(model_method.operator_for(:belongs_to)).to eq nil
75
+ expect(model_method.operator_for(:has_many)).to eq nil
76
+ end
77
+
78
+ # it "should populate method dictionary for a given AR model" do
79
+ #
80
+ # DataShift::ModelMethodsManager.find_methods( Project )
81
+ # DataShift::ModelMethodsManager.find_methods( Milestone )
82
+ #
83
+ # DataShift::MethodDictionary.has_many.should_not be_empty
84
+ # DataShift::MethodDictionary.has_many[Project].should include('milestones')
85
+ #
86
+ # DataShift::MethodDictionary.assignments.should_not be_empty
87
+ # DataShift::MethodDictionary.assignments[Project].should include('id')
88
+ # DataShift::MethodDictionary.assignments[Project].should include('value_as_string')
89
+ # DataShift::MethodDictionary.assignments[Project].should include('value_as_text')
90
+ #
91
+ # DataShift::MethodDictionary.belongs_to.should_not be_empty
92
+ # expect(DataShift::MethodDictionary.belongs_to[Project]).to include 'user'
93
+ #
94
+ #
95
+ # DataShift::MethodDictionary.column_types.should be_is_a(Hash)
96
+ # DataShift::MethodDictionary.column_types.should_not be_empty
97
+ # DataShift::MethodDictionary.column_types[Project].size.should == Project.columns.size
98
+ #
99
+ #
100
+ # end
101
+ #
102
+ # it "should populate assigment members without the equivalent association names" do
103
+ #
104
+ # DataShift::ModelMethodsManager.find_methods( Project )
105
+ #
106
+ # # we should remove has-many & belongs_to from basic assignment set as they require a DB lookup
107
+ # # or a Model.create call, not a simple assignment
108
+ #
109
+ # DataShift::MethodDictionary.assignments_for(Project).should_not include( DataShift::MethodDictionary.belongs_to_for(Project) )
110
+ # DataShift::MethodDictionary.assignments_for(Project).should_not include( DataShift::MethodDictionary.has_many_for(Project) )
111
+ # end
112
+ #
113
+ #
114
+ #
115
+ #
116
+ #
117
+ # # Note : Not all assignments will currently have a column type, for example
118
+ # # those that are derived from a delegate_belongs_to
119
+ #
120
+ # it "should populate column types for assignment operators in method details" do
121
+ #
122
+ # DataShift::ModelMethodsManager.find_methods( Project )
123
+ #
124
+ # DataShift::MethodDictionary.build_method_details( Project )
125
+ #
126
+ # [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
127
+ #
128
+ # method_details = DataShift::MethodDictionary.find_method_detail( Project, format )
129
+ #
130
+ # method_details.class.should == DataShift::ModelMethod
131
+ #
132
+ # method_details.col_type.should_not be_nil
133
+ # method_details.col_type.name.should == 'value_as_string'
134
+ # method_details.col_type.default.should == nil
135
+ # method_details.col_type.sql_type.should include 'varchar(255)' # db specific, sqlite
136
+ # method_details.col_type.type.should == :string
137
+ # end
138
+ # end
139
+ #
140
+ # it "should populate required Class for assignment operators based on column type" do
141
+ #
142
+ # DataShift::ModelMethodsManager.find_methods( Project )
143
+ #
144
+ # DataShift::MethodDictionary.build_method_details( Project )
145
+ #
146
+ # [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
147
+ #
148
+ # method_details = DataShift::MethodDictionary.find_method_detail( Project, format )
149
+ #
150
+ # method_details.operator_class_name.should == 'String'
151
+ # method_details.operator_class.should be_is_a(Class)
152
+ # method_details.operator_class.should == String
153
+ # end
154
+ #
155
+ # end
156
+ #
157
+ #
158
+ # it "should populate method_details on Class for belongs_to" do
159
+ #
160
+ # DataShift::ModelMethodsManager.find_methods( Owner )
161
+ #
162
+ # DataShift::MethodDictionary.build_method_details( Owner )
163
+ #
164
+ # [:project, 'PROJECT'].each do |format|
165
+ #
166
+ # method_details = DataShift::MethodDictionary.find_method_detail( Owner, format )
167
+ #
168
+ # expect(method_details.operator_class_name).to eq 'Project'
169
+ # expect(method_details.operator_class).to be_a(Class)
170
+ # expect(method_details.operator_class).to eq Project
171
+ # end
172
+ #
173
+ # end
174
+ #
175
+ #
176
+ # it "should populate belongs_to operator for method details for different forms of a column name" do
177
+ #
178
+ # DataShift::ModelMethodsManager.find_methods( Project )
179
+ # DataShift::ModelMethodsManager.find_methods( Milestone )
180
+ #
181
+ # DataShift::MethodDictionary.build_method_details( Project )
182
+ # DataShift::MethodDictionary.build_method_details( Milestone )
183
+ #
184
+ # # milestone.project = project.id
185
+ # [:project, 'project', "PROJECT", "prOJECt"].each do |format|
186
+ #
187
+ # method_details = DataShift::MethodDictionary.find_method_detail( Milestone, format )
188
+ #
189
+ # method_details.should_not be_nil
190
+ #
191
+ # method_details.operator.should == 'project'
192
+ # method_details.operator_for(:belongs_to).should == 'project'
193
+ #
194
+ # method_details.operator_for(:assignment).should be_nil
195
+ # method_details.operator_for(:has_many).should be_nil
196
+ # end
197
+ #
198
+ # end
199
+ #
200
+ # it "should populate required Class for belongs_to operator method details" do
201
+ #
202
+ # DataShift::ModelMethodsManager.find_methods( LoaderRelease )
203
+ # DataShift::ModelMethodsManager.find_methods( LongAndComplexTableLinkedToVersion )
204
+ #
205
+ # DataShift::MethodDictionary.build_method_details( LoaderRelease )
206
+ # DataShift::MethodDictionary.build_method_details( LongAndComplexTableLinkedToVersion )
207
+ #
208
+ #
209
+ # # release.project = project.id
210
+ # [:project, 'project', "PROJECT", "prOJECt"].each do |format|
211
+ #
212
+ # method_details = DataShift::MethodDictionary.find_method_detail( LoaderRelease, format )
213
+ #
214
+ # method_details.operator_class_name.should == 'Project'
215
+ # method_details.operator_class.should == Project
216
+ # end
217
+ #
218
+ #
219
+ # [:version, "Version", "verSION"].each do |format|
220
+ # method_details = DataShift::MethodDictionary.find_method_detail( LongAndComplexTableLinkedToVersion, format )
221
+ #
222
+ # method_details.operator_type.should == :belongs_to
223
+ #
224
+ # method_details.operator_class_name.should == 'Version'
225
+ # method_details.operator_class.should == Version
226
+ # end
227
+ # end
228
+ #
229
+ # it "should populate required Class for has_one operator method details" do
230
+ #
231
+ # DataShift::ModelMethodsManager.find_methods( Version )
232
+ # DataShift::MethodDictionary.build_method_details( Version )
233
+ #
234
+ # # version.long_and_complex_table_linked_to_version = LongAndComplexTableLinkedToVersion.create()
235
+ #
236
+ # [:long_and_complex_table_linked_to_version, 'LongAndComplexTableLinkedToVersion', "Long And Complex_Table_Linked To Version", "Long_And_Complex_Table_Linked_To_Version"].each do |format|
237
+ # method_details = DataShift::MethodDictionary.find_method_detail( Version, format )
238
+ #
239
+ # method_details.should_not be_nil
240
+ #
241
+ # method_details.operator.should == 'long_and_complex_table_linked_to_version'
242
+ #
243
+ # method_details.operator_type.should == :has_one
244
+ #
245
+ # method_details.operator_class_name.should == 'LongAndComplexTableLinkedToVersion'
246
+ # method_details.operator_class.should == LongAndComplexTableLinkedToVersion
247
+ # end
248
+ # end
249
+ #
250
+ # it "should return false on for?(klass) if find_operators hasn't been called for klass" do
251
+ # DataShift::MethodDictionary.clear
252
+ # DataShift::MethodDictionary::for?(Project).should == false
253
+ # DataShift::MethodDictionary::find_operators(Project)
254
+ # DataShift::MethodDictionary::for?(Project).should == true
255
+ #
256
+ # end
257
+ #
258
+ # it "should return false on for?(klass) if find_operators hasn't been called for klass" do
259
+ # DataShift::MethodDictionary.clear
260
+ # DataShift::MethodDictionary::for?(Project).should == false
261
+ # end
262
+ #
263
+ # it "should find has_many operator for method details" do
264
+ #
265
+ # DataShift::ModelMethodsManager.find_methods( Project )
266
+ #
267
+ # DataShift::MethodDictionary.build_method_details( Project )
268
+ #
269
+ # [:milestones, "Mile Stones", 'mileSTONES', 'MileStones'].each do |format|
270
+ #
271
+ # method_details = DataShift::MethodDictionary.find_method_detail( Project, format )
272
+ #
273
+ # method_details.class.should == DataShift::ModelMethod
274
+ #
275
+ # result = 'milestones'
276
+ # method_details.operator.should == result
277
+ # method_details.operator_for(:has_many).should == result
278
+ #
279
+ # method_details.operator_for(:belongs_to).should be_nil
280
+ # method_details.operator_for(:assignments).should be_nil
281
+ # end
282
+ #
283
+ # end
284
+ #
285
+ #
286
+ # it "should return nil when non existent column name" do
287
+ #
288
+ # DataShift::ModelMethodsManager.find_methods( Project )
289
+ #
290
+ # ["On sale", 'on_sale'].each do |format|
291
+ # detail = DataShift::MethodDictionary.find_method_detail( Project, format )
292
+ #
293
+ # detail.should be_nil
294
+ # end
295
+ # end
296
+ #
297
+ #
298
+ # it "should not by default map setter methods" do
299
+ # DataShift::ModelMethodsManager.find_methods( Milestone )
300
+ #
301
+ # DataShift::MethodDictionary.assignments[Milestone].should_not include('title')
302
+ # end
303
+ #
304
+ # it "should support inclusion of setter methods" do
305
+ #
306
+ # Milestone.new.respond_to?('title=').should == true
307
+ # Milestone.new.respond_to?('milestone_setter=').should == true
308
+ #
309
+ # DataShift::MethodDictionary.setters( Milestone ).should include('title=') # delegate
310
+ # DataShift::MethodDictionary.setters( Milestone ).should include('milestone_setter=') # normal method
311
+ # end
312
+ #
313
+ # it "should support reload and inclusion of setter methods" do
314
+ #
315
+ # DataShift::ModelMethodsManager.find_methods( Project )
316
+ # DataShift::ModelMethodsManager.find_methods( Milestone )
317
+ #
318
+ # DataShift::MethodDictionary.assignments[Milestone].should_not include('title')
319
+ #
320
+ # DataShift::ModelMethodsManager.find_methods( Milestone, :reload => true, :instance_methods => true )
321
+ #
322
+ # # Milestone delegates :title to Project
323
+ # DataShift::MethodDictionary.assignments[Milestone].should include('title')
324
+ # DataShift::MethodDictionary.assignments[Milestone].should include('milestone_setter')
325
+ # end
326
+ end
327
+ end
328
+ end
329
+ end