datashift 0.16.0 → 0.40.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (422) hide show
  1. checksums.yaml +4 -4
  2. data/{LICENSE.txt → LICENSE} +0 -0
  3. data/Rakefile +1 -20
  4. data/datashift.thor +125 -0
  5. data/lib/applications/apache_poi_extensions.rb +21 -52
  6. data/lib/applications/excel.rb +64 -57
  7. data/lib/applications/hssf_row_extensions.rb +66 -0
  8. data/lib/applications/jexcel_file.rb +99 -95
  9. data/lib/applications/jexcel_file_extensions.rb +76 -83
  10. data/lib/applications/jruby/word.rb +36 -36
  11. data/lib/applications/ruby_poi_translations.rb +34 -32
  12. data/lib/applications/spreadsheet_extensions.rb +21 -19
  13. data/lib/datashift.rb +49 -59
  14. data/lib/datashift/binder.rb +217 -0
  15. data/lib/datashift/column_packer.rb +21 -72
  16. data/lib/datashift/configuration.rb +317 -0
  17. data/lib/datashift/context_factory.rb +88 -0
  18. data/lib/datashift/core_ext/array.rb +15 -0
  19. data/lib/datashift/core_ext/csv_ext.rb +46 -0
  20. data/lib/datashift/core_ext/string.rb +49 -0
  21. data/lib/datashift/core_ext/to_b.rb +11 -0
  22. data/lib/datashift/delimiters.rb +55 -61
  23. data/lib/datashift/doc_context.rb +137 -0
  24. data/lib/datashift/excel_base.rb +93 -81
  25. data/lib/datashift/exceptions.rb +30 -28
  26. data/lib/datashift/file_definitions.rb +44 -39
  27. data/lib/datashift/guards.rb +5 -5
  28. data/lib/datashift/header.rb +25 -0
  29. data/lib/datashift/headers.rb +94 -0
  30. data/lib/datashift/inbound_data/column.rb +44 -0
  31. data/lib/datashift/inbound_data/lookup_support.rb +33 -0
  32. data/lib/datashift/inbound_data/method_binding.rb +139 -0
  33. data/lib/datashift/load_object.rb +37 -12
  34. data/lib/datashift/logging.rb +54 -27
  35. data/lib/datashift/mandatory.rb +39 -0
  36. data/lib/datashift/mapping/data_flow_schema.rb +198 -0
  37. data/lib/datashift/{model_mapper.rb → mapping/mapper_utils.rb} +30 -10
  38. data/lib/datashift/model_methods/catalogue.rb +183 -0
  39. data/lib/datashift/model_methods/collection.rb +140 -0
  40. data/lib/datashift/model_methods/model_method.rb +162 -0
  41. data/lib/datashift/model_methods/model_methods_manager.rb +76 -0
  42. data/lib/datashift/model_methods/operator.rb +62 -0
  43. data/lib/datashift/node_collection.rb +26 -0
  44. data/lib/datashift/node_context.rb +68 -0
  45. data/lib/datashift/populator.rb +308 -282
  46. data/lib/datashift/progress_monitor.rb +91 -0
  47. data/lib/datashift/querying.rb +110 -52
  48. data/lib/datashift/templates/import_export_config.erb +55 -0
  49. data/lib/datashift/transformation/factory.rb +219 -0
  50. data/lib/datashift/transformation/remove.rb +44 -0
  51. data/lib/datashift/version.rb +3 -0
  52. data/lib/exporters/configuration.rb +84 -0
  53. data/lib/exporters/csv_exporter.rb +54 -52
  54. data/lib/exporters/excel_exporter.rb +80 -61
  55. data/lib/exporters/exporter_base.rb +8 -8
  56. data/lib/generators/config_generator.rb +218 -0
  57. data/lib/generators/csv_generator.rb +13 -70
  58. data/lib/generators/excel_generator.rb +23 -111
  59. data/lib/generators/generator_base.rb +15 -70
  60. data/lib/loaders/configuration.rb +90 -0
  61. data/lib/loaders/csv_loader.rb +63 -101
  62. data/lib/loaders/excel_loader.rb +71 -156
  63. data/lib/loaders/failure_data.rb +40 -0
  64. data/lib/loaders/file_loader.rb +16 -0
  65. data/lib/loaders/loader_base.rb +82 -410
  66. data/lib/loaders/loader_factory.rb +42 -0
  67. data/lib/loaders/paperclip/attachment_loader.rb +157 -140
  68. data/lib/loaders/paperclip/datashift_paperclip.rb +18 -35
  69. data/lib/loaders/paperclip/image_loading.rb +40 -35
  70. data/lib/loaders/reporters/basic_stdout_reporter.rb +40 -0
  71. data/lib/loaders/reporters/reporter.rb +26 -0
  72. data/lib/tasks/config.thor +65 -0
  73. data/{tasks → lib/tasks}/config/seed_fu_product_template.erb +0 -0
  74. data/{tasks → lib/tasks}/config/tidy_config.txt +0 -0
  75. data/lib/tasks/export.thor +192 -0
  76. data/lib/tasks/generate.thor +190 -0
  77. data/lib/tasks/import.thor +142 -0
  78. data/lib/{thor → tasks}/paperclip.thor +69 -69
  79. data/{tasks → lib/tasks/to_convert_to_thor}/db_tasks.rake +20 -20
  80. data/lib/tasks/tools.thor +109 -0
  81. data/spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg +0 -0
  82. data/spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg +0 -0
  83. data/spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg +0 -0
  84. data/spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg +0 -0
  85. data/spec/datashift/binder_spec.rb +266 -0
  86. data/spec/datashift/config_generator_spec.rb +186 -0
  87. data/spec/datashift/configuration.rb +66 -0
  88. data/spec/datashift/context_factory_spec.rb +63 -0
  89. data/spec/datashift/data_flow_schema_spec.rb +150 -0
  90. data/spec/datashift/datashift_spec.rb +52 -0
  91. data/spec/datashift/excel_base_spec.rb +57 -0
  92. data/spec/datashift/excel_spec.rb +188 -0
  93. data/spec/datashift/failure_data_spec.rb +27 -0
  94. data/spec/{file_definitions.rb → datashift/file_definitions.rb} +9 -10
  95. data/spec/datashift/headers_spec.rb +56 -0
  96. data/spec/datashift/inbound_data_spec.rb +47 -0
  97. data/spec/datashift/mapper_utils_spec.rb +38 -0
  98. data/spec/datashift/method_binding_spec.rb +60 -0
  99. data/spec/datashift/model_method_spec.rb +109 -0
  100. data/spec/datashift/model_methods_catalogue.rb +111 -0
  101. data/spec/datashift/model_methods_collection_spec.rb +138 -0
  102. data/spec/datashift/model_methods_manager_spec.rb +329 -0
  103. data/spec/datashift/populator_spec.rb +117 -0
  104. data/spec/datashift/thor_spec.rb +314 -0
  105. data/spec/datashift/transformation/factory_spec.rb +195 -0
  106. data/spec/datashift/transformation/transformer_remove_spec.rb +43 -0
  107. data/spec/dummy/Gemfile +53 -0
  108. data/spec/dummy/Gemfile.lock +197 -0
  109. data/spec/dummy/README.rdoc +28 -0
  110. data/spec/dummy/Rakefile +6 -0
  111. data/spec/dummy/app/assets/javascripts/application.js +16 -0
  112. data/spec/dummy/app/assets/javascripts/categories.js +2 -0
  113. data/spec/dummy/app/assets/javascripts/digitals.js +2 -0
  114. data/spec/dummy/app/assets/javascripts/empties.js +2 -0
  115. data/spec/dummy/app/assets/javascripts/loader_releases.js +2 -0
  116. data/spec/dummy/app/assets/javascripts/long_and_complex_table_linked_to_versions.js +2 -0
  117. data/spec/dummy/app/assets/javascripts/milestones.js +2 -0
  118. data/spec/dummy/app/assets/javascripts/owners.js +2 -0
  119. data/spec/dummy/app/assets/javascripts/projects.js +2 -0
  120. data/spec/dummy/app/assets/javascripts/users.js +2 -0
  121. data/spec/dummy/app/assets/javascripts/versions.js +2 -0
  122. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  123. data/spec/dummy/app/assets/stylesheets/categories.css +4 -0
  124. data/spec/dummy/app/assets/stylesheets/digitals.css +4 -0
  125. data/spec/dummy/app/assets/stylesheets/empties.css +4 -0
  126. data/spec/dummy/app/assets/stylesheets/loader_releases.css +4 -0
  127. data/spec/dummy/app/assets/stylesheets/long_and_complex_table_linked_to_versions.css +4 -0
  128. data/spec/dummy/app/assets/stylesheets/milestones.css +4 -0
  129. data/spec/dummy/app/assets/stylesheets/owners.css +4 -0
  130. data/spec/dummy/app/assets/stylesheets/projects.css +4 -0
  131. data/spec/dummy/app/assets/stylesheets/scaffold.css +56 -0
  132. data/spec/dummy/app/assets/stylesheets/users.css +4 -0
  133. data/spec/dummy/app/assets/stylesheets/versions.css +4 -0
  134. data/spec/dummy/app/controllers/application_controller.rb +5 -0
  135. data/spec/dummy/app/controllers/categories_controller.rb +58 -0
  136. data/spec/dummy/app/controllers/digitals_controller.rb +58 -0
  137. data/spec/dummy/app/controllers/empties_controller.rb +58 -0
  138. data/spec/dummy/app/controllers/loader_releases_controller.rb +58 -0
  139. data/spec/dummy/app/controllers/long_and_complex_table_linked_to_versions_controller.rb +58 -0
  140. data/spec/dummy/app/controllers/milestones_controller.rb +58 -0
  141. data/spec/dummy/app/controllers/owners_controller.rb +58 -0
  142. data/spec/dummy/app/controllers/projects_controller.rb +58 -0
  143. data/spec/dummy/app/controllers/users_controller.rb +58 -0
  144. data/spec/dummy/app/controllers/versions_controller.rb +58 -0
  145. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  146. data/spec/dummy/app/helpers/categories_helper.rb +2 -0
  147. data/spec/dummy/app/helpers/digitals_helper.rb +2 -0
  148. data/spec/dummy/app/helpers/empties_helper.rb +2 -0
  149. data/spec/dummy/app/helpers/loader_releases_helper.rb +2 -0
  150. data/spec/dummy/app/helpers/long_and_complex_table_linked_to_versions_helper.rb +2 -0
  151. data/spec/dummy/app/helpers/milestones_helper.rb +2 -0
  152. data/spec/dummy/app/helpers/owners_helper.rb +2 -0
  153. data/spec/dummy/app/helpers/projects_helper.rb +2 -0
  154. data/spec/dummy/app/helpers/users_helper.rb +2 -0
  155. data/spec/dummy/app/helpers/versions_helper.rb +2 -0
  156. data/spec/dummy/app/models/category.rb +6 -0
  157. data/spec/dummy/app/models/digital.rb +22 -0
  158. data/spec/dummy/app/models/empty.rb +2 -0
  159. data/spec/dummy/app/models/loader_release.rb +10 -0
  160. data/spec/dummy/app/models/long_and_complex_table_linked_to_version.rb +6 -0
  161. data/spec/dummy/app/models/milestone.rb +15 -0
  162. data/spec/dummy/app/models/owner.rb +13 -0
  163. data/spec/dummy/app/models/project.rb +53 -0
  164. data/spec/dummy/app/models/user.rb +5 -0
  165. data/spec/dummy/app/models/version.rb +7 -0
  166. data/spec/dummy/app/views/categories/_form.html.erb +17 -0
  167. data/spec/dummy/app/views/categories/edit.html.erb +6 -0
  168. data/spec/dummy/app/views/categories/index.html.erb +25 -0
  169. data/spec/dummy/app/views/categories/new.html.erb +5 -0
  170. data/spec/dummy/app/views/categories/show.html.erb +4 -0
  171. data/spec/dummy/app/views/digitals/_form.html.erb +17 -0
  172. data/spec/dummy/app/views/digitals/edit.html.erb +6 -0
  173. data/spec/dummy/app/views/digitals/index.html.erb +25 -0
  174. data/spec/dummy/app/views/digitals/new.html.erb +5 -0
  175. data/spec/dummy/app/views/digitals/show.html.erb +4 -0
  176. data/spec/dummy/app/views/empties/_form.html.erb +17 -0
  177. data/spec/dummy/app/views/empties/edit.html.erb +6 -0
  178. data/spec/dummy/app/views/empties/index.html.erb +25 -0
  179. data/spec/dummy/app/views/empties/new.html.erb +5 -0
  180. data/spec/dummy/app/views/empties/show.html.erb +4 -0
  181. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  182. data/spec/dummy/app/views/loader_releases/_form.html.erb +17 -0
  183. data/spec/dummy/app/views/loader_releases/edit.html.erb +6 -0
  184. data/spec/dummy/app/views/loader_releases/index.html.erb +25 -0
  185. data/spec/dummy/app/views/loader_releases/new.html.erb +5 -0
  186. data/spec/dummy/app/views/loader_releases/show.html.erb +4 -0
  187. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/_form.html.erb +17 -0
  188. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/edit.html.erb +6 -0
  189. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/index.html.erb +25 -0
  190. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/new.html.erb +5 -0
  191. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/show.html.erb +4 -0
  192. data/spec/dummy/app/views/milestones/_form.html.erb +17 -0
  193. data/spec/dummy/app/views/milestones/edit.html.erb +6 -0
  194. data/spec/dummy/app/views/milestones/index.html.erb +25 -0
  195. data/spec/dummy/app/views/milestones/new.html.erb +5 -0
  196. data/spec/dummy/app/views/milestones/show.html.erb +4 -0
  197. data/spec/dummy/app/views/owners/_form.html.erb +17 -0
  198. data/spec/dummy/app/views/owners/edit.html.erb +6 -0
  199. data/spec/dummy/app/views/owners/index.html.erb +25 -0
  200. data/spec/dummy/app/views/owners/new.html.erb +5 -0
  201. data/spec/dummy/app/views/owners/show.html.erb +4 -0
  202. data/spec/dummy/app/views/projects/_form.html.erb +17 -0
  203. data/spec/dummy/app/views/projects/edit.html.erb +6 -0
  204. data/spec/dummy/app/views/projects/index.html.erb +25 -0
  205. data/spec/dummy/app/views/projects/new.html.erb +5 -0
  206. data/spec/dummy/app/views/projects/show.html.erb +4 -0
  207. data/spec/dummy/app/views/users/_form.html.erb +17 -0
  208. data/spec/dummy/app/views/users/edit.html.erb +6 -0
  209. data/spec/dummy/app/views/users/index.html.erb +25 -0
  210. data/spec/dummy/app/views/users/new.html.erb +5 -0
  211. data/spec/dummy/app/views/users/show.html.erb +4 -0
  212. data/spec/dummy/app/views/versions/_form.html.erb +17 -0
  213. data/spec/dummy/app/views/versions/edit.html.erb +6 -0
  214. data/spec/dummy/app/views/versions/index.html.erb +25 -0
  215. data/spec/dummy/app/views/versions/new.html.erb +5 -0
  216. data/spec/dummy/app/views/versions/show.html.erb +4 -0
  217. data/spec/dummy/bin/bundle +3 -0
  218. data/spec/dummy/bin/rails +9 -0
  219. data/spec/dummy/bin/rake +9 -0
  220. data/spec/dummy/bin/setup +29 -0
  221. data/spec/dummy/bin/spring +16 -0
  222. data/spec/dummy/config.ru +4 -0
  223. data/spec/dummy/config/application.rb +26 -0
  224. data/spec/dummy/config/boot.rb +3 -0
  225. data/spec/dummy/config/database.yml +25 -0
  226. data/spec/dummy/config/environment.rb +5 -0
  227. data/spec/dummy/config/environments/development.rb +41 -0
  228. data/spec/dummy/config/environments/production.rb +79 -0
  229. data/spec/dummy/config/environments/test.rb +42 -0
  230. data/spec/dummy/config/initializers/assets.rb +11 -0
  231. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  232. data/spec/dummy/config/initializers/cookies_serializer.rb +3 -0
  233. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  234. data/spec/dummy/config/initializers/inflections.rb +16 -0
  235. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  236. data/spec/dummy/config/initializers/session_store.rb +3 -0
  237. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  238. data/spec/dummy/config/locales/en.yml +23 -0
  239. data/spec/dummy/config/routes.rb +76 -0
  240. data/spec/dummy/config/secrets.yml +22 -0
  241. data/spec/dummy/db/development.sqlite3 +0 -0
  242. data/spec/dummy/db/migrate/20110803201325_create_test_bed.rb +98 -0
  243. data/spec/dummy/db/migrate/20121009161700_add_digitals.rb +24 -0
  244. data/spec/dummy/db/migrate/20161005123106_create_digitals.rb +8 -0
  245. data/spec/dummy/db/migrate/20161005123106_create_long_and_complex_table_linked_to_versions.rb +8 -0
  246. data/spec/dummy/db/migrate/20161005123107_create_loader_releases.rb +8 -0
  247. data/spec/dummy/db/migrate/20161005123108_create_owners.rb +8 -0
  248. data/spec/dummy/db/migrate/20161005123109_create_empties.rb +8 -0
  249. data/spec/dummy/db/migrate/20161005123110_create_projects.rb +8 -0
  250. data/spec/dummy/db/migrate/20161005123111_create_users.rb +8 -0
  251. data/spec/dummy/db/migrate/20161005123111_create_versions.rb +8 -0
  252. data/spec/dummy/db/migrate/20161005123112_create_milestones.rb +8 -0
  253. data/spec/dummy/db/migrate/20161005123113_create_categories.rb +8 -0
  254. data/spec/dummy/db/schema.rb +93 -0
  255. data/spec/dummy/db/seeds.rb +9 -0
  256. data/spec/dummy/db/test.sqlite3 +0 -0
  257. data/spec/dummy/log/test.log +69 -0
  258. data/spec/dummy/public/404.html +67 -0
  259. data/spec/dummy/public/422.html +67 -0
  260. data/spec/dummy/public/500.html +66 -0
  261. data/spec/dummy/public/favicon.ico +0 -0
  262. data/spec/dummy/public/robots.txt +5 -0
  263. data/spec/dummy/sandbox_example.thor +4 -0
  264. data/spec/dummy/test/controllers/categories_controller_test.rb +49 -0
  265. data/spec/dummy/test/controllers/digitals_controller_test.rb +49 -0
  266. data/spec/dummy/test/controllers/empties_controller_test.rb +49 -0
  267. data/spec/dummy/test/controllers/loader_releases_controller_test.rb +49 -0
  268. data/spec/dummy/test/controllers/long_and_complex_table_linked_to_versions_controller_test.rb +49 -0
  269. data/spec/dummy/test/controllers/milestones_controller_test.rb +49 -0
  270. data/spec/dummy/test/controllers/owners_controller_test.rb +49 -0
  271. data/spec/dummy/test/controllers/projects_controller_test.rb +49 -0
  272. data/spec/dummy/test/controllers/users_controller_test.rb +49 -0
  273. data/spec/dummy/test/controllers/versions_controller_test.rb +49 -0
  274. data/spec/dummy/test/factories/categories.rb +5 -0
  275. data/spec/dummy/test/factories/digitals.rb +5 -0
  276. data/spec/dummy/test/factories/empties.rb +5 -0
  277. data/spec/dummy/test/factories/loader_releases.rb +5 -0
  278. data/spec/dummy/test/factories/long_and_complex_table_linked_to_versions.rb +5 -0
  279. data/spec/dummy/test/factories/milestones.rb +5 -0
  280. data/spec/dummy/test/factories/owners.rb +5 -0
  281. data/spec/dummy/test/factories/projects.rb +5 -0
  282. data/spec/dummy/test/factories/users.rb +5 -0
  283. data/spec/dummy/test/factories/versions.rb +5 -0
  284. data/spec/dummy/test/models/category_test.rb +7 -0
  285. data/spec/dummy/test/models/digital_test.rb +7 -0
  286. data/spec/dummy/test/models/empty_test.rb +7 -0
  287. data/spec/dummy/test/models/loader_release_test.rb +7 -0
  288. data/spec/dummy/test/models/long_and_complex_table_linked_to_version_test.rb +7 -0
  289. data/spec/dummy/test/models/milestone_test.rb +7 -0
  290. data/spec/dummy/test/models/owner_test.rb +7 -0
  291. data/spec/dummy/test/models/project_test.rb +7 -0
  292. data/spec/dummy/test/models/user_test.rb +7 -0
  293. data/spec/dummy/test/models/version_test.rb +7 -0
  294. data/spec/dummy/test/test_helper.rb +10 -0
  295. data/spec/exporters/csv_exporter_spec.rb +240 -0
  296. data/spec/exporters/csv_generator_spec.rb +139 -0
  297. data/spec/exporters/excel_exporter_spec.rb +193 -0
  298. data/spec/exporters/excel_generator_spec.rb +181 -0
  299. data/spec/exporters/generator_base_spec.rb +45 -0
  300. data/spec/factories/categories.rb +7 -0
  301. data/spec/factories/factories.rb +18 -0
  302. data/spec/factories/milestone.rb +16 -0
  303. data/spec/factories/projects.rb +41 -0
  304. data/spec/fixtures/BadAssociationName.xls +0 -0
  305. data/spec/fixtures/DemoNegativeTesting.xls +0 -0
  306. data/spec/fixtures/ProjectConfiguration.yml +18 -0
  307. data/spec/fixtures/ProjectsMultiCategories.xls +0 -0
  308. data/spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls +0 -0
  309. data/spec/fixtures/ProjectsSingleCategories.xls +0 -0
  310. data/spec/fixtures/ProjectsSingleCategories.xlsx +0 -0
  311. data/spec/fixtures/SimpleProjects.xls +0 -0
  312. data/spec/fixtures/config/database.yml +28 -0
  313. data/spec/fixtures/csv/BadAssociationName.csv +6 -0
  314. data/spec/fixtures/csv/DemoNegativeTesting.csv +6 -0
  315. data/spec/fixtures/csv/ProjectsMultiCategories.csv +5 -0
  316. data/spec/fixtures/csv/ProjectsMultiCategoriesHeaderLookup.csv +5 -0
  317. data/spec/fixtures/csv/ProjectsSingleCategories.csv +5 -0
  318. data/spec/fixtures/csv/SimpleProjects.csv +4 -0
  319. data/spec/fixtures/db/migrate/20110803201325_create_test_bed.rb +98 -0
  320. data/spec/fixtures/db/migrate/20121009161700_add_digitals.rb +24 -0
  321. data/spec/fixtures/db/seeds.rb +9 -0
  322. data/spec/fixtures/images/DEMO_001_ror_bag.jpeg +0 -0
  323. data/spec/fixtures/images/DEMO_002_Powerstation.jpeg +0 -0
  324. data/spec/fixtures/images/DEMO_003_ror_mug.jpeg +0 -0
  325. data/spec/fixtures/images/DEMO_004_ror_ringer.jpeg +0 -0
  326. data/spec/fixtures/load_datashift.thor +3 -0
  327. data/spec/fixtures/models/category.rb +6 -0
  328. data/spec/fixtures/models/digital.rb +22 -0
  329. data/spec/fixtures/models/empty.rb +2 -0
  330. data/spec/fixtures/models/loader_release.rb +10 -0
  331. data/spec/fixtures/models/long_and_complex_table_linked_to_version.rb +6 -0
  332. data/spec/fixtures/models/milestone.rb +15 -0
  333. data/spec/fixtures/models/owner.rb +13 -0
  334. data/spec/fixtures/models/project.rb +53 -0
  335. data/spec/fixtures/models/user.rb +5 -0
  336. data/spec/fixtures/models/version.rb +7 -0
  337. data/spec/fixtures/results/exp_project_assoc_headers.xls +0 -0
  338. data/spec/fixtures/results/exp_project_collection_spec.csv +2 -0
  339. data/spec/fixtures/results/exp_project_export.xls +0 -0
  340. data/spec/fixtures/results/exp_project_first_export.xls +0 -0
  341. data/spec/fixtures/results/exp_project_plus_assoc.xls +0 -0
  342. data/spec/fixtures/results/exp_project_plus_assoc_export_spec.csv +9 -0
  343. data/spec/fixtures/results/gen_project_plus_assoc_template.xls +0 -0
  344. data/spec/fixtures/results/gen_project_plus_some_assoc_template.xls +0 -0
  345. data/spec/fixtures/results/gen_project_template.xls +0 -0
  346. data/spec/fixtures/results/project_and_assoc_in_hash_export.xls +0 -0
  347. data/spec/fixtures/results/project_and_assoc_in_json_export.csv +9 -0
  348. data/spec/fixtures/results/project_and_assoc_in_json_export.xls +0 -0
  349. data/spec/fixtures/results/project_export_spec_with_custom_delim_,.csv +2 -0
  350. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/243.csv +2 -0
  351. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/247.csv +2 -0
  352. data/spec/fixtures/results/project_plus_assoc_template.csv +1 -0
  353. data/spec/fixtures/results/project_plus_some_assoc_template.csv +1 -0
  354. data/spec/fixtures/results/project_remove_export_spec.csv +2 -0
  355. data/spec/fixtures/results/project_template.csv +1 -0
  356. data/spec/fixtures/results/project_with_methods_export_spec.csv +2 -0
  357. data/spec/fixtures/results/thor_spec_gen_project.csv +1 -0
  358. data/spec/fixtures/sandbox_example.thor +4 -0
  359. data/spec/fixtures/simple_export_spec.xls +0 -0
  360. data/spec/fixtures/simple_template_spec.xls +0 -0
  361. data/spec/fixtures/test_model_defs.rb +7 -0
  362. data/spec/loaders/csv_loader_spec.rb +206 -0
  363. data/spec/loaders/data_flow_excel_loader_spec.rb +290 -0
  364. data/spec/loaders/excel_loader_failures_spec.rb +67 -0
  365. data/spec/loaders/excel_loader_spec.rb +294 -0
  366. data/spec/loaders/loader_base_spec.rb +29 -0
  367. data/spec/loaders/paperclip_loader_spec.rb +106 -0
  368. data/spec/log/datashift.log +14930 -0
  369. data/spec/private/digitals/1/DEMO_003_ror_mug.jpeg +0 -0
  370. data/spec/private/digitals/2/DEMO_002_Powerstation.jpeg +0 -0
  371. data/spec/private/digitals/3/DEMO_004_ror_ringer.jpeg +0 -0
  372. data/spec/private/digitals/4/DEMO_001_ror_bag.jpeg +0 -0
  373. data/spec/spec_helper.rb +26 -230
  374. data/spec/support/clear_and_manage_contexts.rb +25 -0
  375. data/spec/support/database_cleaner.rb +32 -0
  376. data/spec/support/datashift_test_helpers.rb +153 -0
  377. data/spec/support/files_paths_helper.rb +13 -0
  378. data/spec/support/fixtures/results/mapping_template.yaml +15 -0
  379. data/spec/support/sandbox.rb +136 -0
  380. metadata +804 -85
  381. data/README.markdown +0 -274
  382. data/README.rdoc +0 -19
  383. data/VERSION +0 -1
  384. data/datashift.gemspec +0 -48
  385. data/lib/applications/jruby/old_pre_proxy_jexcel_file.rb +0 -437
  386. data/lib/datashift/data_transforms.rb +0 -83
  387. data/lib/datashift/mapping_file_definitions.rb +0 -88
  388. data/lib/datashift/mapping_service.rb +0 -91
  389. data/lib/datashift/method_detail.rb +0 -165
  390. data/lib/datashift/method_details_manager.rb +0 -95
  391. data/lib/datashift/method_dictionary.rb +0 -281
  392. data/lib/datashift/method_mapper.rb +0 -174
  393. data/lib/datashift/thor_base.rb +0 -38
  394. data/lib/generators/mapping_generator.rb +0 -112
  395. data/lib/helpers/core_ext/csv_file.rb +0 -33
  396. data/lib/helpers/core_ext/to_b.rb +0 -24
  397. data/lib/loaders/reporter.rb +0 -58
  398. data/lib/thor/export.thor +0 -175
  399. data/lib/thor/generate.thor +0 -191
  400. data/lib/thor/import.thor +0 -110
  401. data/lib/thor/mapping.thor +0 -65
  402. data/lib/thor/tools.thor +0 -84
  403. data/spec/Gemfile +0 -31
  404. data/spec/Gemfile.lock +0 -134
  405. data/spec/csv_exporter_spec.rb +0 -144
  406. data/spec/csv_generator_spec.rb +0 -159
  407. data/spec/csv_loader_spec.rb +0 -212
  408. data/spec/datashift_spec.rb +0 -55
  409. data/spec/excel_exporter_spec.rb +0 -199
  410. data/spec/excel_generator_spec.rb +0 -203
  411. data/spec/excel_loader_spec.rb +0 -237
  412. data/spec/excel_spec.rb +0 -203
  413. data/spec/loader_base_spec.rb +0 -166
  414. data/spec/mapping_spec.rb +0 -117
  415. data/spec/method_dictionary_spec.rb +0 -300
  416. data/spec/method_mapper_spec.rb +0 -100
  417. data/spec/model_mapper_spec.rb +0 -41
  418. data/spec/paperclip_loader_spec.rb +0 -92
  419. data/spec/populator_spec.rb +0 -128
  420. data/spec/thor_spec.rb +0 -90
  421. data/tasks/file_tasks.rake +0 -37
  422. data/tasks/word_to_seedfu.rake +0 -167
@@ -0,0 +1,138 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2011
2
+ # Author :: Tom Statter
3
+ # Date :: Mar 2015
4
+ # License:: MIT
5
+ #
6
+ # Details:: Specs for ModelMapperManager and associated DataShift::ModelMethodsMgrs::Dictionary
7
+ #
8
+ require File.join(File.dirname(__FILE__), '/../spec_helper')
9
+
10
+ module DataShift
11
+
12
+ module ModelMethods
13
+
14
+ describe 'Collection' do
15
+
16
+ before(:all) do
17
+ DataShift::ModelMethods::Catalogue.clear
18
+ DataShift::ModelMethods::Manager.clear
19
+ end
20
+
21
+ context 'Creation' do
22
+ it 'should be' do
23
+ expect(Collection.new( Project )).to be
24
+ end
25
+
26
+ it 'should manage method collection at class level' do
27
+ expect(Collection.new( Project ).managed_class).to eq Project
28
+ end
29
+
30
+ end
31
+
32
+ context 'Empty' do
33
+ let(:collection) { Collection.new(Project) }
34
+
35
+ it 'should report name of collected class' do
36
+ c = Collection.new( Project )
37
+ expect(c.managed_class).to eq Project
38
+ end
39
+
40
+ it 'should provide access to a set of model methods by type' do
41
+ expect(collection.by_optype).to be_a Hash
42
+ end
43
+
44
+ it 'should return nil when no such optype and_operator' do
45
+ expect(collection.find_by_name_and_type(:no_such_op, :belongs_to)).to be_nil
46
+ end
47
+ end
48
+
49
+ context 'Build a Collection' do
50
+
51
+ before(:all) do
52
+ Catalogue.clear
53
+ Manager.clear
54
+
55
+ Manager.catalog_class( Project )
56
+ end
57
+
58
+ let(:assignments) { Catalogue.assignments_for(Project) }
59
+
60
+ let(:name) { assignments.first }
61
+
62
+ let(:model_method) { DataShift::ModelMethod.new(Project, name, :assignment) }
63
+
64
+ let(:collection) { Collection.new(Project) }
65
+
66
+ before(:each) do
67
+ collection.add( model_method )
68
+ end
69
+
70
+ it 'should provide access to complete set of model methods for a class' do
71
+ expect(collection.model_method_list).to be_a Array
72
+ end
73
+
74
+ it 'should provide access to iterate through the complete set of model methods' do
75
+ expect(collection.respond_to? :each).to eq true
76
+ end
77
+
78
+ it 'should enable adding model methods to create manager' do
79
+ expect(collection.model_method_list.size).to be 1
80
+ expect(collection.by_optype.size).to be 1
81
+ end
82
+
83
+ it 'should enable extraction of details from manager' do
84
+
85
+ collection.add( DataShift::ModelMethod.new(Project, assignments[1], :assignment) )
86
+
87
+ expect(collection.for_type(:assignment).size).to eq 2
88
+ expect(collection.for_type(:assignment).first).to be_a ModelMethod
89
+
90
+ expect(collection.by_optype[:assignment]).to be_a Array
91
+ expect(collection.by_optype[:assignment].size).to eq 2
92
+
93
+ expect(collection.by_optype[:assignment].include?(model_method)).to eq true
94
+ expect(collection.by_optype[:assignment].first.operator).to eq name
95
+ end
96
+
97
+ it 'should enable finding specific operators' do
98
+ expect( collection.find_by_name_and_type(name, :assignment ) ).to eq model_method
99
+ end
100
+ end
101
+
102
+ context 'Full Collection from Class and operations' do
103
+
104
+ let(:collection) { ModelMethods::Manager.catalog_class(Project) }
105
+
106
+ it 'should be sortable based on operator and op type' do
107
+
108
+ expect(collection).to be_a DataShift::ModelMethods::Collection
109
+
110
+ collection.unshift( DataShift::ModelMethod.new(Project, 'aaa', :has_many) )
111
+ collection.add( DataShift::ModelMethod.new(Project, 'aaa', :assignment) )
112
+ collection.unshift( DataShift::ModelMethod.new(Project, 'zzz', :assignment) )
113
+
114
+ expect(collection.first.operator).to eq 'zzz'
115
+ expect(collection.last.operator_type? :assignment).to eq true
116
+
117
+ collection.sort!
118
+
119
+ expect(collection.first.operator).to eq 'aaa'
120
+ expect(collection.last.operator_type? :has_many).to eq true
121
+ end
122
+
123
+
124
+ it 'should be Searchable across op types' do
125
+ names = Project.reflect_on_all_associations(:belongs_to).map { |i| i.name.to_s }
126
+
127
+ mm = collection.search( names.last )
128
+
129
+ expect(mm.operator).to eq names.last
130
+ expect(mm.operator_type? :belongs_to).to eq true
131
+ end
132
+
133
+ end
134
+ end
135
+
136
+ end
137
+
138
+ end
@@ -0,0 +1,329 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2015
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2015
4
+ # License:: MIT
5
+ #
6
+ #
7
+ require File.join(File.dirname(__FILE__), '/../spec_helper')
8
+
9
+ module DataShift
10
+
11
+ module ModelMethods
12
+
13
+ describe ':ModelMethods ManagerDictionary' do
14
+ include_context 'ClearAllCatalogues'
15
+
16
+ it 'should provide a dictionary of class => manager' do
17
+ expect(Manager.collections).to be_a Hash
18
+ expect(Manager.collections.empty?).to eq true
19
+ end
20
+
21
+ it 'should provide a dictionary of class => manager' do
22
+ expect(Manager.collections[Milestone]).to be_nil
23
+ end
24
+
25
+ it 'should provide facilities to check a class been collected' do
26
+ expect(Manager.for?(Milestone)).to eq false
27
+ end
28
+
29
+ it 'should return nil as an empty collection' do
30
+ expect(Manager.for(Milestone)).to be nil
31
+ end
32
+
33
+ it 'should enable a class to be collected and return new collection' do
34
+ expect(Manager.catalog_class(Milestone)).to be_a Collection
35
+ end
36
+
37
+ context 'Cataloged' do
38
+ before(:each) do
39
+ Manager.catalog_class(Milestone)
40
+ end
41
+
42
+ it 'should report when a class has been collected' do
43
+ expect(Manager.for?(Milestone)).to eq true
44
+ end
45
+
46
+ it 'should provide facilities to return a populated collection' do
47
+ expect(Manager.for(Milestone)).to be_a Collection
48
+ end
49
+
50
+ it 'should provide access to a collection' do
51
+ expect(Manager.collections[Milestone]).to be_a Collection
52
+
53
+ expect(Manager.collections[Milestone]).to eq Manager.for(Milestone)
54
+ end
55
+
56
+ it 'should populate assignment operators for method details for different forms of a column name' do
57
+ Manager.catalog_class(Project)
58
+ Manager.catalog_class(Milestone)
59
+
60
+ collection = Manager.for(Project)
61
+
62
+ expect(collection.managed_class).to eq Project
63
+
64
+ model_method = collection.find_by_name_and_type('value_as_string', :assignment)
65
+
66
+ expect(model_method).to be_a DataShift::ModelMethod
67
+
68
+ expect(model_method.operator).to eq 'value_as_string'
69
+ expect(model_method.operator_for(:assignment)).to eq 'value_as_string'
70
+
71
+ expect(model_method.operator?('value_as_string')).to eq true
72
+ expect(model_method.operator?('blah_as_string')).to eq false
73
+
74
+ expect(model_method.operator_for(:belongs_to)).to eq nil
75
+ expect(model_method.operator_for(:has_many)).to eq nil
76
+ end
77
+
78
+ # it "should populate method dictionary for a given AR model" do
79
+ #
80
+ # DataShift::ModelMethodsManager.find_methods( Project )
81
+ # DataShift::ModelMethodsManager.find_methods( Milestone )
82
+ #
83
+ # DataShift::MethodDictionary.has_many.should_not be_empty
84
+ # DataShift::MethodDictionary.has_many[Project].should include('milestones')
85
+ #
86
+ # DataShift::MethodDictionary.assignments.should_not be_empty
87
+ # DataShift::MethodDictionary.assignments[Project].should include('id')
88
+ # DataShift::MethodDictionary.assignments[Project].should include('value_as_string')
89
+ # DataShift::MethodDictionary.assignments[Project].should include('value_as_text')
90
+ #
91
+ # DataShift::MethodDictionary.belongs_to.should_not be_empty
92
+ # expect(DataShift::MethodDictionary.belongs_to[Project]).to include 'user'
93
+ #
94
+ #
95
+ # DataShift::MethodDictionary.column_types.should be_is_a(Hash)
96
+ # DataShift::MethodDictionary.column_types.should_not be_empty
97
+ # DataShift::MethodDictionary.column_types[Project].size.should == Project.columns.size
98
+ #
99
+ #
100
+ # end
101
+ #
102
+ # it "should populate assigment members without the equivalent association names" do
103
+ #
104
+ # DataShift::ModelMethodsManager.find_methods( Project )
105
+ #
106
+ # # we should remove has-many & belongs_to from basic assignment set as they require a DB lookup
107
+ # # or a Model.create call, not a simple assignment
108
+ #
109
+ # DataShift::MethodDictionary.assignments_for(Project).should_not include( DataShift::MethodDictionary.belongs_to_for(Project) )
110
+ # DataShift::MethodDictionary.assignments_for(Project).should_not include( DataShift::MethodDictionary.has_many_for(Project) )
111
+ # end
112
+ #
113
+ #
114
+ #
115
+ #
116
+ #
117
+ # # Note : Not all assignments will currently have a column type, for example
118
+ # # those that are derived from a delegate_belongs_to
119
+ #
120
+ # it "should populate column types for assignment operators in method details" do
121
+ #
122
+ # DataShift::ModelMethodsManager.find_methods( Project )
123
+ #
124
+ # DataShift::MethodDictionary.build_method_details( Project )
125
+ #
126
+ # [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
127
+ #
128
+ # method_details = DataShift::MethodDictionary.find_method_detail( Project, format )
129
+ #
130
+ # method_details.class.should == DataShift::ModelMethod
131
+ #
132
+ # method_details.col_type.should_not be_nil
133
+ # method_details.col_type.name.should == 'value_as_string'
134
+ # method_details.col_type.default.should == nil
135
+ # method_details.col_type.sql_type.should include 'varchar(255)' # db specific, sqlite
136
+ # method_details.col_type.type.should == :string
137
+ # end
138
+ # end
139
+ #
140
+ # it "should populate required Class for assignment operators based on column type" do
141
+ #
142
+ # DataShift::ModelMethodsManager.find_methods( Project )
143
+ #
144
+ # DataShift::MethodDictionary.build_method_details( Project )
145
+ #
146
+ # [:value_as_string, 'value_as_string', "VALUE as_STRING", "value as string"].each do |format|
147
+ #
148
+ # method_details = DataShift::MethodDictionary.find_method_detail( Project, format )
149
+ #
150
+ # method_details.operator_class_name.should == 'String'
151
+ # method_details.operator_class.should be_is_a(Class)
152
+ # method_details.operator_class.should == String
153
+ # end
154
+ #
155
+ # end
156
+ #
157
+ #
158
+ # it "should populate method_details on Class for belongs_to" do
159
+ #
160
+ # DataShift::ModelMethodsManager.find_methods( Owner )
161
+ #
162
+ # DataShift::MethodDictionary.build_method_details( Owner )
163
+ #
164
+ # [:project, 'PROJECT'].each do |format|
165
+ #
166
+ # method_details = DataShift::MethodDictionary.find_method_detail( Owner, format )
167
+ #
168
+ # expect(method_details.operator_class_name).to eq 'Project'
169
+ # expect(method_details.operator_class).to be_a(Class)
170
+ # expect(method_details.operator_class).to eq Project
171
+ # end
172
+ #
173
+ # end
174
+ #
175
+ #
176
+ # it "should populate belongs_to operator for method details for different forms of a column name" do
177
+ #
178
+ # DataShift::ModelMethodsManager.find_methods( Project )
179
+ # DataShift::ModelMethodsManager.find_methods( Milestone )
180
+ #
181
+ # DataShift::MethodDictionary.build_method_details( Project )
182
+ # DataShift::MethodDictionary.build_method_details( Milestone )
183
+ #
184
+ # # milestone.project = project.id
185
+ # [:project, 'project', "PROJECT", "prOJECt"].each do |format|
186
+ #
187
+ # method_details = DataShift::MethodDictionary.find_method_detail( Milestone, format )
188
+ #
189
+ # method_details.should_not be_nil
190
+ #
191
+ # method_details.operator.should == 'project'
192
+ # method_details.operator_for(:belongs_to).should == 'project'
193
+ #
194
+ # method_details.operator_for(:assignment).should be_nil
195
+ # method_details.operator_for(:has_many).should be_nil
196
+ # end
197
+ #
198
+ # end
199
+ #
200
+ # it "should populate required Class for belongs_to operator method details" do
201
+ #
202
+ # DataShift::ModelMethodsManager.find_methods( LoaderRelease )
203
+ # DataShift::ModelMethodsManager.find_methods( LongAndComplexTableLinkedToVersion )
204
+ #
205
+ # DataShift::MethodDictionary.build_method_details( LoaderRelease )
206
+ # DataShift::MethodDictionary.build_method_details( LongAndComplexTableLinkedToVersion )
207
+ #
208
+ #
209
+ # # release.project = project.id
210
+ # [:project, 'project', "PROJECT", "prOJECt"].each do |format|
211
+ #
212
+ # method_details = DataShift::MethodDictionary.find_method_detail( LoaderRelease, format )
213
+ #
214
+ # method_details.operator_class_name.should == 'Project'
215
+ # method_details.operator_class.should == Project
216
+ # end
217
+ #
218
+ #
219
+ # [:version, "Version", "verSION"].each do |format|
220
+ # method_details = DataShift::MethodDictionary.find_method_detail( LongAndComplexTableLinkedToVersion, format )
221
+ #
222
+ # method_details.operator_type.should == :belongs_to
223
+ #
224
+ # method_details.operator_class_name.should == 'Version'
225
+ # method_details.operator_class.should == Version
226
+ # end
227
+ # end
228
+ #
229
+ # it "should populate required Class for has_one operator method details" do
230
+ #
231
+ # DataShift::ModelMethodsManager.find_methods( Version )
232
+ # DataShift::MethodDictionary.build_method_details( Version )
233
+ #
234
+ # # version.long_and_complex_table_linked_to_version = LongAndComplexTableLinkedToVersion.create()
235
+ #
236
+ # [:long_and_complex_table_linked_to_version, 'LongAndComplexTableLinkedToVersion', "Long And Complex_Table_Linked To Version", "Long_And_Complex_Table_Linked_To_Version"].each do |format|
237
+ # method_details = DataShift::MethodDictionary.find_method_detail( Version, format )
238
+ #
239
+ # method_details.should_not be_nil
240
+ #
241
+ # method_details.operator.should == 'long_and_complex_table_linked_to_version'
242
+ #
243
+ # method_details.operator_type.should == :has_one
244
+ #
245
+ # method_details.operator_class_name.should == 'LongAndComplexTableLinkedToVersion'
246
+ # method_details.operator_class.should == LongAndComplexTableLinkedToVersion
247
+ # end
248
+ # end
249
+ #
250
+ # it "should return false on for?(klass) if find_operators hasn't been called for klass" do
251
+ # DataShift::MethodDictionary.clear
252
+ # DataShift::MethodDictionary::for?(Project).should == false
253
+ # DataShift::MethodDictionary::find_operators(Project)
254
+ # DataShift::MethodDictionary::for?(Project).should == true
255
+ #
256
+ # end
257
+ #
258
+ # it "should return false on for?(klass) if find_operators hasn't been called for klass" do
259
+ # DataShift::MethodDictionary.clear
260
+ # DataShift::MethodDictionary::for?(Project).should == false
261
+ # end
262
+ #
263
+ # it "should find has_many operator for method details" do
264
+ #
265
+ # DataShift::ModelMethodsManager.find_methods( Project )
266
+ #
267
+ # DataShift::MethodDictionary.build_method_details( Project )
268
+ #
269
+ # [:milestones, "Mile Stones", 'mileSTONES', 'MileStones'].each do |format|
270
+ #
271
+ # method_details = DataShift::MethodDictionary.find_method_detail( Project, format )
272
+ #
273
+ # method_details.class.should == DataShift::ModelMethod
274
+ #
275
+ # result = 'milestones'
276
+ # method_details.operator.should == result
277
+ # method_details.operator_for(:has_many).should == result
278
+ #
279
+ # method_details.operator_for(:belongs_to).should be_nil
280
+ # method_details.operator_for(:assignments).should be_nil
281
+ # end
282
+ #
283
+ # end
284
+ #
285
+ #
286
+ # it "should return nil when non existent column name" do
287
+ #
288
+ # DataShift::ModelMethodsManager.find_methods( Project )
289
+ #
290
+ # ["On sale", 'on_sale'].each do |format|
291
+ # detail = DataShift::MethodDictionary.find_method_detail( Project, format )
292
+ #
293
+ # detail.should be_nil
294
+ # end
295
+ # end
296
+ #
297
+ #
298
+ # it "should not by default map setter methods" do
299
+ # DataShift::ModelMethodsManager.find_methods( Milestone )
300
+ #
301
+ # DataShift::MethodDictionary.assignments[Milestone].should_not include('title')
302
+ # end
303
+ #
304
+ # it "should support inclusion of setter methods" do
305
+ #
306
+ # Milestone.new.respond_to?('title=').should == true
307
+ # Milestone.new.respond_to?('milestone_setter=').should == true
308
+ #
309
+ # DataShift::MethodDictionary.setters( Milestone ).should include('title=') # delegate
310
+ # DataShift::MethodDictionary.setters( Milestone ).should include('milestone_setter=') # normal method
311
+ # end
312
+ #
313
+ # it "should support reload and inclusion of setter methods" do
314
+ #
315
+ # DataShift::ModelMethodsManager.find_methods( Project )
316
+ # DataShift::ModelMethodsManager.find_methods( Milestone )
317
+ #
318
+ # DataShift::MethodDictionary.assignments[Milestone].should_not include('title')
319
+ #
320
+ # DataShift::ModelMethodsManager.find_methods( Milestone, :reload => true, :instance_methods => true )
321
+ #
322
+ # # Milestone delegates :title to Project
323
+ # DataShift::MethodDictionary.assignments[Milestone].should include('title')
324
+ # DataShift::MethodDictionary.assignments[Milestone].should include('milestone_setter')
325
+ # end
326
+ end
327
+ end
328
+ end
329
+ end