datashift 0.16.0 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (422) hide show
  1. checksums.yaml +4 -4
  2. data/{LICENSE.txt → LICENSE} +0 -0
  3. data/Rakefile +1 -20
  4. data/datashift.thor +125 -0
  5. data/lib/applications/apache_poi_extensions.rb +21 -52
  6. data/lib/applications/excel.rb +64 -57
  7. data/lib/applications/hssf_row_extensions.rb +66 -0
  8. data/lib/applications/jexcel_file.rb +99 -95
  9. data/lib/applications/jexcel_file_extensions.rb +76 -83
  10. data/lib/applications/jruby/word.rb +36 -36
  11. data/lib/applications/ruby_poi_translations.rb +34 -32
  12. data/lib/applications/spreadsheet_extensions.rb +21 -19
  13. data/lib/datashift.rb +49 -59
  14. data/lib/datashift/binder.rb +217 -0
  15. data/lib/datashift/column_packer.rb +21 -72
  16. data/lib/datashift/configuration.rb +317 -0
  17. data/lib/datashift/context_factory.rb +88 -0
  18. data/lib/datashift/core_ext/array.rb +15 -0
  19. data/lib/datashift/core_ext/csv_ext.rb +46 -0
  20. data/lib/datashift/core_ext/string.rb +49 -0
  21. data/lib/datashift/core_ext/to_b.rb +11 -0
  22. data/lib/datashift/delimiters.rb +55 -61
  23. data/lib/datashift/doc_context.rb +137 -0
  24. data/lib/datashift/excel_base.rb +93 -81
  25. data/lib/datashift/exceptions.rb +30 -28
  26. data/lib/datashift/file_definitions.rb +44 -39
  27. data/lib/datashift/guards.rb +5 -5
  28. data/lib/datashift/header.rb +25 -0
  29. data/lib/datashift/headers.rb +94 -0
  30. data/lib/datashift/inbound_data/column.rb +44 -0
  31. data/lib/datashift/inbound_data/lookup_support.rb +33 -0
  32. data/lib/datashift/inbound_data/method_binding.rb +139 -0
  33. data/lib/datashift/load_object.rb +37 -12
  34. data/lib/datashift/logging.rb +54 -27
  35. data/lib/datashift/mandatory.rb +39 -0
  36. data/lib/datashift/mapping/data_flow_schema.rb +198 -0
  37. data/lib/datashift/{model_mapper.rb → mapping/mapper_utils.rb} +30 -10
  38. data/lib/datashift/model_methods/catalogue.rb +183 -0
  39. data/lib/datashift/model_methods/collection.rb +140 -0
  40. data/lib/datashift/model_methods/model_method.rb +162 -0
  41. data/lib/datashift/model_methods/model_methods_manager.rb +76 -0
  42. data/lib/datashift/model_methods/operator.rb +62 -0
  43. data/lib/datashift/node_collection.rb +26 -0
  44. data/lib/datashift/node_context.rb +68 -0
  45. data/lib/datashift/populator.rb +308 -282
  46. data/lib/datashift/progress_monitor.rb +91 -0
  47. data/lib/datashift/querying.rb +110 -52
  48. data/lib/datashift/templates/import_export_config.erb +55 -0
  49. data/lib/datashift/transformation/factory.rb +219 -0
  50. data/lib/datashift/transformation/remove.rb +44 -0
  51. data/lib/datashift/version.rb +3 -0
  52. data/lib/exporters/configuration.rb +84 -0
  53. data/lib/exporters/csv_exporter.rb +54 -52
  54. data/lib/exporters/excel_exporter.rb +80 -61
  55. data/lib/exporters/exporter_base.rb +8 -8
  56. data/lib/generators/config_generator.rb +218 -0
  57. data/lib/generators/csv_generator.rb +13 -70
  58. data/lib/generators/excel_generator.rb +23 -111
  59. data/lib/generators/generator_base.rb +15 -70
  60. data/lib/loaders/configuration.rb +90 -0
  61. data/lib/loaders/csv_loader.rb +63 -101
  62. data/lib/loaders/excel_loader.rb +71 -156
  63. data/lib/loaders/failure_data.rb +40 -0
  64. data/lib/loaders/file_loader.rb +16 -0
  65. data/lib/loaders/loader_base.rb +82 -410
  66. data/lib/loaders/loader_factory.rb +42 -0
  67. data/lib/loaders/paperclip/attachment_loader.rb +157 -140
  68. data/lib/loaders/paperclip/datashift_paperclip.rb +18 -35
  69. data/lib/loaders/paperclip/image_loading.rb +40 -35
  70. data/lib/loaders/reporters/basic_stdout_reporter.rb +40 -0
  71. data/lib/loaders/reporters/reporter.rb +26 -0
  72. data/lib/tasks/config.thor +65 -0
  73. data/{tasks → lib/tasks}/config/seed_fu_product_template.erb +0 -0
  74. data/{tasks → lib/tasks}/config/tidy_config.txt +0 -0
  75. data/lib/tasks/export.thor +192 -0
  76. data/lib/tasks/generate.thor +190 -0
  77. data/lib/tasks/import.thor +142 -0
  78. data/lib/{thor → tasks}/paperclip.thor +69 -69
  79. data/{tasks → lib/tasks/to_convert_to_thor}/db_tasks.rake +20 -20
  80. data/lib/tasks/tools.thor +109 -0
  81. data/spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg +0 -0
  82. data/spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg +0 -0
  83. data/spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg +0 -0
  84. data/spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg +0 -0
  85. data/spec/datashift/binder_spec.rb +266 -0
  86. data/spec/datashift/config_generator_spec.rb +186 -0
  87. data/spec/datashift/configuration.rb +66 -0
  88. data/spec/datashift/context_factory_spec.rb +63 -0
  89. data/spec/datashift/data_flow_schema_spec.rb +150 -0
  90. data/spec/datashift/datashift_spec.rb +52 -0
  91. data/spec/datashift/excel_base_spec.rb +57 -0
  92. data/spec/datashift/excel_spec.rb +188 -0
  93. data/spec/datashift/failure_data_spec.rb +27 -0
  94. data/spec/{file_definitions.rb → datashift/file_definitions.rb} +9 -10
  95. data/spec/datashift/headers_spec.rb +56 -0
  96. data/spec/datashift/inbound_data_spec.rb +47 -0
  97. data/spec/datashift/mapper_utils_spec.rb +38 -0
  98. data/spec/datashift/method_binding_spec.rb +60 -0
  99. data/spec/datashift/model_method_spec.rb +109 -0
  100. data/spec/datashift/model_methods_catalogue.rb +111 -0
  101. data/spec/datashift/model_methods_collection_spec.rb +138 -0
  102. data/spec/datashift/model_methods_manager_spec.rb +329 -0
  103. data/spec/datashift/populator_spec.rb +117 -0
  104. data/spec/datashift/thor_spec.rb +314 -0
  105. data/spec/datashift/transformation/factory_spec.rb +195 -0
  106. data/spec/datashift/transformation/transformer_remove_spec.rb +43 -0
  107. data/spec/dummy/Gemfile +53 -0
  108. data/spec/dummy/Gemfile.lock +197 -0
  109. data/spec/dummy/README.rdoc +28 -0
  110. data/spec/dummy/Rakefile +6 -0
  111. data/spec/dummy/app/assets/javascripts/application.js +16 -0
  112. data/spec/dummy/app/assets/javascripts/categories.js +2 -0
  113. data/spec/dummy/app/assets/javascripts/digitals.js +2 -0
  114. data/spec/dummy/app/assets/javascripts/empties.js +2 -0
  115. data/spec/dummy/app/assets/javascripts/loader_releases.js +2 -0
  116. data/spec/dummy/app/assets/javascripts/long_and_complex_table_linked_to_versions.js +2 -0
  117. data/spec/dummy/app/assets/javascripts/milestones.js +2 -0
  118. data/spec/dummy/app/assets/javascripts/owners.js +2 -0
  119. data/spec/dummy/app/assets/javascripts/projects.js +2 -0
  120. data/spec/dummy/app/assets/javascripts/users.js +2 -0
  121. data/spec/dummy/app/assets/javascripts/versions.js +2 -0
  122. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  123. data/spec/dummy/app/assets/stylesheets/categories.css +4 -0
  124. data/spec/dummy/app/assets/stylesheets/digitals.css +4 -0
  125. data/spec/dummy/app/assets/stylesheets/empties.css +4 -0
  126. data/spec/dummy/app/assets/stylesheets/loader_releases.css +4 -0
  127. data/spec/dummy/app/assets/stylesheets/long_and_complex_table_linked_to_versions.css +4 -0
  128. data/spec/dummy/app/assets/stylesheets/milestones.css +4 -0
  129. data/spec/dummy/app/assets/stylesheets/owners.css +4 -0
  130. data/spec/dummy/app/assets/stylesheets/projects.css +4 -0
  131. data/spec/dummy/app/assets/stylesheets/scaffold.css +56 -0
  132. data/spec/dummy/app/assets/stylesheets/users.css +4 -0
  133. data/spec/dummy/app/assets/stylesheets/versions.css +4 -0
  134. data/spec/dummy/app/controllers/application_controller.rb +5 -0
  135. data/spec/dummy/app/controllers/categories_controller.rb +58 -0
  136. data/spec/dummy/app/controllers/digitals_controller.rb +58 -0
  137. data/spec/dummy/app/controllers/empties_controller.rb +58 -0
  138. data/spec/dummy/app/controllers/loader_releases_controller.rb +58 -0
  139. data/spec/dummy/app/controllers/long_and_complex_table_linked_to_versions_controller.rb +58 -0
  140. data/spec/dummy/app/controllers/milestones_controller.rb +58 -0
  141. data/spec/dummy/app/controllers/owners_controller.rb +58 -0
  142. data/spec/dummy/app/controllers/projects_controller.rb +58 -0
  143. data/spec/dummy/app/controllers/users_controller.rb +58 -0
  144. data/spec/dummy/app/controllers/versions_controller.rb +58 -0
  145. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  146. data/spec/dummy/app/helpers/categories_helper.rb +2 -0
  147. data/spec/dummy/app/helpers/digitals_helper.rb +2 -0
  148. data/spec/dummy/app/helpers/empties_helper.rb +2 -0
  149. data/spec/dummy/app/helpers/loader_releases_helper.rb +2 -0
  150. data/spec/dummy/app/helpers/long_and_complex_table_linked_to_versions_helper.rb +2 -0
  151. data/spec/dummy/app/helpers/milestones_helper.rb +2 -0
  152. data/spec/dummy/app/helpers/owners_helper.rb +2 -0
  153. data/spec/dummy/app/helpers/projects_helper.rb +2 -0
  154. data/spec/dummy/app/helpers/users_helper.rb +2 -0
  155. data/spec/dummy/app/helpers/versions_helper.rb +2 -0
  156. data/spec/dummy/app/models/category.rb +6 -0
  157. data/spec/dummy/app/models/digital.rb +22 -0
  158. data/spec/dummy/app/models/empty.rb +2 -0
  159. data/spec/dummy/app/models/loader_release.rb +10 -0
  160. data/spec/dummy/app/models/long_and_complex_table_linked_to_version.rb +6 -0
  161. data/spec/dummy/app/models/milestone.rb +15 -0
  162. data/spec/dummy/app/models/owner.rb +13 -0
  163. data/spec/dummy/app/models/project.rb +53 -0
  164. data/spec/dummy/app/models/user.rb +5 -0
  165. data/spec/dummy/app/models/version.rb +7 -0
  166. data/spec/dummy/app/views/categories/_form.html.erb +17 -0
  167. data/spec/dummy/app/views/categories/edit.html.erb +6 -0
  168. data/spec/dummy/app/views/categories/index.html.erb +25 -0
  169. data/spec/dummy/app/views/categories/new.html.erb +5 -0
  170. data/spec/dummy/app/views/categories/show.html.erb +4 -0
  171. data/spec/dummy/app/views/digitals/_form.html.erb +17 -0
  172. data/spec/dummy/app/views/digitals/edit.html.erb +6 -0
  173. data/spec/dummy/app/views/digitals/index.html.erb +25 -0
  174. data/spec/dummy/app/views/digitals/new.html.erb +5 -0
  175. data/spec/dummy/app/views/digitals/show.html.erb +4 -0
  176. data/spec/dummy/app/views/empties/_form.html.erb +17 -0
  177. data/spec/dummy/app/views/empties/edit.html.erb +6 -0
  178. data/spec/dummy/app/views/empties/index.html.erb +25 -0
  179. data/spec/dummy/app/views/empties/new.html.erb +5 -0
  180. data/spec/dummy/app/views/empties/show.html.erb +4 -0
  181. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  182. data/spec/dummy/app/views/loader_releases/_form.html.erb +17 -0
  183. data/spec/dummy/app/views/loader_releases/edit.html.erb +6 -0
  184. data/spec/dummy/app/views/loader_releases/index.html.erb +25 -0
  185. data/spec/dummy/app/views/loader_releases/new.html.erb +5 -0
  186. data/spec/dummy/app/views/loader_releases/show.html.erb +4 -0
  187. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/_form.html.erb +17 -0
  188. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/edit.html.erb +6 -0
  189. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/index.html.erb +25 -0
  190. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/new.html.erb +5 -0
  191. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/show.html.erb +4 -0
  192. data/spec/dummy/app/views/milestones/_form.html.erb +17 -0
  193. data/spec/dummy/app/views/milestones/edit.html.erb +6 -0
  194. data/spec/dummy/app/views/milestones/index.html.erb +25 -0
  195. data/spec/dummy/app/views/milestones/new.html.erb +5 -0
  196. data/spec/dummy/app/views/milestones/show.html.erb +4 -0
  197. data/spec/dummy/app/views/owners/_form.html.erb +17 -0
  198. data/spec/dummy/app/views/owners/edit.html.erb +6 -0
  199. data/spec/dummy/app/views/owners/index.html.erb +25 -0
  200. data/spec/dummy/app/views/owners/new.html.erb +5 -0
  201. data/spec/dummy/app/views/owners/show.html.erb +4 -0
  202. data/spec/dummy/app/views/projects/_form.html.erb +17 -0
  203. data/spec/dummy/app/views/projects/edit.html.erb +6 -0
  204. data/spec/dummy/app/views/projects/index.html.erb +25 -0
  205. data/spec/dummy/app/views/projects/new.html.erb +5 -0
  206. data/spec/dummy/app/views/projects/show.html.erb +4 -0
  207. data/spec/dummy/app/views/users/_form.html.erb +17 -0
  208. data/spec/dummy/app/views/users/edit.html.erb +6 -0
  209. data/spec/dummy/app/views/users/index.html.erb +25 -0
  210. data/spec/dummy/app/views/users/new.html.erb +5 -0
  211. data/spec/dummy/app/views/users/show.html.erb +4 -0
  212. data/spec/dummy/app/views/versions/_form.html.erb +17 -0
  213. data/spec/dummy/app/views/versions/edit.html.erb +6 -0
  214. data/spec/dummy/app/views/versions/index.html.erb +25 -0
  215. data/spec/dummy/app/views/versions/new.html.erb +5 -0
  216. data/spec/dummy/app/views/versions/show.html.erb +4 -0
  217. data/spec/dummy/bin/bundle +3 -0
  218. data/spec/dummy/bin/rails +9 -0
  219. data/spec/dummy/bin/rake +9 -0
  220. data/spec/dummy/bin/setup +29 -0
  221. data/spec/dummy/bin/spring +16 -0
  222. data/spec/dummy/config.ru +4 -0
  223. data/spec/dummy/config/application.rb +26 -0
  224. data/spec/dummy/config/boot.rb +3 -0
  225. data/spec/dummy/config/database.yml +25 -0
  226. data/spec/dummy/config/environment.rb +5 -0
  227. data/spec/dummy/config/environments/development.rb +41 -0
  228. data/spec/dummy/config/environments/production.rb +79 -0
  229. data/spec/dummy/config/environments/test.rb +42 -0
  230. data/spec/dummy/config/initializers/assets.rb +11 -0
  231. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  232. data/spec/dummy/config/initializers/cookies_serializer.rb +3 -0
  233. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  234. data/spec/dummy/config/initializers/inflections.rb +16 -0
  235. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  236. data/spec/dummy/config/initializers/session_store.rb +3 -0
  237. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  238. data/spec/dummy/config/locales/en.yml +23 -0
  239. data/spec/dummy/config/routes.rb +76 -0
  240. data/spec/dummy/config/secrets.yml +22 -0
  241. data/spec/dummy/db/development.sqlite3 +0 -0
  242. data/spec/dummy/db/migrate/20110803201325_create_test_bed.rb +98 -0
  243. data/spec/dummy/db/migrate/20121009161700_add_digitals.rb +24 -0
  244. data/spec/dummy/db/migrate/20161005123106_create_digitals.rb +8 -0
  245. data/spec/dummy/db/migrate/20161005123106_create_long_and_complex_table_linked_to_versions.rb +8 -0
  246. data/spec/dummy/db/migrate/20161005123107_create_loader_releases.rb +8 -0
  247. data/spec/dummy/db/migrate/20161005123108_create_owners.rb +8 -0
  248. data/spec/dummy/db/migrate/20161005123109_create_empties.rb +8 -0
  249. data/spec/dummy/db/migrate/20161005123110_create_projects.rb +8 -0
  250. data/spec/dummy/db/migrate/20161005123111_create_users.rb +8 -0
  251. data/spec/dummy/db/migrate/20161005123111_create_versions.rb +8 -0
  252. data/spec/dummy/db/migrate/20161005123112_create_milestones.rb +8 -0
  253. data/spec/dummy/db/migrate/20161005123113_create_categories.rb +8 -0
  254. data/spec/dummy/db/schema.rb +93 -0
  255. data/spec/dummy/db/seeds.rb +9 -0
  256. data/spec/dummy/db/test.sqlite3 +0 -0
  257. data/spec/dummy/log/test.log +69 -0
  258. data/spec/dummy/public/404.html +67 -0
  259. data/spec/dummy/public/422.html +67 -0
  260. data/spec/dummy/public/500.html +66 -0
  261. data/spec/dummy/public/favicon.ico +0 -0
  262. data/spec/dummy/public/robots.txt +5 -0
  263. data/spec/dummy/sandbox_example.thor +4 -0
  264. data/spec/dummy/test/controllers/categories_controller_test.rb +49 -0
  265. data/spec/dummy/test/controllers/digitals_controller_test.rb +49 -0
  266. data/spec/dummy/test/controllers/empties_controller_test.rb +49 -0
  267. data/spec/dummy/test/controllers/loader_releases_controller_test.rb +49 -0
  268. data/spec/dummy/test/controllers/long_and_complex_table_linked_to_versions_controller_test.rb +49 -0
  269. data/spec/dummy/test/controllers/milestones_controller_test.rb +49 -0
  270. data/spec/dummy/test/controllers/owners_controller_test.rb +49 -0
  271. data/spec/dummy/test/controllers/projects_controller_test.rb +49 -0
  272. data/spec/dummy/test/controllers/users_controller_test.rb +49 -0
  273. data/spec/dummy/test/controllers/versions_controller_test.rb +49 -0
  274. data/spec/dummy/test/factories/categories.rb +5 -0
  275. data/spec/dummy/test/factories/digitals.rb +5 -0
  276. data/spec/dummy/test/factories/empties.rb +5 -0
  277. data/spec/dummy/test/factories/loader_releases.rb +5 -0
  278. data/spec/dummy/test/factories/long_and_complex_table_linked_to_versions.rb +5 -0
  279. data/spec/dummy/test/factories/milestones.rb +5 -0
  280. data/spec/dummy/test/factories/owners.rb +5 -0
  281. data/spec/dummy/test/factories/projects.rb +5 -0
  282. data/spec/dummy/test/factories/users.rb +5 -0
  283. data/spec/dummy/test/factories/versions.rb +5 -0
  284. data/spec/dummy/test/models/category_test.rb +7 -0
  285. data/spec/dummy/test/models/digital_test.rb +7 -0
  286. data/spec/dummy/test/models/empty_test.rb +7 -0
  287. data/spec/dummy/test/models/loader_release_test.rb +7 -0
  288. data/spec/dummy/test/models/long_and_complex_table_linked_to_version_test.rb +7 -0
  289. data/spec/dummy/test/models/milestone_test.rb +7 -0
  290. data/spec/dummy/test/models/owner_test.rb +7 -0
  291. data/spec/dummy/test/models/project_test.rb +7 -0
  292. data/spec/dummy/test/models/user_test.rb +7 -0
  293. data/spec/dummy/test/models/version_test.rb +7 -0
  294. data/spec/dummy/test/test_helper.rb +10 -0
  295. data/spec/exporters/csv_exporter_spec.rb +240 -0
  296. data/spec/exporters/csv_generator_spec.rb +139 -0
  297. data/spec/exporters/excel_exporter_spec.rb +193 -0
  298. data/spec/exporters/excel_generator_spec.rb +181 -0
  299. data/spec/exporters/generator_base_spec.rb +45 -0
  300. data/spec/factories/categories.rb +7 -0
  301. data/spec/factories/factories.rb +18 -0
  302. data/spec/factories/milestone.rb +16 -0
  303. data/spec/factories/projects.rb +41 -0
  304. data/spec/fixtures/BadAssociationName.xls +0 -0
  305. data/spec/fixtures/DemoNegativeTesting.xls +0 -0
  306. data/spec/fixtures/ProjectConfiguration.yml +18 -0
  307. data/spec/fixtures/ProjectsMultiCategories.xls +0 -0
  308. data/spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls +0 -0
  309. data/spec/fixtures/ProjectsSingleCategories.xls +0 -0
  310. data/spec/fixtures/ProjectsSingleCategories.xlsx +0 -0
  311. data/spec/fixtures/SimpleProjects.xls +0 -0
  312. data/spec/fixtures/config/database.yml +28 -0
  313. data/spec/fixtures/csv/BadAssociationName.csv +6 -0
  314. data/spec/fixtures/csv/DemoNegativeTesting.csv +6 -0
  315. data/spec/fixtures/csv/ProjectsMultiCategories.csv +5 -0
  316. data/spec/fixtures/csv/ProjectsMultiCategoriesHeaderLookup.csv +5 -0
  317. data/spec/fixtures/csv/ProjectsSingleCategories.csv +5 -0
  318. data/spec/fixtures/csv/SimpleProjects.csv +4 -0
  319. data/spec/fixtures/db/migrate/20110803201325_create_test_bed.rb +98 -0
  320. data/spec/fixtures/db/migrate/20121009161700_add_digitals.rb +24 -0
  321. data/spec/fixtures/db/seeds.rb +9 -0
  322. data/spec/fixtures/images/DEMO_001_ror_bag.jpeg +0 -0
  323. data/spec/fixtures/images/DEMO_002_Powerstation.jpeg +0 -0
  324. data/spec/fixtures/images/DEMO_003_ror_mug.jpeg +0 -0
  325. data/spec/fixtures/images/DEMO_004_ror_ringer.jpeg +0 -0
  326. data/spec/fixtures/load_datashift.thor +3 -0
  327. data/spec/fixtures/models/category.rb +6 -0
  328. data/spec/fixtures/models/digital.rb +22 -0
  329. data/spec/fixtures/models/empty.rb +2 -0
  330. data/spec/fixtures/models/loader_release.rb +10 -0
  331. data/spec/fixtures/models/long_and_complex_table_linked_to_version.rb +6 -0
  332. data/spec/fixtures/models/milestone.rb +15 -0
  333. data/spec/fixtures/models/owner.rb +13 -0
  334. data/spec/fixtures/models/project.rb +53 -0
  335. data/spec/fixtures/models/user.rb +5 -0
  336. data/spec/fixtures/models/version.rb +7 -0
  337. data/spec/fixtures/results/exp_project_assoc_headers.xls +0 -0
  338. data/spec/fixtures/results/exp_project_collection_spec.csv +2 -0
  339. data/spec/fixtures/results/exp_project_export.xls +0 -0
  340. data/spec/fixtures/results/exp_project_first_export.xls +0 -0
  341. data/spec/fixtures/results/exp_project_plus_assoc.xls +0 -0
  342. data/spec/fixtures/results/exp_project_plus_assoc_export_spec.csv +9 -0
  343. data/spec/fixtures/results/gen_project_plus_assoc_template.xls +0 -0
  344. data/spec/fixtures/results/gen_project_plus_some_assoc_template.xls +0 -0
  345. data/spec/fixtures/results/gen_project_template.xls +0 -0
  346. data/spec/fixtures/results/project_and_assoc_in_hash_export.xls +0 -0
  347. data/spec/fixtures/results/project_and_assoc_in_json_export.csv +9 -0
  348. data/spec/fixtures/results/project_and_assoc_in_json_export.xls +0 -0
  349. data/spec/fixtures/results/project_export_spec_with_custom_delim_,.csv +2 -0
  350. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/243.csv +2 -0
  351. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/247.csv +2 -0
  352. data/spec/fixtures/results/project_plus_assoc_template.csv +1 -0
  353. data/spec/fixtures/results/project_plus_some_assoc_template.csv +1 -0
  354. data/spec/fixtures/results/project_remove_export_spec.csv +2 -0
  355. data/spec/fixtures/results/project_template.csv +1 -0
  356. data/spec/fixtures/results/project_with_methods_export_spec.csv +2 -0
  357. data/spec/fixtures/results/thor_spec_gen_project.csv +1 -0
  358. data/spec/fixtures/sandbox_example.thor +4 -0
  359. data/spec/fixtures/simple_export_spec.xls +0 -0
  360. data/spec/fixtures/simple_template_spec.xls +0 -0
  361. data/spec/fixtures/test_model_defs.rb +7 -0
  362. data/spec/loaders/csv_loader_spec.rb +206 -0
  363. data/spec/loaders/data_flow_excel_loader_spec.rb +290 -0
  364. data/spec/loaders/excel_loader_failures_spec.rb +67 -0
  365. data/spec/loaders/excel_loader_spec.rb +294 -0
  366. data/spec/loaders/loader_base_spec.rb +29 -0
  367. data/spec/loaders/paperclip_loader_spec.rb +106 -0
  368. data/spec/log/datashift.log +14930 -0
  369. data/spec/private/digitals/1/DEMO_003_ror_mug.jpeg +0 -0
  370. data/spec/private/digitals/2/DEMO_002_Powerstation.jpeg +0 -0
  371. data/spec/private/digitals/3/DEMO_004_ror_ringer.jpeg +0 -0
  372. data/spec/private/digitals/4/DEMO_001_ror_bag.jpeg +0 -0
  373. data/spec/spec_helper.rb +26 -230
  374. data/spec/support/clear_and_manage_contexts.rb +25 -0
  375. data/spec/support/database_cleaner.rb +32 -0
  376. data/spec/support/datashift_test_helpers.rb +153 -0
  377. data/spec/support/files_paths_helper.rb +13 -0
  378. data/spec/support/fixtures/results/mapping_template.yaml +15 -0
  379. data/spec/support/sandbox.rb +136 -0
  380. metadata +804 -85
  381. data/README.markdown +0 -274
  382. data/README.rdoc +0 -19
  383. data/VERSION +0 -1
  384. data/datashift.gemspec +0 -48
  385. data/lib/applications/jruby/old_pre_proxy_jexcel_file.rb +0 -437
  386. data/lib/datashift/data_transforms.rb +0 -83
  387. data/lib/datashift/mapping_file_definitions.rb +0 -88
  388. data/lib/datashift/mapping_service.rb +0 -91
  389. data/lib/datashift/method_detail.rb +0 -165
  390. data/lib/datashift/method_details_manager.rb +0 -95
  391. data/lib/datashift/method_dictionary.rb +0 -281
  392. data/lib/datashift/method_mapper.rb +0 -174
  393. data/lib/datashift/thor_base.rb +0 -38
  394. data/lib/generators/mapping_generator.rb +0 -112
  395. data/lib/helpers/core_ext/csv_file.rb +0 -33
  396. data/lib/helpers/core_ext/to_b.rb +0 -24
  397. data/lib/loaders/reporter.rb +0 -58
  398. data/lib/thor/export.thor +0 -175
  399. data/lib/thor/generate.thor +0 -191
  400. data/lib/thor/import.thor +0 -110
  401. data/lib/thor/mapping.thor +0 -65
  402. data/lib/thor/tools.thor +0 -84
  403. data/spec/Gemfile +0 -31
  404. data/spec/Gemfile.lock +0 -134
  405. data/spec/csv_exporter_spec.rb +0 -144
  406. data/spec/csv_generator_spec.rb +0 -159
  407. data/spec/csv_loader_spec.rb +0 -212
  408. data/spec/datashift_spec.rb +0 -55
  409. data/spec/excel_exporter_spec.rb +0 -199
  410. data/spec/excel_generator_spec.rb +0 -203
  411. data/spec/excel_loader_spec.rb +0 -237
  412. data/spec/excel_spec.rb +0 -203
  413. data/spec/loader_base_spec.rb +0 -166
  414. data/spec/mapping_spec.rb +0 -117
  415. data/spec/method_dictionary_spec.rb +0 -300
  416. data/spec/method_mapper_spec.rb +0 -100
  417. data/spec/model_mapper_spec.rb +0 -41
  418. data/spec/paperclip_loader_spec.rb +0 -92
  419. data/spec/populator_spec.rb +0 -128
  420. data/spec/thor_spec.rb +0 -90
  421. data/tasks/file_tasks.rake +0 -37
  422. data/tasks/word_to_seedfu.rake +0 -167
@@ -0,0 +1,43 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2015
2
+ # Author :: Tom Statter
3
+ # License:: MIT
4
+ #
5
+ # Details:: Specs around Transforming inbound data
6
+ #
7
+ require File.join(File.dirname(__FILE__), '/../../spec_helper')
8
+
9
+ module DataShift
10
+
11
+ describe 'Transformation Remove' do
12
+ include_context 'ClearThenManageProject'
13
+
14
+ context 'Column removals' do
15
+ it 'should process options to remove unwanted columns' do
16
+ headers = [:a, :b, :c, :d, :e, :f]
17
+
18
+ DataShift::Configuration.configure do |config|
19
+ config.remove_columns = [:b, :f]
20
+ end
21
+
22
+ DataShift::Transformation::Remove.new.unwanted_columns(headers )
23
+
24
+ expect(headers).to_not include [:b, :f]
25
+ end
26
+
27
+ it 'should process options to remove unwant5ed columns' do
28
+ headers = [:a, :id, :c, :d, :e, :created_on, :f, :updated_on]
29
+
30
+ DataShift::Configuration.configure do |config|
31
+ config.remove_rails = true
32
+ end
33
+
34
+ DataShift::Transformation::Remove.new.unwanted_columns(headers)
35
+
36
+ expect(headers).to_not include [:id, :created_on, :updated_on]
37
+ end
38
+
39
+ end
40
+
41
+ end
42
+
43
+ end
@@ -0,0 +1,53 @@
1
+ source 'https://rubygems.org'
2
+
3
+
4
+ # Bundle edge Rails instead: gem 'rails', github: 'rails/rails'
5
+ gem 'rails', '4.2.7.1'
6
+ # Use sqlite3 as the database for Active Record
7
+ gem 'sqlite3'
8
+ # Use SCSS for stylesheets
9
+ gem 'sass-rails', '~> 5.0'
10
+ # Use Uglifier as compressor for JavaScript assets
11
+ gem 'uglifier', '>= 1.3.0'
12
+ # Use CoffeeScript for .coffee assets and views
13
+ gem 'coffee-rails', '~> 4.1.0'
14
+ # See https://github.com/rails/execjs#readme for more supported runtimes
15
+ # gem 'therubyracer', platforms: :ruby
16
+
17
+ # Use jquery as the JavaScript library
18
+ gem 'jquery-rails'
19
+ # Turbolinks makes following links in your web application faster. Read more: https://github.com/rails/turbolinks
20
+ gem 'turbolinks'
21
+ # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
22
+ gem 'jbuilder', '~> 2.0'
23
+ # bundle exec rake doc:rails generates the API under doc/api.
24
+ gem 'sdoc', '~> 0.4.0', group: :doc
25
+
26
+ # Use ActiveModel has_secure_password
27
+ # gem 'bcrypt', '~> 3.1.7'
28
+
29
+ # Use Unicorn as the app server
30
+ # gem 'unicorn'
31
+
32
+ # Use Capistrano for deployment
33
+ # gem 'capistrano-rails', group: :development
34
+
35
+ group :development, :test do
36
+ # Call 'byebug' anywhere in the code to stop execution and get a debugger console
37
+ gem 'byebug'
38
+ end
39
+
40
+ group :development do
41
+ # Access an IRB console on exception pages or by using <%= console %> in views
42
+ gem 'web-console', '~> 2.0'
43
+
44
+ # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
45
+ gem 'spring'
46
+ end
47
+
48
+
49
+ gem 'datashift', path: '/home/rubyuser/SoftwareDev/git/datashift'
50
+
51
+ gem 'awesome_print'
52
+
53
+ gem 'factory_girl_rails'
@@ -0,0 +1,197 @@
1
+ PATH
2
+ remote: /home/rubyuser/SoftwareDev/git/datashift
3
+ specs:
4
+ datashift (0.2.0)
5
+ paperclip (~> 4)
6
+ rails (~> 4.2)
7
+ rubyzip (~> 1.2)
8
+ spreadsheet (~> 1.1)
9
+ thor (~> 0.19.1)
10
+ thread_safe (~> 0.3, >= 0.3)
11
+
12
+ GEM
13
+ remote: https://rubygems.org/
14
+ specs:
15
+ actionmailer (4.2.7.1)
16
+ actionpack (= 4.2.7.1)
17
+ actionview (= 4.2.7.1)
18
+ activejob (= 4.2.7.1)
19
+ mail (~> 2.5, >= 2.5.4)
20
+ rails-dom-testing (~> 1.0, >= 1.0.5)
21
+ actionpack (4.2.7.1)
22
+ actionview (= 4.2.7.1)
23
+ activesupport (= 4.2.7.1)
24
+ rack (~> 1.6)
25
+ rack-test (~> 0.6.2)
26
+ rails-dom-testing (~> 1.0, >= 1.0.5)
27
+ rails-html-sanitizer (~> 1.0, >= 1.0.2)
28
+ actionview (4.2.7.1)
29
+ activesupport (= 4.2.7.1)
30
+ builder (~> 3.1)
31
+ erubis (~> 2.7.0)
32
+ rails-dom-testing (~> 1.0, >= 1.0.5)
33
+ rails-html-sanitizer (~> 1.0, >= 1.0.2)
34
+ activejob (4.2.7.1)
35
+ activesupport (= 4.2.7.1)
36
+ globalid (>= 0.3.0)
37
+ activemodel (4.2.7.1)
38
+ activesupport (= 4.2.7.1)
39
+ builder (~> 3.1)
40
+ activerecord (4.2.7.1)
41
+ activemodel (= 4.2.7.1)
42
+ activesupport (= 4.2.7.1)
43
+ arel (~> 6.0)
44
+ activesupport (4.2.7.1)
45
+ i18n (~> 0.7)
46
+ json (~> 1.7, >= 1.7.7)
47
+ minitest (~> 5.1)
48
+ thread_safe (~> 0.3, >= 0.3.4)
49
+ tzinfo (~> 1.1)
50
+ arel (6.0.3)
51
+ awesome_print (1.7.0)
52
+ binding_of_caller (0.7.2)
53
+ debug_inspector (>= 0.0.1)
54
+ builder (3.2.2)
55
+ byebug (9.0.6)
56
+ climate_control (0.0.3)
57
+ activesupport (>= 3.0)
58
+ cocaine (0.5.8)
59
+ climate_control (>= 0.0.3, < 1.0)
60
+ coffee-rails (4.1.1)
61
+ coffee-script (>= 2.2.0)
62
+ railties (>= 4.0.0, < 5.1.x)
63
+ coffee-script (2.4.1)
64
+ coffee-script-source
65
+ execjs
66
+ coffee-script-source (1.10.0)
67
+ concurrent-ruby (1.0.2)
68
+ debug_inspector (0.0.2)
69
+ erubis (2.7.0)
70
+ execjs (2.7.0)
71
+ factory_girl (4.7.0)
72
+ activesupport (>= 3.0.0)
73
+ factory_girl_rails (4.7.0)
74
+ factory_girl (~> 4.7.0)
75
+ railties (>= 3.0.0)
76
+ globalid (0.3.7)
77
+ activesupport (>= 4.1.0)
78
+ i18n (0.7.0)
79
+ jbuilder (2.6.0)
80
+ activesupport (>= 3.0.0, < 5.1)
81
+ multi_json (~> 1.2)
82
+ jquery-rails (4.2.1)
83
+ rails-dom-testing (>= 1, < 3)
84
+ railties (>= 4.2.0)
85
+ thor (>= 0.14, < 2.0)
86
+ json (1.8.3)
87
+ loofah (2.0.3)
88
+ nokogiri (>= 1.5.9)
89
+ mail (2.6.4)
90
+ mime-types (>= 1.16, < 4)
91
+ mime-types (3.1)
92
+ mime-types-data (~> 3.2015)
93
+ mime-types-data (3.2016.0521)
94
+ mimemagic (0.3.0)
95
+ mini_portile2 (2.1.0)
96
+ minitest (5.9.1)
97
+ multi_json (1.12.1)
98
+ nokogiri (1.6.8.1)
99
+ mini_portile2 (~> 2.1.0)
100
+ paperclip (4.3.7)
101
+ activemodel (>= 3.2.0)
102
+ activesupport (>= 3.2.0)
103
+ cocaine (~> 0.5.5)
104
+ mime-types
105
+ mimemagic (= 0.3.0)
106
+ rack (1.6.4)
107
+ rack-test (0.6.3)
108
+ rack (>= 1.0)
109
+ rails (4.2.7.1)
110
+ actionmailer (= 4.2.7.1)
111
+ actionpack (= 4.2.7.1)
112
+ actionview (= 4.2.7.1)
113
+ activejob (= 4.2.7.1)
114
+ activemodel (= 4.2.7.1)
115
+ activerecord (= 4.2.7.1)
116
+ activesupport (= 4.2.7.1)
117
+ bundler (>= 1.3.0, < 2.0)
118
+ railties (= 4.2.7.1)
119
+ sprockets-rails
120
+ rails-deprecated_sanitizer (1.0.3)
121
+ activesupport (>= 4.2.0.alpha)
122
+ rails-dom-testing (1.0.7)
123
+ activesupport (>= 4.2.0.beta, < 5.0)
124
+ nokogiri (~> 1.6.0)
125
+ rails-deprecated_sanitizer (>= 1.0.1)
126
+ rails-html-sanitizer (1.0.3)
127
+ loofah (~> 2.0)
128
+ railties (4.2.7.1)
129
+ actionpack (= 4.2.7.1)
130
+ activesupport (= 4.2.7.1)
131
+ rake (>= 0.8.7)
132
+ thor (>= 0.18.1, < 2.0)
133
+ rake (11.3.0)
134
+ rdoc (4.2.2)
135
+ json (~> 1.4)
136
+ ruby-ole (1.2.12)
137
+ rubyzip (1.2.0)
138
+ sass (3.4.22)
139
+ sass-rails (5.0.6)
140
+ railties (>= 4.0.0, < 6)
141
+ sass (~> 3.1)
142
+ sprockets (>= 2.8, < 4.0)
143
+ sprockets-rails (>= 2.0, < 4.0)
144
+ tilt (>= 1.1, < 3)
145
+ sdoc (0.4.1)
146
+ json (~> 1.7, >= 1.7.7)
147
+ rdoc (~> 4.0)
148
+ spreadsheet (1.1.3)
149
+ ruby-ole (>= 1.0)
150
+ spring (2.0.0)
151
+ activesupport (>= 4.2)
152
+ sprockets (3.7.0)
153
+ concurrent-ruby (~> 1.0)
154
+ rack (> 1, < 3)
155
+ sprockets-rails (3.2.0)
156
+ actionpack (>= 4.0)
157
+ activesupport (>= 4.0)
158
+ sprockets (>= 3.0.0)
159
+ sqlite3 (1.3.11)
160
+ thor (0.19.1)
161
+ thread_safe (0.3.5)
162
+ tilt (2.0.5)
163
+ turbolinks (5.0.1)
164
+ turbolinks-source (~> 5)
165
+ turbolinks-source (5.0.0)
166
+ tzinfo (1.2.2)
167
+ thread_safe (~> 0.1)
168
+ uglifier (3.0.2)
169
+ execjs (>= 0.3.0, < 3)
170
+ web-console (2.3.0)
171
+ activemodel (>= 4.0)
172
+ binding_of_caller (>= 0.7.2)
173
+ railties (>= 4.0)
174
+ sprockets-rails (>= 2.0, < 4.0)
175
+
176
+ PLATFORMS
177
+ ruby
178
+
179
+ DEPENDENCIES
180
+ awesome_print
181
+ byebug
182
+ coffee-rails (~> 4.1.0)
183
+ datashift!
184
+ factory_girl_rails
185
+ jbuilder (~> 2.0)
186
+ jquery-rails
187
+ rails (= 4.2.7.1)
188
+ sass-rails (~> 5.0)
189
+ sdoc (~> 0.4.0)
190
+ spring
191
+ sqlite3
192
+ turbolinks
193
+ uglifier (>= 1.3.0)
194
+ web-console (~> 2.0)
195
+
196
+ BUNDLED WITH
197
+ 1.12.5
@@ -0,0 +1,28 @@
1
+ == README
2
+
3
+ This README would normally document whatever steps are necessary to get the
4
+ application up and running.
5
+
6
+ Things you may want to cover:
7
+
8
+ * Ruby version
9
+
10
+ * System dependencies
11
+
12
+ * Configuration
13
+
14
+ * Database creation
15
+
16
+ * Database initialization
17
+
18
+ * How to run the test suite
19
+
20
+ * Services (job queues, cache servers, search engines, etc.)
21
+
22
+ * Deployment instructions
23
+
24
+ * ...
25
+
26
+
27
+ Please feel free to use a different markup language if you do not plan to run
28
+ <tt>rake doc:app</tt>.
@@ -0,0 +1,6 @@
1
+ # Add your own tasks in files placed in lib/tasks ending in .rake,
2
+ # for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
3
+
4
+ require File.expand_path('../config/application', __FILE__)
5
+
6
+ Rails.application.load_tasks
@@ -0,0 +1,16 @@
1
+ // This is a manifest file that'll be compiled into application.js, which will include all the files
2
+ // listed below.
3
+ //
4
+ // Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
5
+ // or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
6
+ //
7
+ // It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
8
+ // compiled file.
9
+ //
10
+ // Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
11
+ // about supported directives.
12
+ //
13
+ //= require jquery
14
+ //= require jquery_ujs
15
+ //= require turbolinks
16
+ //= require_tree .
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,2 @@
1
+ // Place all the behaviors and hooks related to the matching controller here.
2
+ // All this logic will automatically be available in application.js.
@@ -0,0 +1,15 @@
1
+ /*
2
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
3
+ * listed below.
4
+ *
5
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
6
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
7
+ *
8
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
9
+ * compiled file so the styles you add here take precedence over styles defined in any styles
10
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
11
+ * file per style scope.
12
+ *
13
+ *= require_tree .
14
+ *= require_self
15
+ */
@@ -0,0 +1,4 @@
1
+ /*
2
+ Place all the styles related to the matching controller here.
3
+ They will automatically be included in application.css.
4
+ */
@@ -0,0 +1,4 @@
1
+ /*
2
+ Place all the styles related to the matching controller here.
3
+ They will automatically be included in application.css.
4
+ */
@@ -0,0 +1,4 @@
1
+ /*
2
+ Place all the styles related to the matching controller here.
3
+ They will automatically be included in application.css.
4
+ */
@@ -0,0 +1,4 @@
1
+ /*
2
+ Place all the styles related to the matching controller here.
3
+ They will automatically be included in application.css.
4
+ */
@@ -0,0 +1,4 @@
1
+ /*
2
+ Place all the styles related to the matching controller here.
3
+ They will automatically be included in application.css.
4
+ */
@@ -0,0 +1,4 @@
1
+ /*
2
+ Place all the styles related to the matching controller here.
3
+ They will automatically be included in application.css.
4
+ */
@@ -0,0 +1,4 @@
1
+ /*
2
+ Place all the styles related to the matching controller here.
3
+ They will automatically be included in application.css.
4
+ */