datashift 0.16.0 → 0.40.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (422) hide show
  1. checksums.yaml +4 -4
  2. data/{LICENSE.txt → LICENSE} +0 -0
  3. data/Rakefile +1 -20
  4. data/datashift.thor +125 -0
  5. data/lib/applications/apache_poi_extensions.rb +21 -52
  6. data/lib/applications/excel.rb +64 -57
  7. data/lib/applications/hssf_row_extensions.rb +66 -0
  8. data/lib/applications/jexcel_file.rb +99 -95
  9. data/lib/applications/jexcel_file_extensions.rb +76 -83
  10. data/lib/applications/jruby/word.rb +36 -36
  11. data/lib/applications/ruby_poi_translations.rb +34 -32
  12. data/lib/applications/spreadsheet_extensions.rb +21 -19
  13. data/lib/datashift.rb +49 -59
  14. data/lib/datashift/binder.rb +217 -0
  15. data/lib/datashift/column_packer.rb +21 -72
  16. data/lib/datashift/configuration.rb +317 -0
  17. data/lib/datashift/context_factory.rb +88 -0
  18. data/lib/datashift/core_ext/array.rb +15 -0
  19. data/lib/datashift/core_ext/csv_ext.rb +46 -0
  20. data/lib/datashift/core_ext/string.rb +49 -0
  21. data/lib/datashift/core_ext/to_b.rb +11 -0
  22. data/lib/datashift/delimiters.rb +55 -61
  23. data/lib/datashift/doc_context.rb +137 -0
  24. data/lib/datashift/excel_base.rb +93 -81
  25. data/lib/datashift/exceptions.rb +30 -28
  26. data/lib/datashift/file_definitions.rb +44 -39
  27. data/lib/datashift/guards.rb +5 -5
  28. data/lib/datashift/header.rb +25 -0
  29. data/lib/datashift/headers.rb +94 -0
  30. data/lib/datashift/inbound_data/column.rb +44 -0
  31. data/lib/datashift/inbound_data/lookup_support.rb +33 -0
  32. data/lib/datashift/inbound_data/method_binding.rb +139 -0
  33. data/lib/datashift/load_object.rb +37 -12
  34. data/lib/datashift/logging.rb +54 -27
  35. data/lib/datashift/mandatory.rb +39 -0
  36. data/lib/datashift/mapping/data_flow_schema.rb +198 -0
  37. data/lib/datashift/{model_mapper.rb → mapping/mapper_utils.rb} +30 -10
  38. data/lib/datashift/model_methods/catalogue.rb +183 -0
  39. data/lib/datashift/model_methods/collection.rb +140 -0
  40. data/lib/datashift/model_methods/model_method.rb +162 -0
  41. data/lib/datashift/model_methods/model_methods_manager.rb +76 -0
  42. data/lib/datashift/model_methods/operator.rb +62 -0
  43. data/lib/datashift/node_collection.rb +26 -0
  44. data/lib/datashift/node_context.rb +68 -0
  45. data/lib/datashift/populator.rb +308 -282
  46. data/lib/datashift/progress_monitor.rb +91 -0
  47. data/lib/datashift/querying.rb +110 -52
  48. data/lib/datashift/templates/import_export_config.erb +55 -0
  49. data/lib/datashift/transformation/factory.rb +219 -0
  50. data/lib/datashift/transformation/remove.rb +44 -0
  51. data/lib/datashift/version.rb +3 -0
  52. data/lib/exporters/configuration.rb +84 -0
  53. data/lib/exporters/csv_exporter.rb +54 -52
  54. data/lib/exporters/excel_exporter.rb +80 -61
  55. data/lib/exporters/exporter_base.rb +8 -8
  56. data/lib/generators/config_generator.rb +218 -0
  57. data/lib/generators/csv_generator.rb +13 -70
  58. data/lib/generators/excel_generator.rb +23 -111
  59. data/lib/generators/generator_base.rb +15 -70
  60. data/lib/loaders/configuration.rb +90 -0
  61. data/lib/loaders/csv_loader.rb +63 -101
  62. data/lib/loaders/excel_loader.rb +71 -156
  63. data/lib/loaders/failure_data.rb +40 -0
  64. data/lib/loaders/file_loader.rb +16 -0
  65. data/lib/loaders/loader_base.rb +82 -410
  66. data/lib/loaders/loader_factory.rb +42 -0
  67. data/lib/loaders/paperclip/attachment_loader.rb +157 -140
  68. data/lib/loaders/paperclip/datashift_paperclip.rb +18 -35
  69. data/lib/loaders/paperclip/image_loading.rb +40 -35
  70. data/lib/loaders/reporters/basic_stdout_reporter.rb +40 -0
  71. data/lib/loaders/reporters/reporter.rb +26 -0
  72. data/lib/tasks/config.thor +65 -0
  73. data/{tasks → lib/tasks}/config/seed_fu_product_template.erb +0 -0
  74. data/{tasks → lib/tasks}/config/tidy_config.txt +0 -0
  75. data/lib/tasks/export.thor +192 -0
  76. data/lib/tasks/generate.thor +190 -0
  77. data/lib/tasks/import.thor +142 -0
  78. data/lib/{thor → tasks}/paperclip.thor +69 -69
  79. data/{tasks → lib/tasks/to_convert_to_thor}/db_tasks.rake +20 -20
  80. data/lib/tasks/tools.thor +109 -0
  81. data/spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg +0 -0
  82. data/spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg +0 -0
  83. data/spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg +0 -0
  84. data/spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg +0 -0
  85. data/spec/datashift/binder_spec.rb +266 -0
  86. data/spec/datashift/config_generator_spec.rb +186 -0
  87. data/spec/datashift/configuration.rb +66 -0
  88. data/spec/datashift/context_factory_spec.rb +63 -0
  89. data/spec/datashift/data_flow_schema_spec.rb +150 -0
  90. data/spec/datashift/datashift_spec.rb +52 -0
  91. data/spec/datashift/excel_base_spec.rb +57 -0
  92. data/spec/datashift/excel_spec.rb +188 -0
  93. data/spec/datashift/failure_data_spec.rb +27 -0
  94. data/spec/{file_definitions.rb → datashift/file_definitions.rb} +9 -10
  95. data/spec/datashift/headers_spec.rb +56 -0
  96. data/spec/datashift/inbound_data_spec.rb +47 -0
  97. data/spec/datashift/mapper_utils_spec.rb +38 -0
  98. data/spec/datashift/method_binding_spec.rb +60 -0
  99. data/spec/datashift/model_method_spec.rb +109 -0
  100. data/spec/datashift/model_methods_catalogue.rb +111 -0
  101. data/spec/datashift/model_methods_collection_spec.rb +138 -0
  102. data/spec/datashift/model_methods_manager_spec.rb +329 -0
  103. data/spec/datashift/populator_spec.rb +117 -0
  104. data/spec/datashift/thor_spec.rb +314 -0
  105. data/spec/datashift/transformation/factory_spec.rb +195 -0
  106. data/spec/datashift/transformation/transformer_remove_spec.rb +43 -0
  107. data/spec/dummy/Gemfile +53 -0
  108. data/spec/dummy/Gemfile.lock +197 -0
  109. data/spec/dummy/README.rdoc +28 -0
  110. data/spec/dummy/Rakefile +6 -0
  111. data/spec/dummy/app/assets/javascripts/application.js +16 -0
  112. data/spec/dummy/app/assets/javascripts/categories.js +2 -0
  113. data/spec/dummy/app/assets/javascripts/digitals.js +2 -0
  114. data/spec/dummy/app/assets/javascripts/empties.js +2 -0
  115. data/spec/dummy/app/assets/javascripts/loader_releases.js +2 -0
  116. data/spec/dummy/app/assets/javascripts/long_and_complex_table_linked_to_versions.js +2 -0
  117. data/spec/dummy/app/assets/javascripts/milestones.js +2 -0
  118. data/spec/dummy/app/assets/javascripts/owners.js +2 -0
  119. data/spec/dummy/app/assets/javascripts/projects.js +2 -0
  120. data/spec/dummy/app/assets/javascripts/users.js +2 -0
  121. data/spec/dummy/app/assets/javascripts/versions.js +2 -0
  122. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  123. data/spec/dummy/app/assets/stylesheets/categories.css +4 -0
  124. data/spec/dummy/app/assets/stylesheets/digitals.css +4 -0
  125. data/spec/dummy/app/assets/stylesheets/empties.css +4 -0
  126. data/spec/dummy/app/assets/stylesheets/loader_releases.css +4 -0
  127. data/spec/dummy/app/assets/stylesheets/long_and_complex_table_linked_to_versions.css +4 -0
  128. data/spec/dummy/app/assets/stylesheets/milestones.css +4 -0
  129. data/spec/dummy/app/assets/stylesheets/owners.css +4 -0
  130. data/spec/dummy/app/assets/stylesheets/projects.css +4 -0
  131. data/spec/dummy/app/assets/stylesheets/scaffold.css +56 -0
  132. data/spec/dummy/app/assets/stylesheets/users.css +4 -0
  133. data/spec/dummy/app/assets/stylesheets/versions.css +4 -0
  134. data/spec/dummy/app/controllers/application_controller.rb +5 -0
  135. data/spec/dummy/app/controllers/categories_controller.rb +58 -0
  136. data/spec/dummy/app/controllers/digitals_controller.rb +58 -0
  137. data/spec/dummy/app/controllers/empties_controller.rb +58 -0
  138. data/spec/dummy/app/controllers/loader_releases_controller.rb +58 -0
  139. data/spec/dummy/app/controllers/long_and_complex_table_linked_to_versions_controller.rb +58 -0
  140. data/spec/dummy/app/controllers/milestones_controller.rb +58 -0
  141. data/spec/dummy/app/controllers/owners_controller.rb +58 -0
  142. data/spec/dummy/app/controllers/projects_controller.rb +58 -0
  143. data/spec/dummy/app/controllers/users_controller.rb +58 -0
  144. data/spec/dummy/app/controllers/versions_controller.rb +58 -0
  145. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  146. data/spec/dummy/app/helpers/categories_helper.rb +2 -0
  147. data/spec/dummy/app/helpers/digitals_helper.rb +2 -0
  148. data/spec/dummy/app/helpers/empties_helper.rb +2 -0
  149. data/spec/dummy/app/helpers/loader_releases_helper.rb +2 -0
  150. data/spec/dummy/app/helpers/long_and_complex_table_linked_to_versions_helper.rb +2 -0
  151. data/spec/dummy/app/helpers/milestones_helper.rb +2 -0
  152. data/spec/dummy/app/helpers/owners_helper.rb +2 -0
  153. data/spec/dummy/app/helpers/projects_helper.rb +2 -0
  154. data/spec/dummy/app/helpers/users_helper.rb +2 -0
  155. data/spec/dummy/app/helpers/versions_helper.rb +2 -0
  156. data/spec/dummy/app/models/category.rb +6 -0
  157. data/spec/dummy/app/models/digital.rb +22 -0
  158. data/spec/dummy/app/models/empty.rb +2 -0
  159. data/spec/dummy/app/models/loader_release.rb +10 -0
  160. data/spec/dummy/app/models/long_and_complex_table_linked_to_version.rb +6 -0
  161. data/spec/dummy/app/models/milestone.rb +15 -0
  162. data/spec/dummy/app/models/owner.rb +13 -0
  163. data/spec/dummy/app/models/project.rb +53 -0
  164. data/spec/dummy/app/models/user.rb +5 -0
  165. data/spec/dummy/app/models/version.rb +7 -0
  166. data/spec/dummy/app/views/categories/_form.html.erb +17 -0
  167. data/spec/dummy/app/views/categories/edit.html.erb +6 -0
  168. data/spec/dummy/app/views/categories/index.html.erb +25 -0
  169. data/spec/dummy/app/views/categories/new.html.erb +5 -0
  170. data/spec/dummy/app/views/categories/show.html.erb +4 -0
  171. data/spec/dummy/app/views/digitals/_form.html.erb +17 -0
  172. data/spec/dummy/app/views/digitals/edit.html.erb +6 -0
  173. data/spec/dummy/app/views/digitals/index.html.erb +25 -0
  174. data/spec/dummy/app/views/digitals/new.html.erb +5 -0
  175. data/spec/dummy/app/views/digitals/show.html.erb +4 -0
  176. data/spec/dummy/app/views/empties/_form.html.erb +17 -0
  177. data/spec/dummy/app/views/empties/edit.html.erb +6 -0
  178. data/spec/dummy/app/views/empties/index.html.erb +25 -0
  179. data/spec/dummy/app/views/empties/new.html.erb +5 -0
  180. data/spec/dummy/app/views/empties/show.html.erb +4 -0
  181. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  182. data/spec/dummy/app/views/loader_releases/_form.html.erb +17 -0
  183. data/spec/dummy/app/views/loader_releases/edit.html.erb +6 -0
  184. data/spec/dummy/app/views/loader_releases/index.html.erb +25 -0
  185. data/spec/dummy/app/views/loader_releases/new.html.erb +5 -0
  186. data/spec/dummy/app/views/loader_releases/show.html.erb +4 -0
  187. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/_form.html.erb +17 -0
  188. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/edit.html.erb +6 -0
  189. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/index.html.erb +25 -0
  190. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/new.html.erb +5 -0
  191. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/show.html.erb +4 -0
  192. data/spec/dummy/app/views/milestones/_form.html.erb +17 -0
  193. data/spec/dummy/app/views/milestones/edit.html.erb +6 -0
  194. data/spec/dummy/app/views/milestones/index.html.erb +25 -0
  195. data/spec/dummy/app/views/milestones/new.html.erb +5 -0
  196. data/spec/dummy/app/views/milestones/show.html.erb +4 -0
  197. data/spec/dummy/app/views/owners/_form.html.erb +17 -0
  198. data/spec/dummy/app/views/owners/edit.html.erb +6 -0
  199. data/spec/dummy/app/views/owners/index.html.erb +25 -0
  200. data/spec/dummy/app/views/owners/new.html.erb +5 -0
  201. data/spec/dummy/app/views/owners/show.html.erb +4 -0
  202. data/spec/dummy/app/views/projects/_form.html.erb +17 -0
  203. data/spec/dummy/app/views/projects/edit.html.erb +6 -0
  204. data/spec/dummy/app/views/projects/index.html.erb +25 -0
  205. data/spec/dummy/app/views/projects/new.html.erb +5 -0
  206. data/spec/dummy/app/views/projects/show.html.erb +4 -0
  207. data/spec/dummy/app/views/users/_form.html.erb +17 -0
  208. data/spec/dummy/app/views/users/edit.html.erb +6 -0
  209. data/spec/dummy/app/views/users/index.html.erb +25 -0
  210. data/spec/dummy/app/views/users/new.html.erb +5 -0
  211. data/spec/dummy/app/views/users/show.html.erb +4 -0
  212. data/spec/dummy/app/views/versions/_form.html.erb +17 -0
  213. data/spec/dummy/app/views/versions/edit.html.erb +6 -0
  214. data/spec/dummy/app/views/versions/index.html.erb +25 -0
  215. data/spec/dummy/app/views/versions/new.html.erb +5 -0
  216. data/spec/dummy/app/views/versions/show.html.erb +4 -0
  217. data/spec/dummy/bin/bundle +3 -0
  218. data/spec/dummy/bin/rails +9 -0
  219. data/spec/dummy/bin/rake +9 -0
  220. data/spec/dummy/bin/setup +29 -0
  221. data/spec/dummy/bin/spring +16 -0
  222. data/spec/dummy/config.ru +4 -0
  223. data/spec/dummy/config/application.rb +26 -0
  224. data/spec/dummy/config/boot.rb +3 -0
  225. data/spec/dummy/config/database.yml +25 -0
  226. data/spec/dummy/config/environment.rb +5 -0
  227. data/spec/dummy/config/environments/development.rb +41 -0
  228. data/spec/dummy/config/environments/production.rb +79 -0
  229. data/spec/dummy/config/environments/test.rb +42 -0
  230. data/spec/dummy/config/initializers/assets.rb +11 -0
  231. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  232. data/spec/dummy/config/initializers/cookies_serializer.rb +3 -0
  233. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  234. data/spec/dummy/config/initializers/inflections.rb +16 -0
  235. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  236. data/spec/dummy/config/initializers/session_store.rb +3 -0
  237. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  238. data/spec/dummy/config/locales/en.yml +23 -0
  239. data/spec/dummy/config/routes.rb +76 -0
  240. data/spec/dummy/config/secrets.yml +22 -0
  241. data/spec/dummy/db/development.sqlite3 +0 -0
  242. data/spec/dummy/db/migrate/20110803201325_create_test_bed.rb +98 -0
  243. data/spec/dummy/db/migrate/20121009161700_add_digitals.rb +24 -0
  244. data/spec/dummy/db/migrate/20161005123106_create_digitals.rb +8 -0
  245. data/spec/dummy/db/migrate/20161005123106_create_long_and_complex_table_linked_to_versions.rb +8 -0
  246. data/spec/dummy/db/migrate/20161005123107_create_loader_releases.rb +8 -0
  247. data/spec/dummy/db/migrate/20161005123108_create_owners.rb +8 -0
  248. data/spec/dummy/db/migrate/20161005123109_create_empties.rb +8 -0
  249. data/spec/dummy/db/migrate/20161005123110_create_projects.rb +8 -0
  250. data/spec/dummy/db/migrate/20161005123111_create_users.rb +8 -0
  251. data/spec/dummy/db/migrate/20161005123111_create_versions.rb +8 -0
  252. data/spec/dummy/db/migrate/20161005123112_create_milestones.rb +8 -0
  253. data/spec/dummy/db/migrate/20161005123113_create_categories.rb +8 -0
  254. data/spec/dummy/db/schema.rb +93 -0
  255. data/spec/dummy/db/seeds.rb +9 -0
  256. data/spec/dummy/db/test.sqlite3 +0 -0
  257. data/spec/dummy/log/test.log +69 -0
  258. data/spec/dummy/public/404.html +67 -0
  259. data/spec/dummy/public/422.html +67 -0
  260. data/spec/dummy/public/500.html +66 -0
  261. data/spec/dummy/public/favicon.ico +0 -0
  262. data/spec/dummy/public/robots.txt +5 -0
  263. data/spec/dummy/sandbox_example.thor +4 -0
  264. data/spec/dummy/test/controllers/categories_controller_test.rb +49 -0
  265. data/spec/dummy/test/controllers/digitals_controller_test.rb +49 -0
  266. data/spec/dummy/test/controllers/empties_controller_test.rb +49 -0
  267. data/spec/dummy/test/controllers/loader_releases_controller_test.rb +49 -0
  268. data/spec/dummy/test/controllers/long_and_complex_table_linked_to_versions_controller_test.rb +49 -0
  269. data/spec/dummy/test/controllers/milestones_controller_test.rb +49 -0
  270. data/spec/dummy/test/controllers/owners_controller_test.rb +49 -0
  271. data/spec/dummy/test/controllers/projects_controller_test.rb +49 -0
  272. data/spec/dummy/test/controllers/users_controller_test.rb +49 -0
  273. data/spec/dummy/test/controllers/versions_controller_test.rb +49 -0
  274. data/spec/dummy/test/factories/categories.rb +5 -0
  275. data/spec/dummy/test/factories/digitals.rb +5 -0
  276. data/spec/dummy/test/factories/empties.rb +5 -0
  277. data/spec/dummy/test/factories/loader_releases.rb +5 -0
  278. data/spec/dummy/test/factories/long_and_complex_table_linked_to_versions.rb +5 -0
  279. data/spec/dummy/test/factories/milestones.rb +5 -0
  280. data/spec/dummy/test/factories/owners.rb +5 -0
  281. data/spec/dummy/test/factories/projects.rb +5 -0
  282. data/spec/dummy/test/factories/users.rb +5 -0
  283. data/spec/dummy/test/factories/versions.rb +5 -0
  284. data/spec/dummy/test/models/category_test.rb +7 -0
  285. data/spec/dummy/test/models/digital_test.rb +7 -0
  286. data/spec/dummy/test/models/empty_test.rb +7 -0
  287. data/spec/dummy/test/models/loader_release_test.rb +7 -0
  288. data/spec/dummy/test/models/long_and_complex_table_linked_to_version_test.rb +7 -0
  289. data/spec/dummy/test/models/milestone_test.rb +7 -0
  290. data/spec/dummy/test/models/owner_test.rb +7 -0
  291. data/spec/dummy/test/models/project_test.rb +7 -0
  292. data/spec/dummy/test/models/user_test.rb +7 -0
  293. data/spec/dummy/test/models/version_test.rb +7 -0
  294. data/spec/dummy/test/test_helper.rb +10 -0
  295. data/spec/exporters/csv_exporter_spec.rb +240 -0
  296. data/spec/exporters/csv_generator_spec.rb +139 -0
  297. data/spec/exporters/excel_exporter_spec.rb +193 -0
  298. data/spec/exporters/excel_generator_spec.rb +181 -0
  299. data/spec/exporters/generator_base_spec.rb +45 -0
  300. data/spec/factories/categories.rb +7 -0
  301. data/spec/factories/factories.rb +18 -0
  302. data/spec/factories/milestone.rb +16 -0
  303. data/spec/factories/projects.rb +41 -0
  304. data/spec/fixtures/BadAssociationName.xls +0 -0
  305. data/spec/fixtures/DemoNegativeTesting.xls +0 -0
  306. data/spec/fixtures/ProjectConfiguration.yml +18 -0
  307. data/spec/fixtures/ProjectsMultiCategories.xls +0 -0
  308. data/spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls +0 -0
  309. data/spec/fixtures/ProjectsSingleCategories.xls +0 -0
  310. data/spec/fixtures/ProjectsSingleCategories.xlsx +0 -0
  311. data/spec/fixtures/SimpleProjects.xls +0 -0
  312. data/spec/fixtures/config/database.yml +28 -0
  313. data/spec/fixtures/csv/BadAssociationName.csv +6 -0
  314. data/spec/fixtures/csv/DemoNegativeTesting.csv +6 -0
  315. data/spec/fixtures/csv/ProjectsMultiCategories.csv +5 -0
  316. data/spec/fixtures/csv/ProjectsMultiCategoriesHeaderLookup.csv +5 -0
  317. data/spec/fixtures/csv/ProjectsSingleCategories.csv +5 -0
  318. data/spec/fixtures/csv/SimpleProjects.csv +4 -0
  319. data/spec/fixtures/db/migrate/20110803201325_create_test_bed.rb +98 -0
  320. data/spec/fixtures/db/migrate/20121009161700_add_digitals.rb +24 -0
  321. data/spec/fixtures/db/seeds.rb +9 -0
  322. data/spec/fixtures/images/DEMO_001_ror_bag.jpeg +0 -0
  323. data/spec/fixtures/images/DEMO_002_Powerstation.jpeg +0 -0
  324. data/spec/fixtures/images/DEMO_003_ror_mug.jpeg +0 -0
  325. data/spec/fixtures/images/DEMO_004_ror_ringer.jpeg +0 -0
  326. data/spec/fixtures/load_datashift.thor +3 -0
  327. data/spec/fixtures/models/category.rb +6 -0
  328. data/spec/fixtures/models/digital.rb +22 -0
  329. data/spec/fixtures/models/empty.rb +2 -0
  330. data/spec/fixtures/models/loader_release.rb +10 -0
  331. data/spec/fixtures/models/long_and_complex_table_linked_to_version.rb +6 -0
  332. data/spec/fixtures/models/milestone.rb +15 -0
  333. data/spec/fixtures/models/owner.rb +13 -0
  334. data/spec/fixtures/models/project.rb +53 -0
  335. data/spec/fixtures/models/user.rb +5 -0
  336. data/spec/fixtures/models/version.rb +7 -0
  337. data/spec/fixtures/results/exp_project_assoc_headers.xls +0 -0
  338. data/spec/fixtures/results/exp_project_collection_spec.csv +2 -0
  339. data/spec/fixtures/results/exp_project_export.xls +0 -0
  340. data/spec/fixtures/results/exp_project_first_export.xls +0 -0
  341. data/spec/fixtures/results/exp_project_plus_assoc.xls +0 -0
  342. data/spec/fixtures/results/exp_project_plus_assoc_export_spec.csv +9 -0
  343. data/spec/fixtures/results/gen_project_plus_assoc_template.xls +0 -0
  344. data/spec/fixtures/results/gen_project_plus_some_assoc_template.xls +0 -0
  345. data/spec/fixtures/results/gen_project_template.xls +0 -0
  346. data/spec/fixtures/results/project_and_assoc_in_hash_export.xls +0 -0
  347. data/spec/fixtures/results/project_and_assoc_in_json_export.csv +9 -0
  348. data/spec/fixtures/results/project_and_assoc_in_json_export.xls +0 -0
  349. data/spec/fixtures/results/project_export_spec_with_custom_delim_,.csv +2 -0
  350. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/243.csv +2 -0
  351. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/247.csv +2 -0
  352. data/spec/fixtures/results/project_plus_assoc_template.csv +1 -0
  353. data/spec/fixtures/results/project_plus_some_assoc_template.csv +1 -0
  354. data/spec/fixtures/results/project_remove_export_spec.csv +2 -0
  355. data/spec/fixtures/results/project_template.csv +1 -0
  356. data/spec/fixtures/results/project_with_methods_export_spec.csv +2 -0
  357. data/spec/fixtures/results/thor_spec_gen_project.csv +1 -0
  358. data/spec/fixtures/sandbox_example.thor +4 -0
  359. data/spec/fixtures/simple_export_spec.xls +0 -0
  360. data/spec/fixtures/simple_template_spec.xls +0 -0
  361. data/spec/fixtures/test_model_defs.rb +7 -0
  362. data/spec/loaders/csv_loader_spec.rb +206 -0
  363. data/spec/loaders/data_flow_excel_loader_spec.rb +290 -0
  364. data/spec/loaders/excel_loader_failures_spec.rb +67 -0
  365. data/spec/loaders/excel_loader_spec.rb +294 -0
  366. data/spec/loaders/loader_base_spec.rb +29 -0
  367. data/spec/loaders/paperclip_loader_spec.rb +106 -0
  368. data/spec/log/datashift.log +14930 -0
  369. data/spec/private/digitals/1/DEMO_003_ror_mug.jpeg +0 -0
  370. data/spec/private/digitals/2/DEMO_002_Powerstation.jpeg +0 -0
  371. data/spec/private/digitals/3/DEMO_004_ror_ringer.jpeg +0 -0
  372. data/spec/private/digitals/4/DEMO_001_ror_bag.jpeg +0 -0
  373. data/spec/spec_helper.rb +26 -230
  374. data/spec/support/clear_and_manage_contexts.rb +25 -0
  375. data/spec/support/database_cleaner.rb +32 -0
  376. data/spec/support/datashift_test_helpers.rb +153 -0
  377. data/spec/support/files_paths_helper.rb +13 -0
  378. data/spec/support/fixtures/results/mapping_template.yaml +15 -0
  379. data/spec/support/sandbox.rb +136 -0
  380. metadata +804 -85
  381. data/README.markdown +0 -274
  382. data/README.rdoc +0 -19
  383. data/VERSION +0 -1
  384. data/datashift.gemspec +0 -48
  385. data/lib/applications/jruby/old_pre_proxy_jexcel_file.rb +0 -437
  386. data/lib/datashift/data_transforms.rb +0 -83
  387. data/lib/datashift/mapping_file_definitions.rb +0 -88
  388. data/lib/datashift/mapping_service.rb +0 -91
  389. data/lib/datashift/method_detail.rb +0 -165
  390. data/lib/datashift/method_details_manager.rb +0 -95
  391. data/lib/datashift/method_dictionary.rb +0 -281
  392. data/lib/datashift/method_mapper.rb +0 -174
  393. data/lib/datashift/thor_base.rb +0 -38
  394. data/lib/generators/mapping_generator.rb +0 -112
  395. data/lib/helpers/core_ext/csv_file.rb +0 -33
  396. data/lib/helpers/core_ext/to_b.rb +0 -24
  397. data/lib/loaders/reporter.rb +0 -58
  398. data/lib/thor/export.thor +0 -175
  399. data/lib/thor/generate.thor +0 -191
  400. data/lib/thor/import.thor +0 -110
  401. data/lib/thor/mapping.thor +0 -65
  402. data/lib/thor/tools.thor +0 -84
  403. data/spec/Gemfile +0 -31
  404. data/spec/Gemfile.lock +0 -134
  405. data/spec/csv_exporter_spec.rb +0 -144
  406. data/spec/csv_generator_spec.rb +0 -159
  407. data/spec/csv_loader_spec.rb +0 -212
  408. data/spec/datashift_spec.rb +0 -55
  409. data/spec/excel_exporter_spec.rb +0 -199
  410. data/spec/excel_generator_spec.rb +0 -203
  411. data/spec/excel_loader_spec.rb +0 -237
  412. data/spec/excel_spec.rb +0 -203
  413. data/spec/loader_base_spec.rb +0 -166
  414. data/spec/mapping_spec.rb +0 -117
  415. data/spec/method_dictionary_spec.rb +0 -300
  416. data/spec/method_mapper_spec.rb +0 -100
  417. data/spec/model_mapper_spec.rb +0 -41
  418. data/spec/paperclip_loader_spec.rb +0 -92
  419. data/spec/populator_spec.rb +0 -128
  420. data/spec/thor_spec.rb +0 -90
  421. data/tasks/file_tasks.rake +0 -37
  422. data/tasks/word_to_seedfu.rake +0 -167
@@ -0,0 +1,88 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2015
2
+ # Author :: Tom Statter
3
+ # License:: MIT
4
+ #
5
+ # Details:: Creates, configures and caches objects for use by current context
6
+ #
7
+ # SomeModule::MyAboutToBeLoadedClass:
8
+ # datashift_defaults:
9
+ # password: 'ns_spree_123'
10
+ # available_on: <%= Time.now.to_s(:db) %>
11
+ # shipping_category: 'Default'
12
+ # phone: '9999999999'
13
+ #
14
+ # datashift_substitutions:
15
+ # order: ['#', 'NUMBER-']
16
+ # "Lineitems": ['10PAC', '5PAC']
17
+ #
18
+ # datashift_populators:
19
+ # promo_code: PromoCodePopulator
20
+ #
21
+ module DataShift
22
+ class ContextFactory
23
+
24
+ class << self; attr_accessor :config end
25
+
26
+ # Clear out map of operators to populator Class
27
+ def self.clear_populators
28
+ populators.clear
29
+ end
30
+
31
+ # Return map of operators to populator Class
32
+ def self.populators
33
+ @populators ||= {}
34
+ end
35
+
36
+ # Options :
37
+ # formatter
38
+ # populator
39
+ #
40
+ def self.configure(load_object_class, yaml_file)
41
+
42
+ @config = YAML.load( ERB.new( IO.read(yaml_file) ).result )
43
+
44
+ @config[:datashift_populators].each do |_operator, type|
45
+ populator = ::Object.const_get(type).new
46
+
47
+ populator.configure_from(load_object_class, yaml_file)
48
+
49
+ populators[@config[:datashift_populators]]
50
+ end if @config[:datashift_populators]
51
+
52
+ end
53
+
54
+ # Move to CONTEXT
55
+ # @populator = if(options[:populator].is_a?(String))
56
+ # ::Object.const_get(options[:populator]).new
57
+ # elsif(options[:populator].is_a?(Class))
58
+ # options[:populator].new
59
+ # else
60
+ # DataShift::Populator.new
61
+ # end
62
+
63
+ # Set a Populator to be used against an INBOUND operator
64
+
65
+ class << self
66
+ attr_writer :global_populator_class
67
+ end
68
+
69
+ def self.global_populator_class
70
+ @global_populator_class || DataShift::Populator
71
+ end
72
+
73
+ def self.set_populator(method_binding, klass)
74
+ operator = method_binding.is_a?(DataShift::MethodBinding) ? method_binding.operator : method_binding
75
+ populators[operator] = klass
76
+ end
77
+
78
+ def self.get_populator(method_binding)
79
+
80
+ if(populators.key?(method_binding.operator))
81
+ return populators[method_binding.operator].new
82
+ end unless method_binding.nil? || method_binding.invalid?
83
+
84
+ global_populator_class.new
85
+ end
86
+
87
+ end
88
+ end
@@ -0,0 +1,15 @@
1
+ Array.class_eval do
2
+
3
+ ARRAY_FWDABLE_EXCLUDED_METHODS = [
4
+ :class, :singleton_class, :clone, :dup, :initialize_dup, :initialize_clone,
5
+ :freeze, :methods, :singleton_methods, :protected_methods, :private_methods, :public_methods,
6
+ :instance_variables, :instance_variable_get, :instance_variable_set, :instance_variable_defined?,
7
+ :instance_of?, :kind_of?, :is_a?, :tap, :send, :public_send, :respond_to?, :respond_to_missing?,
8
+ :extend, :display, :method, :public_method, :define_singleton_method, :object_id, :equal?,
9
+ :instance_eval, :instance_exec, :__send__, :__id__
10
+ ].freeze
11
+
12
+ def self.delegated_methods_for_fwdable
13
+ Array.instance_methods - ARRAY_FWDABLE_EXCLUDED_METHODS
14
+ end
15
+ end
@@ -0,0 +1,46 @@
1
+ # Copyright:: Autotelik Media Ltd
2
+ # Author :: Tom Statter
3
+ #
4
+ require 'csv'
5
+ require 'datashift/column_packer'
6
+ require 'datashift/model_methods/model_method'
7
+
8
+ class CSV
9
+
10
+ include DataShift::ColumnPacker
11
+
12
+ # Convert an AR instance to a set of CSV columns
13
+ # Additional non instance data can be included by supplying list of methods to call
14
+ # on the record
15
+ #
16
+ # Returns the data added
17
+ #
18
+ def ar_to_csv(record, remove_list = [], method_list = [])
19
+
20
+ serializable_hash = record.serializable_hash(except: remove_list)
21
+
22
+ csv_data = serializable_hash.values.collect { |c| escape_for_csv(c) }
23
+
24
+ [*method_list].each do |x|
25
+ csv_data << escape_for_csv(record.send(x)) if record.respond_to?(x)
26
+ end
27
+
28
+ csv_data
29
+ end
30
+
31
+ def ar_to_row(record, remove_list = [], method_list = [])
32
+ add_row( ar_to_csv(record, remove_list, method_list) )
33
+ end
34
+
35
+ def ar_association_to_csv(record, model_method, _options = {})
36
+ # pack association instances into single column
37
+ csv_data = if DataShift::ModelMethod.association_type?(model_method.operator_type)
38
+ record_to_column( record.send(model_method.operator) )
39
+ else
40
+ escape_for_csv( record.send(model_method.operator) )
41
+ end
42
+
43
+ csv_data
44
+ end
45
+
46
+ end
@@ -0,0 +1,49 @@
1
+ String.class_eval do
2
+ # Convert DSL string forms into a hash
3
+ # e.g
4
+ #
5
+ # "{:name => 'autechre'}" => Hash['name'] = autechre'
6
+ # "{:cost_price => '13.45', :price => 23, :sale_price => 4.23 }"
7
+ # "{:cost_price => '13.45', :price => 23, :sale_price => 4.23 }"
8
+
9
+ def to_hash_object
10
+
11
+ h = {}
12
+
13
+ gsub(/[{}]/, '').split(',').each do |e|
14
+ e.strip!
15
+
16
+ k, v = if e.include?('=>')
17
+ e.split('=>')
18
+ else
19
+ e.split(': ')
20
+ end
21
+
22
+ k = k.gsub(/[:']/, '').strip # easier to treat all keys as strings
23
+ v = v.to_s.strip
24
+
25
+ h[k] = if v =~ /['"]/
26
+ v.gsub(/["']/, '')
27
+ elsif v =~ /^\d+$|^\d*\.\d+$|^\.\d+$/
28
+ v.to_f
29
+ else
30
+ v
31
+ end
32
+ h
33
+ end
34
+
35
+ h
36
+ end
37
+
38
+ TRUE_REGEXP = /^(yes|true|on|t|1|\-1)$/i
39
+ FALSE_REGEXP = /^(no|false|off|f|0)$/i
40
+
41
+ def to_b
42
+ case self
43
+ when TRUE_REGEXP then true
44
+ when FALSE_REGEXP then false
45
+ else
46
+ to_i != 0
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,11 @@
1
+
2
+ Object.class_eval do
3
+ def to_b
4
+ case self
5
+ when true, false then self
6
+ when nil then false
7
+ else
8
+ to_i != 0
9
+ end
10
+ end
11
+ end
@@ -1,21 +1,17 @@
1
- # Copyright:: (c) Autotelik Media Ltd 2011
1
+ # Copyright:: (c) Autotelik Media Ltd 2016
2
2
  # Author :: Tom Statter
3
- # Date :: Aug 2010
3
+ # Date :: Aug 2016
4
4
  # License:: MIT
5
5
  #
6
6
  # Details:: Module providing standard location for delimiters used in both export/import
7
- #
8
- # For example we support multiple entries in a single column, so the string
7
+ #
8
+ # For example we support multiple entries in a single column, so the string
9
9
  # needs to be formatted with recognisable delimiters seperating each of the multiple values.
10
10
  #
11
11
  module DataShift
12
12
 
13
13
  module Delimiters
14
14
 
15
- # I made these class methods, feeling delims are 'global'
16
- # I dunno now if thats good pattern or not
17
-
18
-
19
15
  # As well as just the column name, support embedding find operators for that column
20
16
  # in the heading .. i.e Column header => 'BlogPosts:user_id'
21
17
  # ... association has many BlogPosts selected via find_by_user_id
@@ -23,18 +19,19 @@ module DataShift
23
19
  # in the heading .. i.e Column header => 'BlogPosts:user_name:John Smith'
24
20
  # ... association has many BlogPosts selected via find_by_user_name("John Smith")
25
21
  #
26
- def self.column_delim
22
+ def column_delim
27
23
  @column_delim ||= ':'
28
24
  @column_delim
29
25
  end
30
26
 
31
- def self.set_column_delim(x) @column_delim = x; end
32
-
27
+ def column_delim=(x)
28
+ @column_delim = x
29
+ end
33
30
 
34
31
  # Support multiple associations being added to a base object to be specified in a single column.
35
- #
32
+ #
36
33
  # Entry represents the association to find via supplied name, value to use in the lookup.
37
- #
34
+ #
38
35
  # Default syntax :
39
36
  #
40
37
  # Name1:value1, value2|Name2:value1, value2, value3|Name3:value1, value2
@@ -51,38 +48,42 @@ module DataShift
51
48
  # => Find 3 different associations, perform lookup via column called Size
52
49
  # => Jumper.properties << [ small, medium, large ]
53
50
  #
54
- def self.name_value_delim
51
+ def name_value_delim
55
52
  @name_value_delim ||= ':'
56
53
  @name_value_delim
57
54
  end
58
55
 
59
- def self.set_name_value_delim(x) @name_value_delim = x; end
60
-
56
+ def name_value_delim=(x)
57
+ @name_value_delim = x
58
+ end
61
59
 
62
60
  # The simple seperator for a list of values whether it be
63
61
  # "Colour:red,green,blue".split(Delimiters::multi_value_delim) => [red,green,blue]
64
62
  # {name => value, n2 => v2}.split(Delimiters::multi_value_delim) => [ [name => value], [n2 => v2] ]
65
63
 
66
- def self.multi_value_delim
64
+ def multi_value_delim
67
65
  @multi_value_delim ||= ','
68
66
  end
69
-
70
- def self.set_multi_value_delim(x) @multi_value_delim = x; end
67
+
68
+ def multi_value_delim=(x)
69
+ @multi_value_delim = x
70
+ end
71
71
 
72
72
  # Objects can be created with multiple facets in single columns.
73
73
  # In this example a single Product can be configured with a consolidated mime and print types
74
- #
75
- # mime_type:jpeg,PDF ; print_type:colour equivalent to
76
- #
74
+ #
75
+ # mime_type:jpeg,PDF ; print_type:colour equivalent to
76
+ #
77
77
  # => mime_type:jpeg;print_type:colour | mime_type:PDF; print_type:colour
78
-
79
- def self.multi_facet_delim
78
+
79
+ def multi_facet_delim
80
80
  @multi_facet_delim ||= ';'
81
81
  end
82
-
83
- def self.setmulti_facet_delim(x) @multi_facet_delim = x; end
84
-
85
-
82
+
83
+ def setmulti_facet_delim(x)
84
+ @multi_facet_delim = x
85
+ end
86
+
86
87
  # Multiple objects can be embedded in single columns.
87
88
  # In this example a single Category column contains 3 separate entries, New, SecondHand, Retro
88
89
  # object creation/update via hash (which hopefully we should be able to just forward to AR)
@@ -90,39 +91,40 @@ module DataShift
90
91
  # | Category |
91
92
  # 'name =>New, :a => 1, :b => 2|name => SecondHand, :a => 6, :b => 34|Name:Old, :a => 12, :b => 67', 'Next Column'
92
93
  #
93
- def self.multi_assoc_delim
94
+ def multi_assoc_delim
94
95
  @multi_assoc_delim ||= '|'
95
96
  @multi_assoc_delim
96
97
  end
97
98
 
98
- def self.set_multi_assoc_delim(x) @multi_assoc_delim = x; end
99
-
100
-
99
+ def multi_assoc_delim=(x)
100
+ @multi_assoc_delim = x
101
+ end
102
+
101
103
  # Delimiters for {:abc => 2, :efg => 'some text}
102
-
103
- def self.attribute_list_start
104
- @attribute_list_start ||= '{';
104
+
105
+ def attribute_list_start
106
+ @attribute_list_start ||= '{'
105
107
  end
106
108
 
107
- def self.attribute_list_start=(x) @attribute_list_start = x; end
108
-
109
- def self.attribute_list_end
109
+ attr_writer :attribute_list_start
110
+
111
+ def attribute_list_end
110
112
  @attribute_list_end ||= '}'
111
113
  end
112
-
113
- def self.attribute_list_end=(x)
114
- @attribute_list_end = x;
115
- end
116
-
117
- def self.csv_delim
118
- @csv_delim ||= ','
119
- @csv_delim
114
+
115
+ attr_writer :attribute_list_end
116
+
117
+ attr_writer :text_delim
118
+
119
+ attr_writer :key_value_sep
120
+
121
+ def csv_delimiter
122
+ @csv_delimiter ||= ','
120
123
  end
121
-
122
- def self.csv_delim=(x) set_csv_delim(x); end
123
- def self.set_csv_delim(x) @csv_delim = x; end
124
124
 
125
- def self.eol
125
+ attr_writer :csv_delimiter
126
+
127
+ def eol
126
128
  "\n"
127
129
  end
128
130
 
@@ -131,20 +133,12 @@ module DataShift
131
133
  @text_delim ||= "\'"
132
134
  end
133
135
 
134
- def text_delim=(x)
135
- @text_delim = x
136
- end
137
-
138
136
  # seperator for identifying normal key value pairs
139
137
 
140
- def self.key_value_sep
141
- @key_value_sep ||= "=>" #TODO check Ruby version and use appropriate has style ?
142
- end
143
-
144
- def self.key_value_sep=(x)
145
- @key_value_sep = x
138
+ def key_value_sep
139
+ @key_value_sep ||= ' ' # for now assume everyone wants newer less verbose style
146
140
  end
147
141
 
148
142
  end
149
143
 
150
- end
144
+ end
@@ -0,0 +1,137 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2015
2
+ # Author :: Tom Statter
3
+ # Date :: Jan 2015
4
+ # License:: MIT
5
+ #
6
+ # Details:: The document we are dealing with
7
+ # Holds the headers, load object and access to reports for this Document
8
+ # Holds the current node_context (e.g column being processed) in relation to the Document
9
+ #
10
+ module DataShift
11
+
12
+ class DocContext
13
+
14
+ include DataShift::Logging
15
+
16
+ attr_reader :klass
17
+
18
+ attr_accessor :load_object
19
+
20
+ # The current Node Context - method_binding (includes inbound_column), row, data
21
+ attr_accessor :node_context
22
+
23
+ # The inbound document headers
24
+ attr_accessor :headers
25
+
26
+ attr_accessor :progress_monitor, :reporters
27
+
28
+ def initialize( klass )
29
+ reset_klass(klass)
30
+
31
+ @headers = DataShift::Headers.new(:na)
32
+
33
+ @progress_monitor = ProgressMonitor.new
34
+
35
+ @reporters = [DataShift::Reporters::BasicStdoutReporter.new(@progress_monitor)]
36
+ end
37
+
38
+ def reset_klass( klass )
39
+ @klass = klass
40
+ reset
41
+ end
42
+
43
+ # Reset the database object to be populated
44
+ #
45
+ def reset(object = nil)
46
+ @node_context = DataShift::EmptyContext.new
47
+
48
+ @load_object = DataShift::LoadObject.new(object || new_load_object)
49
+ end
50
+
51
+ def new_load_object
52
+ @load_object = klass.new
53
+ @load_object
54
+ end
55
+
56
+ def create_node_context(method_binding, row_idx, data)
57
+ @node_context = DataShift::NodeContext.new(self, method_binding, row_idx, data)
58
+ @node_context
59
+ end
60
+
61
+ # Only save object if all columns ok, or allow errors in individual columns
62
+ def all_or_nothing?
63
+ true
64
+ # TODO: - read in from configration
65
+ end
66
+
67
+ # TOFIX - use delegation to doc_context.progress_monitor
68
+ def loaded_count
69
+ progress_monitor.loaded_count
70
+ end
71
+
72
+ def failed_count
73
+ progress_monitor.failed_count
74
+ end
75
+
76
+ def current_errors
77
+ load_object.errors.full_messages
78
+ end
79
+
80
+ def errors?
81
+ !load_object.errors.empty? || progress_monitor.current_status == :failure
82
+ end
83
+
84
+ # This method usually called during processing to avoid errors with associations like
85
+ # <ActiveRecord::RecordNotSaved: You cannot call create unless the parent is saved>
86
+ # If the object is still invalid at this point probably indicates compulsory
87
+ # columns on model have not been processed before associations on that model.
88
+ #
89
+ # You can provide a custom sort function to the Collection of model methods (which are comparable) to fix this.
90
+ #
91
+ def save_if_new
92
+ return unless load_object.new_record?
93
+
94
+ return save if load_object.valid?
95
+
96
+ raise DataShift::SaveError, "Cannot Save Invalid #{load_object.class} Record : #{current_errors}"
97
+ end
98
+
99
+ # Save the object and then report the outcome to ProgressMonitor, as either success or failure
100
+ #
101
+ def save_and_monitor_progress
102
+ if(errors? && all_or_nothing?)
103
+ # Error already logged with doc_context.failure
104
+ logger.warn "SAVE skipped due to Errors for Row #{node_context.row_index} - #{node_context.method_binding.spp}"
105
+ else
106
+ if save
107
+ @progress_monitor.success(load_object)
108
+
109
+ logger.info("Successfully Processed [#{node_context.method_binding.spp}]")
110
+ logger.info("Successfully SAVED Object #{@progress_monitor.success_inbound_count} - [#{load_object.id}]")
111
+ else
112
+
113
+ failed = FailureData.new(load_object, node_context, current_errors)
114
+
115
+ @progress_monitor.failure(failed)
116
+
117
+ logger.info("Failed to Process [#{node_context.method_binding.spp}]")
118
+ logger.info("Failed to SAVE Object #{@progress_monitor.success_inbound_count} - [#{load_object.inspect}]")
119
+ end
120
+ end
121
+ end
122
+
123
+ def save
124
+ return false unless load_object
125
+
126
+ logger.debug("SAVING #{load_object.class} : #{load_object.inspect}")
127
+ begin
128
+ load_object.save!
129
+ rescue => e
130
+ logger.error( "Save Error : #{e.inspect} on #{load_object.class}")
131
+ logger.error(e.backtrace)
132
+ false
133
+ end
134
+ end
135
+
136
+ end
137
+ end