datashift 0.16.0 → 0.40.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (422) hide show
  1. checksums.yaml +4 -4
  2. data/{LICENSE.txt → LICENSE} +0 -0
  3. data/Rakefile +1 -20
  4. data/datashift.thor +125 -0
  5. data/lib/applications/apache_poi_extensions.rb +21 -52
  6. data/lib/applications/excel.rb +64 -57
  7. data/lib/applications/hssf_row_extensions.rb +66 -0
  8. data/lib/applications/jexcel_file.rb +99 -95
  9. data/lib/applications/jexcel_file_extensions.rb +76 -83
  10. data/lib/applications/jruby/word.rb +36 -36
  11. data/lib/applications/ruby_poi_translations.rb +34 -32
  12. data/lib/applications/spreadsheet_extensions.rb +21 -19
  13. data/lib/datashift.rb +49 -59
  14. data/lib/datashift/binder.rb +217 -0
  15. data/lib/datashift/column_packer.rb +21 -72
  16. data/lib/datashift/configuration.rb +317 -0
  17. data/lib/datashift/context_factory.rb +88 -0
  18. data/lib/datashift/core_ext/array.rb +15 -0
  19. data/lib/datashift/core_ext/csv_ext.rb +46 -0
  20. data/lib/datashift/core_ext/string.rb +49 -0
  21. data/lib/datashift/core_ext/to_b.rb +11 -0
  22. data/lib/datashift/delimiters.rb +55 -61
  23. data/lib/datashift/doc_context.rb +137 -0
  24. data/lib/datashift/excel_base.rb +93 -81
  25. data/lib/datashift/exceptions.rb +30 -28
  26. data/lib/datashift/file_definitions.rb +44 -39
  27. data/lib/datashift/guards.rb +5 -5
  28. data/lib/datashift/header.rb +25 -0
  29. data/lib/datashift/headers.rb +94 -0
  30. data/lib/datashift/inbound_data/column.rb +44 -0
  31. data/lib/datashift/inbound_data/lookup_support.rb +33 -0
  32. data/lib/datashift/inbound_data/method_binding.rb +139 -0
  33. data/lib/datashift/load_object.rb +37 -12
  34. data/lib/datashift/logging.rb +54 -27
  35. data/lib/datashift/mandatory.rb +39 -0
  36. data/lib/datashift/mapping/data_flow_schema.rb +198 -0
  37. data/lib/datashift/{model_mapper.rb → mapping/mapper_utils.rb} +30 -10
  38. data/lib/datashift/model_methods/catalogue.rb +183 -0
  39. data/lib/datashift/model_methods/collection.rb +140 -0
  40. data/lib/datashift/model_methods/model_method.rb +162 -0
  41. data/lib/datashift/model_methods/model_methods_manager.rb +76 -0
  42. data/lib/datashift/model_methods/operator.rb +62 -0
  43. data/lib/datashift/node_collection.rb +26 -0
  44. data/lib/datashift/node_context.rb +68 -0
  45. data/lib/datashift/populator.rb +308 -282
  46. data/lib/datashift/progress_monitor.rb +91 -0
  47. data/lib/datashift/querying.rb +110 -52
  48. data/lib/datashift/templates/import_export_config.erb +55 -0
  49. data/lib/datashift/transformation/factory.rb +219 -0
  50. data/lib/datashift/transformation/remove.rb +44 -0
  51. data/lib/datashift/version.rb +3 -0
  52. data/lib/exporters/configuration.rb +84 -0
  53. data/lib/exporters/csv_exporter.rb +54 -52
  54. data/lib/exporters/excel_exporter.rb +80 -61
  55. data/lib/exporters/exporter_base.rb +8 -8
  56. data/lib/generators/config_generator.rb +218 -0
  57. data/lib/generators/csv_generator.rb +13 -70
  58. data/lib/generators/excel_generator.rb +23 -111
  59. data/lib/generators/generator_base.rb +15 -70
  60. data/lib/loaders/configuration.rb +90 -0
  61. data/lib/loaders/csv_loader.rb +63 -101
  62. data/lib/loaders/excel_loader.rb +71 -156
  63. data/lib/loaders/failure_data.rb +40 -0
  64. data/lib/loaders/file_loader.rb +16 -0
  65. data/lib/loaders/loader_base.rb +82 -410
  66. data/lib/loaders/loader_factory.rb +42 -0
  67. data/lib/loaders/paperclip/attachment_loader.rb +157 -140
  68. data/lib/loaders/paperclip/datashift_paperclip.rb +18 -35
  69. data/lib/loaders/paperclip/image_loading.rb +40 -35
  70. data/lib/loaders/reporters/basic_stdout_reporter.rb +40 -0
  71. data/lib/loaders/reporters/reporter.rb +26 -0
  72. data/lib/tasks/config.thor +65 -0
  73. data/{tasks → lib/tasks}/config/seed_fu_product_template.erb +0 -0
  74. data/{tasks → lib/tasks}/config/tidy_config.txt +0 -0
  75. data/lib/tasks/export.thor +192 -0
  76. data/lib/tasks/generate.thor +190 -0
  77. data/lib/tasks/import.thor +142 -0
  78. data/lib/{thor → tasks}/paperclip.thor +69 -69
  79. data/{tasks → lib/tasks/to_convert_to_thor}/db_tasks.rake +20 -20
  80. data/lib/tasks/tools.thor +109 -0
  81. data/spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg +0 -0
  82. data/spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg +0 -0
  83. data/spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg +0 -0
  84. data/spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg +0 -0
  85. data/spec/datashift/binder_spec.rb +266 -0
  86. data/spec/datashift/config_generator_spec.rb +186 -0
  87. data/spec/datashift/configuration.rb +66 -0
  88. data/spec/datashift/context_factory_spec.rb +63 -0
  89. data/spec/datashift/data_flow_schema_spec.rb +150 -0
  90. data/spec/datashift/datashift_spec.rb +52 -0
  91. data/spec/datashift/excel_base_spec.rb +57 -0
  92. data/spec/datashift/excel_spec.rb +188 -0
  93. data/spec/datashift/failure_data_spec.rb +27 -0
  94. data/spec/{file_definitions.rb → datashift/file_definitions.rb} +9 -10
  95. data/spec/datashift/headers_spec.rb +56 -0
  96. data/spec/datashift/inbound_data_spec.rb +47 -0
  97. data/spec/datashift/mapper_utils_spec.rb +38 -0
  98. data/spec/datashift/method_binding_spec.rb +60 -0
  99. data/spec/datashift/model_method_spec.rb +109 -0
  100. data/spec/datashift/model_methods_catalogue.rb +111 -0
  101. data/spec/datashift/model_methods_collection_spec.rb +138 -0
  102. data/spec/datashift/model_methods_manager_spec.rb +329 -0
  103. data/spec/datashift/populator_spec.rb +117 -0
  104. data/spec/datashift/thor_spec.rb +314 -0
  105. data/spec/datashift/transformation/factory_spec.rb +195 -0
  106. data/spec/datashift/transformation/transformer_remove_spec.rb +43 -0
  107. data/spec/dummy/Gemfile +53 -0
  108. data/spec/dummy/Gemfile.lock +197 -0
  109. data/spec/dummy/README.rdoc +28 -0
  110. data/spec/dummy/Rakefile +6 -0
  111. data/spec/dummy/app/assets/javascripts/application.js +16 -0
  112. data/spec/dummy/app/assets/javascripts/categories.js +2 -0
  113. data/spec/dummy/app/assets/javascripts/digitals.js +2 -0
  114. data/spec/dummy/app/assets/javascripts/empties.js +2 -0
  115. data/spec/dummy/app/assets/javascripts/loader_releases.js +2 -0
  116. data/spec/dummy/app/assets/javascripts/long_and_complex_table_linked_to_versions.js +2 -0
  117. data/spec/dummy/app/assets/javascripts/milestones.js +2 -0
  118. data/spec/dummy/app/assets/javascripts/owners.js +2 -0
  119. data/spec/dummy/app/assets/javascripts/projects.js +2 -0
  120. data/spec/dummy/app/assets/javascripts/users.js +2 -0
  121. data/spec/dummy/app/assets/javascripts/versions.js +2 -0
  122. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  123. data/spec/dummy/app/assets/stylesheets/categories.css +4 -0
  124. data/spec/dummy/app/assets/stylesheets/digitals.css +4 -0
  125. data/spec/dummy/app/assets/stylesheets/empties.css +4 -0
  126. data/spec/dummy/app/assets/stylesheets/loader_releases.css +4 -0
  127. data/spec/dummy/app/assets/stylesheets/long_and_complex_table_linked_to_versions.css +4 -0
  128. data/spec/dummy/app/assets/stylesheets/milestones.css +4 -0
  129. data/spec/dummy/app/assets/stylesheets/owners.css +4 -0
  130. data/spec/dummy/app/assets/stylesheets/projects.css +4 -0
  131. data/spec/dummy/app/assets/stylesheets/scaffold.css +56 -0
  132. data/spec/dummy/app/assets/stylesheets/users.css +4 -0
  133. data/spec/dummy/app/assets/stylesheets/versions.css +4 -0
  134. data/spec/dummy/app/controllers/application_controller.rb +5 -0
  135. data/spec/dummy/app/controllers/categories_controller.rb +58 -0
  136. data/spec/dummy/app/controllers/digitals_controller.rb +58 -0
  137. data/spec/dummy/app/controllers/empties_controller.rb +58 -0
  138. data/spec/dummy/app/controllers/loader_releases_controller.rb +58 -0
  139. data/spec/dummy/app/controllers/long_and_complex_table_linked_to_versions_controller.rb +58 -0
  140. data/spec/dummy/app/controllers/milestones_controller.rb +58 -0
  141. data/spec/dummy/app/controllers/owners_controller.rb +58 -0
  142. data/spec/dummy/app/controllers/projects_controller.rb +58 -0
  143. data/spec/dummy/app/controllers/users_controller.rb +58 -0
  144. data/spec/dummy/app/controllers/versions_controller.rb +58 -0
  145. data/spec/dummy/app/helpers/application_helper.rb +2 -0
  146. data/spec/dummy/app/helpers/categories_helper.rb +2 -0
  147. data/spec/dummy/app/helpers/digitals_helper.rb +2 -0
  148. data/spec/dummy/app/helpers/empties_helper.rb +2 -0
  149. data/spec/dummy/app/helpers/loader_releases_helper.rb +2 -0
  150. data/spec/dummy/app/helpers/long_and_complex_table_linked_to_versions_helper.rb +2 -0
  151. data/spec/dummy/app/helpers/milestones_helper.rb +2 -0
  152. data/spec/dummy/app/helpers/owners_helper.rb +2 -0
  153. data/spec/dummy/app/helpers/projects_helper.rb +2 -0
  154. data/spec/dummy/app/helpers/users_helper.rb +2 -0
  155. data/spec/dummy/app/helpers/versions_helper.rb +2 -0
  156. data/spec/dummy/app/models/category.rb +6 -0
  157. data/spec/dummy/app/models/digital.rb +22 -0
  158. data/spec/dummy/app/models/empty.rb +2 -0
  159. data/spec/dummy/app/models/loader_release.rb +10 -0
  160. data/spec/dummy/app/models/long_and_complex_table_linked_to_version.rb +6 -0
  161. data/spec/dummy/app/models/milestone.rb +15 -0
  162. data/spec/dummy/app/models/owner.rb +13 -0
  163. data/spec/dummy/app/models/project.rb +53 -0
  164. data/spec/dummy/app/models/user.rb +5 -0
  165. data/spec/dummy/app/models/version.rb +7 -0
  166. data/spec/dummy/app/views/categories/_form.html.erb +17 -0
  167. data/spec/dummy/app/views/categories/edit.html.erb +6 -0
  168. data/spec/dummy/app/views/categories/index.html.erb +25 -0
  169. data/spec/dummy/app/views/categories/new.html.erb +5 -0
  170. data/spec/dummy/app/views/categories/show.html.erb +4 -0
  171. data/spec/dummy/app/views/digitals/_form.html.erb +17 -0
  172. data/spec/dummy/app/views/digitals/edit.html.erb +6 -0
  173. data/spec/dummy/app/views/digitals/index.html.erb +25 -0
  174. data/spec/dummy/app/views/digitals/new.html.erb +5 -0
  175. data/spec/dummy/app/views/digitals/show.html.erb +4 -0
  176. data/spec/dummy/app/views/empties/_form.html.erb +17 -0
  177. data/spec/dummy/app/views/empties/edit.html.erb +6 -0
  178. data/spec/dummy/app/views/empties/index.html.erb +25 -0
  179. data/spec/dummy/app/views/empties/new.html.erb +5 -0
  180. data/spec/dummy/app/views/empties/show.html.erb +4 -0
  181. data/spec/dummy/app/views/layouts/application.html.erb +14 -0
  182. data/spec/dummy/app/views/loader_releases/_form.html.erb +17 -0
  183. data/spec/dummy/app/views/loader_releases/edit.html.erb +6 -0
  184. data/spec/dummy/app/views/loader_releases/index.html.erb +25 -0
  185. data/spec/dummy/app/views/loader_releases/new.html.erb +5 -0
  186. data/spec/dummy/app/views/loader_releases/show.html.erb +4 -0
  187. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/_form.html.erb +17 -0
  188. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/edit.html.erb +6 -0
  189. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/index.html.erb +25 -0
  190. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/new.html.erb +5 -0
  191. data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/show.html.erb +4 -0
  192. data/spec/dummy/app/views/milestones/_form.html.erb +17 -0
  193. data/spec/dummy/app/views/milestones/edit.html.erb +6 -0
  194. data/spec/dummy/app/views/milestones/index.html.erb +25 -0
  195. data/spec/dummy/app/views/milestones/new.html.erb +5 -0
  196. data/spec/dummy/app/views/milestones/show.html.erb +4 -0
  197. data/spec/dummy/app/views/owners/_form.html.erb +17 -0
  198. data/spec/dummy/app/views/owners/edit.html.erb +6 -0
  199. data/spec/dummy/app/views/owners/index.html.erb +25 -0
  200. data/spec/dummy/app/views/owners/new.html.erb +5 -0
  201. data/spec/dummy/app/views/owners/show.html.erb +4 -0
  202. data/spec/dummy/app/views/projects/_form.html.erb +17 -0
  203. data/spec/dummy/app/views/projects/edit.html.erb +6 -0
  204. data/spec/dummy/app/views/projects/index.html.erb +25 -0
  205. data/spec/dummy/app/views/projects/new.html.erb +5 -0
  206. data/spec/dummy/app/views/projects/show.html.erb +4 -0
  207. data/spec/dummy/app/views/users/_form.html.erb +17 -0
  208. data/spec/dummy/app/views/users/edit.html.erb +6 -0
  209. data/spec/dummy/app/views/users/index.html.erb +25 -0
  210. data/spec/dummy/app/views/users/new.html.erb +5 -0
  211. data/spec/dummy/app/views/users/show.html.erb +4 -0
  212. data/spec/dummy/app/views/versions/_form.html.erb +17 -0
  213. data/spec/dummy/app/views/versions/edit.html.erb +6 -0
  214. data/spec/dummy/app/views/versions/index.html.erb +25 -0
  215. data/spec/dummy/app/views/versions/new.html.erb +5 -0
  216. data/spec/dummy/app/views/versions/show.html.erb +4 -0
  217. data/spec/dummy/bin/bundle +3 -0
  218. data/spec/dummy/bin/rails +9 -0
  219. data/spec/dummy/bin/rake +9 -0
  220. data/spec/dummy/bin/setup +29 -0
  221. data/spec/dummy/bin/spring +16 -0
  222. data/spec/dummy/config.ru +4 -0
  223. data/spec/dummy/config/application.rb +26 -0
  224. data/spec/dummy/config/boot.rb +3 -0
  225. data/spec/dummy/config/database.yml +25 -0
  226. data/spec/dummy/config/environment.rb +5 -0
  227. data/spec/dummy/config/environments/development.rb +41 -0
  228. data/spec/dummy/config/environments/production.rb +79 -0
  229. data/spec/dummy/config/environments/test.rb +42 -0
  230. data/spec/dummy/config/initializers/assets.rb +11 -0
  231. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  232. data/spec/dummy/config/initializers/cookies_serializer.rb +3 -0
  233. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  234. data/spec/dummy/config/initializers/inflections.rb +16 -0
  235. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  236. data/spec/dummy/config/initializers/session_store.rb +3 -0
  237. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  238. data/spec/dummy/config/locales/en.yml +23 -0
  239. data/spec/dummy/config/routes.rb +76 -0
  240. data/spec/dummy/config/secrets.yml +22 -0
  241. data/spec/dummy/db/development.sqlite3 +0 -0
  242. data/spec/dummy/db/migrate/20110803201325_create_test_bed.rb +98 -0
  243. data/spec/dummy/db/migrate/20121009161700_add_digitals.rb +24 -0
  244. data/spec/dummy/db/migrate/20161005123106_create_digitals.rb +8 -0
  245. data/spec/dummy/db/migrate/20161005123106_create_long_and_complex_table_linked_to_versions.rb +8 -0
  246. data/spec/dummy/db/migrate/20161005123107_create_loader_releases.rb +8 -0
  247. data/spec/dummy/db/migrate/20161005123108_create_owners.rb +8 -0
  248. data/spec/dummy/db/migrate/20161005123109_create_empties.rb +8 -0
  249. data/spec/dummy/db/migrate/20161005123110_create_projects.rb +8 -0
  250. data/spec/dummy/db/migrate/20161005123111_create_users.rb +8 -0
  251. data/spec/dummy/db/migrate/20161005123111_create_versions.rb +8 -0
  252. data/spec/dummy/db/migrate/20161005123112_create_milestones.rb +8 -0
  253. data/spec/dummy/db/migrate/20161005123113_create_categories.rb +8 -0
  254. data/spec/dummy/db/schema.rb +93 -0
  255. data/spec/dummy/db/seeds.rb +9 -0
  256. data/spec/dummy/db/test.sqlite3 +0 -0
  257. data/spec/dummy/log/test.log +69 -0
  258. data/spec/dummy/public/404.html +67 -0
  259. data/spec/dummy/public/422.html +67 -0
  260. data/spec/dummy/public/500.html +66 -0
  261. data/spec/dummy/public/favicon.ico +0 -0
  262. data/spec/dummy/public/robots.txt +5 -0
  263. data/spec/dummy/sandbox_example.thor +4 -0
  264. data/spec/dummy/test/controllers/categories_controller_test.rb +49 -0
  265. data/spec/dummy/test/controllers/digitals_controller_test.rb +49 -0
  266. data/spec/dummy/test/controllers/empties_controller_test.rb +49 -0
  267. data/spec/dummy/test/controllers/loader_releases_controller_test.rb +49 -0
  268. data/spec/dummy/test/controllers/long_and_complex_table_linked_to_versions_controller_test.rb +49 -0
  269. data/spec/dummy/test/controllers/milestones_controller_test.rb +49 -0
  270. data/spec/dummy/test/controllers/owners_controller_test.rb +49 -0
  271. data/spec/dummy/test/controllers/projects_controller_test.rb +49 -0
  272. data/spec/dummy/test/controllers/users_controller_test.rb +49 -0
  273. data/spec/dummy/test/controllers/versions_controller_test.rb +49 -0
  274. data/spec/dummy/test/factories/categories.rb +5 -0
  275. data/spec/dummy/test/factories/digitals.rb +5 -0
  276. data/spec/dummy/test/factories/empties.rb +5 -0
  277. data/spec/dummy/test/factories/loader_releases.rb +5 -0
  278. data/spec/dummy/test/factories/long_and_complex_table_linked_to_versions.rb +5 -0
  279. data/spec/dummy/test/factories/milestones.rb +5 -0
  280. data/spec/dummy/test/factories/owners.rb +5 -0
  281. data/spec/dummy/test/factories/projects.rb +5 -0
  282. data/spec/dummy/test/factories/users.rb +5 -0
  283. data/spec/dummy/test/factories/versions.rb +5 -0
  284. data/spec/dummy/test/models/category_test.rb +7 -0
  285. data/spec/dummy/test/models/digital_test.rb +7 -0
  286. data/spec/dummy/test/models/empty_test.rb +7 -0
  287. data/spec/dummy/test/models/loader_release_test.rb +7 -0
  288. data/spec/dummy/test/models/long_and_complex_table_linked_to_version_test.rb +7 -0
  289. data/spec/dummy/test/models/milestone_test.rb +7 -0
  290. data/spec/dummy/test/models/owner_test.rb +7 -0
  291. data/spec/dummy/test/models/project_test.rb +7 -0
  292. data/spec/dummy/test/models/user_test.rb +7 -0
  293. data/spec/dummy/test/models/version_test.rb +7 -0
  294. data/spec/dummy/test/test_helper.rb +10 -0
  295. data/spec/exporters/csv_exporter_spec.rb +240 -0
  296. data/spec/exporters/csv_generator_spec.rb +139 -0
  297. data/spec/exporters/excel_exporter_spec.rb +193 -0
  298. data/spec/exporters/excel_generator_spec.rb +181 -0
  299. data/spec/exporters/generator_base_spec.rb +45 -0
  300. data/spec/factories/categories.rb +7 -0
  301. data/spec/factories/factories.rb +18 -0
  302. data/spec/factories/milestone.rb +16 -0
  303. data/spec/factories/projects.rb +41 -0
  304. data/spec/fixtures/BadAssociationName.xls +0 -0
  305. data/spec/fixtures/DemoNegativeTesting.xls +0 -0
  306. data/spec/fixtures/ProjectConfiguration.yml +18 -0
  307. data/spec/fixtures/ProjectsMultiCategories.xls +0 -0
  308. data/spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls +0 -0
  309. data/spec/fixtures/ProjectsSingleCategories.xls +0 -0
  310. data/spec/fixtures/ProjectsSingleCategories.xlsx +0 -0
  311. data/spec/fixtures/SimpleProjects.xls +0 -0
  312. data/spec/fixtures/config/database.yml +28 -0
  313. data/spec/fixtures/csv/BadAssociationName.csv +6 -0
  314. data/spec/fixtures/csv/DemoNegativeTesting.csv +6 -0
  315. data/spec/fixtures/csv/ProjectsMultiCategories.csv +5 -0
  316. data/spec/fixtures/csv/ProjectsMultiCategoriesHeaderLookup.csv +5 -0
  317. data/spec/fixtures/csv/ProjectsSingleCategories.csv +5 -0
  318. data/spec/fixtures/csv/SimpleProjects.csv +4 -0
  319. data/spec/fixtures/db/migrate/20110803201325_create_test_bed.rb +98 -0
  320. data/spec/fixtures/db/migrate/20121009161700_add_digitals.rb +24 -0
  321. data/spec/fixtures/db/seeds.rb +9 -0
  322. data/spec/fixtures/images/DEMO_001_ror_bag.jpeg +0 -0
  323. data/spec/fixtures/images/DEMO_002_Powerstation.jpeg +0 -0
  324. data/spec/fixtures/images/DEMO_003_ror_mug.jpeg +0 -0
  325. data/spec/fixtures/images/DEMO_004_ror_ringer.jpeg +0 -0
  326. data/spec/fixtures/load_datashift.thor +3 -0
  327. data/spec/fixtures/models/category.rb +6 -0
  328. data/spec/fixtures/models/digital.rb +22 -0
  329. data/spec/fixtures/models/empty.rb +2 -0
  330. data/spec/fixtures/models/loader_release.rb +10 -0
  331. data/spec/fixtures/models/long_and_complex_table_linked_to_version.rb +6 -0
  332. data/spec/fixtures/models/milestone.rb +15 -0
  333. data/spec/fixtures/models/owner.rb +13 -0
  334. data/spec/fixtures/models/project.rb +53 -0
  335. data/spec/fixtures/models/user.rb +5 -0
  336. data/spec/fixtures/models/version.rb +7 -0
  337. data/spec/fixtures/results/exp_project_assoc_headers.xls +0 -0
  338. data/spec/fixtures/results/exp_project_collection_spec.csv +2 -0
  339. data/spec/fixtures/results/exp_project_export.xls +0 -0
  340. data/spec/fixtures/results/exp_project_first_export.xls +0 -0
  341. data/spec/fixtures/results/exp_project_plus_assoc.xls +0 -0
  342. data/spec/fixtures/results/exp_project_plus_assoc_export_spec.csv +9 -0
  343. data/spec/fixtures/results/gen_project_plus_assoc_template.xls +0 -0
  344. data/spec/fixtures/results/gen_project_plus_some_assoc_template.xls +0 -0
  345. data/spec/fixtures/results/gen_project_template.xls +0 -0
  346. data/spec/fixtures/results/project_and_assoc_in_hash_export.xls +0 -0
  347. data/spec/fixtures/results/project_and_assoc_in_json_export.csv +9 -0
  348. data/spec/fixtures/results/project_and_assoc_in_json_export.xls +0 -0
  349. data/spec/fixtures/results/project_export_spec_with_custom_delim_,.csv +2 -0
  350. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/243.csv +2 -0
  351. data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/247.csv +2 -0
  352. data/spec/fixtures/results/project_plus_assoc_template.csv +1 -0
  353. data/spec/fixtures/results/project_plus_some_assoc_template.csv +1 -0
  354. data/spec/fixtures/results/project_remove_export_spec.csv +2 -0
  355. data/spec/fixtures/results/project_template.csv +1 -0
  356. data/spec/fixtures/results/project_with_methods_export_spec.csv +2 -0
  357. data/spec/fixtures/results/thor_spec_gen_project.csv +1 -0
  358. data/spec/fixtures/sandbox_example.thor +4 -0
  359. data/spec/fixtures/simple_export_spec.xls +0 -0
  360. data/spec/fixtures/simple_template_spec.xls +0 -0
  361. data/spec/fixtures/test_model_defs.rb +7 -0
  362. data/spec/loaders/csv_loader_spec.rb +206 -0
  363. data/spec/loaders/data_flow_excel_loader_spec.rb +290 -0
  364. data/spec/loaders/excel_loader_failures_spec.rb +67 -0
  365. data/spec/loaders/excel_loader_spec.rb +294 -0
  366. data/spec/loaders/loader_base_spec.rb +29 -0
  367. data/spec/loaders/paperclip_loader_spec.rb +106 -0
  368. data/spec/log/datashift.log +14930 -0
  369. data/spec/private/digitals/1/DEMO_003_ror_mug.jpeg +0 -0
  370. data/spec/private/digitals/2/DEMO_002_Powerstation.jpeg +0 -0
  371. data/spec/private/digitals/3/DEMO_004_ror_ringer.jpeg +0 -0
  372. data/spec/private/digitals/4/DEMO_001_ror_bag.jpeg +0 -0
  373. data/spec/spec_helper.rb +26 -230
  374. data/spec/support/clear_and_manage_contexts.rb +25 -0
  375. data/spec/support/database_cleaner.rb +32 -0
  376. data/spec/support/datashift_test_helpers.rb +153 -0
  377. data/spec/support/files_paths_helper.rb +13 -0
  378. data/spec/support/fixtures/results/mapping_template.yaml +15 -0
  379. data/spec/support/sandbox.rb +136 -0
  380. metadata +804 -85
  381. data/README.markdown +0 -274
  382. data/README.rdoc +0 -19
  383. data/VERSION +0 -1
  384. data/datashift.gemspec +0 -48
  385. data/lib/applications/jruby/old_pre_proxy_jexcel_file.rb +0 -437
  386. data/lib/datashift/data_transforms.rb +0 -83
  387. data/lib/datashift/mapping_file_definitions.rb +0 -88
  388. data/lib/datashift/mapping_service.rb +0 -91
  389. data/lib/datashift/method_detail.rb +0 -165
  390. data/lib/datashift/method_details_manager.rb +0 -95
  391. data/lib/datashift/method_dictionary.rb +0 -281
  392. data/lib/datashift/method_mapper.rb +0 -174
  393. data/lib/datashift/thor_base.rb +0 -38
  394. data/lib/generators/mapping_generator.rb +0 -112
  395. data/lib/helpers/core_ext/csv_file.rb +0 -33
  396. data/lib/helpers/core_ext/to_b.rb +0 -24
  397. data/lib/loaders/reporter.rb +0 -58
  398. data/lib/thor/export.thor +0 -175
  399. data/lib/thor/generate.thor +0 -191
  400. data/lib/thor/import.thor +0 -110
  401. data/lib/thor/mapping.thor +0 -65
  402. data/lib/thor/tools.thor +0 -84
  403. data/spec/Gemfile +0 -31
  404. data/spec/Gemfile.lock +0 -134
  405. data/spec/csv_exporter_spec.rb +0 -144
  406. data/spec/csv_generator_spec.rb +0 -159
  407. data/spec/csv_loader_spec.rb +0 -212
  408. data/spec/datashift_spec.rb +0 -55
  409. data/spec/excel_exporter_spec.rb +0 -199
  410. data/spec/excel_generator_spec.rb +0 -203
  411. data/spec/excel_loader_spec.rb +0 -237
  412. data/spec/excel_spec.rb +0 -203
  413. data/spec/loader_base_spec.rb +0 -166
  414. data/spec/mapping_spec.rb +0 -117
  415. data/spec/method_dictionary_spec.rb +0 -300
  416. data/spec/method_mapper_spec.rb +0 -100
  417. data/spec/model_mapper_spec.rb +0 -41
  418. data/spec/paperclip_loader_spec.rb +0 -92
  419. data/spec/populator_spec.rb +0 -128
  420. data/spec/thor_spec.rb +0 -90
  421. data/tasks/file_tasks.rake +0 -37
  422. data/tasks/word_to_seedfu.rake +0 -167
@@ -1,274 +0,0 @@
1
- ## DataShift
2
-
3
- - [Installation](#Installation)
4
- - [Features](#features)
5
- - [Testing](#testing)
6
- - [License](#license)
7
-
8
- Shift data between Excel/CSV files and Rails or Ruby applications
9
-
10
- Comprehensive Wiki here : **https://github.com/autotelik/datashift/wiki**
11
-
12
- ### <a name="Installation">Installation</a>
13
-
14
- Add gem 'datashift' to your Gemfile/bundle or use ```gem install```
15
-
16
- ```ruby
17
- gem 'datashift'
18
- ```
19
-
20
- To use :
21
-
22
- require 'datashift'
23
-
24
- To use the Thor command line applications, pull in the tasks.
25
-
26
- Generally the easiest way is to, create a high level .thor file in your Rails root directory
27
-
28
- e.g mysite.thor
29
-
30
- Edit the file and add the following to pull in the thor commands :
31
-
32
- ```ruby
33
- require 'thor'
34
- require 'datashift'
35
-
36
- DataShift::load_commands
37
- ```
38
-
39
- To keep the availability to only development mode use
40
-
41
- ```ruby DataShift::load_commands if(Rails.env.development?) ```
42
-
43
- To check the available tasks run
44
-
45
- bundle exec thor list datashift
46
-
47
- To get usage information use thor help <command>, for example
48
-
49
- bundle exec thor help datashift:generate:excel
50
-
51
- To use Excel OLE and MS Excel are NOT required.
52
-
53
- Specific tools for Spree E-Commerce now separate gem [datashift_spree](https://github.com/autotelik/datashift_spree "Datashift Spree")
54
-
55
-
56
- #### <a name="Features">Features</a>
57
-
58
- Import and Export ActiveRecord models direct to CSV or Excel/OpenOffice (.xls)
59
-
60
- You can select which associations to include and for import, set configurable defaults or over rides.
61
-
62
- Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby
63
-
64
- Generate a sample template with headers only.
65
-
66
- Export template and populate with model data
67
-
68
- Bulk import tools for Paperclip attachments.
69
-
70
- Easily extendable Loader functionality to deal with non trivial import cases, such
71
- as complex association lookups.
72
-
73
- High level rake and thor command line tasks for import/export provided.
74
-
75
-
76
- Loaders can be configured via YAML with over ride values, default values and mandatory column settings.
77
-
78
- Many example Spreadsheets/CSV files in spec/fixtures, fully documented with comments for each column.
79
-
80
- Features a common Excel interface over both our own wrapper around Apache POI (JRuby) and spreadsheet gem (all main Rubies)
81
-
82
- This means you can switch seamlessly between the two libraries, and if required drop down to make use of advanced
83
- features in the brilliant Apache POI libraries for anyone using JRuby.
84
-
85
- Guards are provided, and used internally, for mixed Ruby setups. Can be used like :
86
-
87
- if(DataShift::Guards::jruby? )
88
- ..do something with Apache
89
- else
90
- ..do something with speadsheet
91
- end
92
-
93
- #### Active Record - Import/Export
94
-
95
- Provides high level tasks for importing data via ActiveRecord models into a DB,
96
- from various sources, currently csv or .xls files (Excel/Open Office)
97
-
98
- Please use thor list and thor help <xxx> to get latest command lines
99
-
100
- bundle exec thor datashift:import:csv model=BlogPost input=BlogPostImport.csv verbose=true
101
-
102
-
103
- Provides high level tasks for exporting data to various sources, currently .xls files (Excel/Open Office)
104
-
105
- bundle exec thor datashift:export:excel model=BlogPost result=BlogExport.xls
106
-
107
-
108
- Import based on column headings with *Semi-Smart Name Lookup*
109
-
110
- On import, first a dictionary of all possible attributes and associations is created for the AR class.
111
-
112
- This enables lookup, of a user supplied name (column heading), managing white space, pluralisation etc .
113
-
114
- Example usage, load from a file or spreadsheet where the column names are only
115
- an approximation of the actual associations, so given 'Product Properties' heading,
116
- finds real association 'product_properties' to send or call on the AR object
117
-
118
-
119
- Can import/export 'belongs_to, 'has_many' and 'has_one' associations, including assignment of multiple objects
120
- via either multiple columns, or via a DSL for creating multiple entries in a single (column).
121
-
122
- The DSL can also be used to define which fields to lookup associations, and assign values to other fields.
123
-
124
- See Wiki for more details on DSL syntax.
125
-
126
- Supports inclusion of delegated attributes and normal instance methods as column headings.
127
-
128
- The library can be easily extended with Loaders to deal with non trivial cases,
129
- for example when multiple lookups required to find right association.
130
-
131
- Spree loaders are an example, these illustrate over riding processing for specific columns with
132
- complicated lookup requirements. Spree is the prime Open Source e-commerce project for Rails,
133
- and the specific loaders and tasks support loading Spree Products, and associated data such as Variants,
134
- OptionTypes, Properties and Images.
135
-
136
- #### Template Generation and Export
137
-
138
- Template generation tasks can be used to export a model's definition as column headings to CSV or .xls.
139
- These can be provided to developers or business users, as a template for data collection and then loading.
140
-
141
- Export tasks can be used to export of a model's definition and any existing data stored in the database.
142
-
143
- This data can be exported directly to CSV or Excel/OpenOffice spreadsheets.
144
-
145
-
146
- #### Example Spreadsheets
147
-
148
- A number of example Spreadsheets with headers and comments, can be found in the spec/fixtures directory.
149
-
150
- Extensive Spree samples - including .xls and csv versions for simple Products or complex Products with multiple
151
- taxons, variants properties etc - can be found in the spec/fixtures/spree subdirectory.
152
-
153
- Column headings contain comments with full descriptions and instructions on syntax.
154
-
155
-
156
- #### Excel
157
-
158
-
159
- MS Excel itself does not need to be installed.
160
-
161
- Our proxy for Excel allows seamless switching between 'spreadsheet' gem and datashift's own JRuby wrapper over Apache POI.
162
-
163
- When using JRuby, Apache POI may offer advanced facilities not found in standard Ruby spreadsheet gem
164
-
165
- The required POI jars are already included.
166
-
167
- Excel/OpenOffice spreadsheets are heavily used in many sectors, so direct support makes it
168
- easier and quicker to migrate your client's data into a Rails/ActiveRecord project,
169
- without converting first to CSV or YAML.
170
-
171
-
172
- #### Associations
173
-
174
- To perform a lookup for an associated model, the primary column(s) must be supplied, along with required select values for those columns.
175
-
176
- A single association column can contain multiple name/value sets, in string form :
177
-
178
- column:lookup_key_1, lookup_key_2,...
179
-
180
- So if our Project model has many Categories, we can supply a Category list, which is keyed on the column Category.reference with :
181
-
182
- |Categories|
183
-
184
- reference:category_001,category_002
185
-
186
- During loading, a call to find_all_by_reference will be made, picking up the 2 categories with matching references,
187
- and our Project model will contain those two i.e project.categories = [category_002,category_003]
188
-
189
-
190
- ### TODO
191
-
192
- - Smart sorting of column processing order ....
193
-
194
- - Does not currently ensure mandatory columns (for valid?) processed first.
195
-
196
- - Look at implementing import/export API using something like https://github.com/ianwhite/orm_adapter
197
- rather than active record, so we can support additional ORMs
198
-
199
-
200
- ### <a name="Testing">Testing</a>
201
- Specs have own Gemfile, so you can specify versions of active record that you want specs to run against :
202
-
203
- Edit
204
- ```ruby spec/Gemfile. ```
205
-
206
- Then run :
207
-
208
- ```ruby
209
- cd spec
210
- bundle install
211
- ```
212
-
213
- #### Changing Versions
214
-
215
- A sandbox will be generated in spec/sandbox if no such directory exists.
216
-
217
- **N.B Manual Step**
218
- When changing versions you probably need to **delete this whole directory** spec/sandbox. Next time you run spree specs it will be auto generated using latest Rails versions
219
-
220
- The database are created in sqlite3 and are stored in spec/fixtures. When switching versions, of say Spree,
221
- you will probably want to and to clear out old versions and retrigger the migrations
222
-
223
- rm spec/fixtures/*.sqlite
224
-
225
- You will probably also want to remove lock file :
226
-
227
- rm spec/Gemfile.lock
228
-
229
- First time the sandbox is regenerated, alot of tests may fail,perhaps not everything loads correctly during regeneration process.
230
-
231
- Invariably the next run, the specs pass, so a fix is low priority.
232
-
233
- #### Run the Tests
234
-
235
- ** N.B You should run the specs from within the specs directory. **
236
- ```ruby
237
- bundle exec rspec -c .
238
- ```
239
-
240
- A datashift **log **will be written within **spec/logs**, which hooks into the standard active record logger
241
-
242
- /log/datashift.log
243
- spec/logs/datashift_spec.log
244
-
245
-
246
-
247
-
248
- ## License
249
-
250
- Copyright:: (c) Autotelik Media Ltd 2015
251
-
252
- Author :: Tom Statter
253
-
254
- Date :: Dec 2015
255
-
256
- The MIT License
257
-
258
- Permission is hereby granted, free of charge, to any person obtaining a copy
259
- of this software and associated documentation files (the "Software"), to deal
260
- in the Software without restriction, including without limitation the rights
261
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
262
- copies of the Software, and to permit persons to whom the Software is
263
- furnished to do so, subject to the following conditions:
264
-
265
- The above copyright notice and this permission notice shall be included in
266
- all copies or substantial portions of the Software.
267
-
268
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
269
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
270
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
271
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
272
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
273
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
274
- THE SOFTWARE.
@@ -1,19 +0,0 @@
1
- = DataShift
2
-
3
- Description goes here.
4
-
5
- == Contributing to DataShift
6
-
7
- * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
8
- * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
9
- * Fork the project.
10
- * Start a feature/bugfix branch.
11
- * Commit and push until you are happy with your contribution.
12
- * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
13
- * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
14
-
15
- == Copyright
16
-
17
- Copyright (c) 2012 Thomas Statter. See LICENSE.txt for
18
- further details.
19
-
data/VERSION DELETED
@@ -1 +0,0 @@
1
- 0.16.0
@@ -1,48 +0,0 @@
1
- require 'rake'
2
-
3
- #TODO version = File.read("VERSION").strip
4
-
5
- Gem::Specification.new do |s|
6
- s.name = "datashift"
7
- s.version = "0.16.0"
8
-
9
- s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
10
-
11
- s.authors = ["Thomas Statter"]
12
-
13
- s.description = "Comprehensive import/export tools between Excel/CSV & ActiveRecord Databases, Rails apps, and any Ruby project."
14
- s.email = "rubygems@autotelik.co.uk"
15
- s.extra_rdoc_files = [
16
- "LICENSE.txt",
17
- "README.markdown",
18
- "README.rdoc"
19
- ]
20
-
21
- s.test_files = FileList["{spec}/*"]
22
-
23
- s.files = FileList[
24
- "LICENSE.txt",
25
- "README.markdown",
26
- "README.rdoc",
27
- "Rakefile",
28
- "VERSION",
29
- "datashift.gemspec",
30
- "tasks/config/seed_fu_product_template.erb",
31
- "tasks/config/tidy_config.txt",
32
- "tasks/db_tasks.rake",
33
- "tasks/file_tasks.rake",
34
- "tasks/word_to_seedfu.rake",
35
- "{lib}/**/*"].exclude("rdoc").exclude("nbproject").exclude("fixtures").exclude(".log").exclude(".contrib").to_a
36
-
37
- s.homepage = "http://github.com/autotelik/datashift"
38
- s.licenses = ["MIT"]
39
- s.require_paths = ["lib"]
40
-
41
- s.summary = "Shift data betwen Excel/CSV and any Ruby app"
42
-
43
- s.add_dependency 'spreadsheet'
44
- s.add_dependency 'rubyzip'
45
-
46
-
47
- end
48
-
@@ -1,437 +0,0 @@
1
- # Copyright:: (c) Autotelik Media Ltd 2011
2
- # Author :: Tom Statter
3
- # Date :: Aug 2010
4
- # License:: MIT
5
- #
6
- # An Excel file helper. Create and populate XSL files
7
- #
8
- # The maximum number of columns and rows in an Excel file is fixed at 256 Columns and 65536 Rows
9
- #
10
- # POI jar location needs to be added to class path.
11
- #
12
- # TODO - Check out http://poi.apache.org/poi-ruby.html
13
- #
14
- if(DataShift::Guards::jruby?)
15
-
16
- require "poi-3.7-20101029.jar"
17
-
18
- class JExcelFile
19
-
20
- java_import org.apache.poi.poifs.filesystem.POIFSFileSystem
21
-
22
- include_class 'org.apache.poi.hssf.usermodel.HSSFCell'
23
- include_class 'org.apache.poi.hssf.usermodel.HSSFWorkbook'
24
- include_class 'org.apache.poi.hssf.usermodel.HSSFCellStyle'
25
- include_class 'org.apache.poi.hssf.usermodel.HSSFDataFormat'
26
- include_class 'org.apache.poi.hssf.usermodel.HSSFClientAnchor'
27
- include_class 'org.apache.poi.hssf.usermodel.HSSFRichTextString'
28
-
29
- include_class 'org.apache.poi.hssf.util.HSSFColor'
30
-
31
- include_class 'java.io.ByteArrayOutputStream'
32
- include_class 'java.util.Date'
33
- include_class 'java.io.FileInputStream'
34
- include_class 'java.io.FileOutputStream'
35
-
36
- attr_accessor :workbook, :row, :date_style
37
- attr_reader :sheet
38
-
39
- MAX_COLUMNS = 256.freeze unless defined?(MAX_COLUMNS)
40
-
41
- def self.date_format
42
- HSSFDataFormat.getBuiltinFormat("m/d/yy h:mm")
43
- end
44
-
45
- # NOTE: this is the POI 3.7 HSSF maximum rows
46
- def self.maxrows
47
- return 65535
48
- end
49
-
50
- # The HSSFWorkbook uses 0 based indexes,
51
-
52
- def initialize(filename = nil)
53
- @workbook = nil
54
- # The @patriarchs hash is a workaround because HSSFSheet.getDrawingPatriarch()
55
- # causes a lot of issues (if it doesn't throw an exception!)
56
- @patriarchs = Hash.new
57
-
58
- @date_style = nil
59
-
60
- open(filename) if(filename)
61
- end
62
-
63
- def open(filename)
64
- inp = FileInputStream.new(filename)
65
-
66
- @workbook = HSSFWorkbook.new(inp)
67
-
68
- @date_style = @workbook.createCellStyle
69
- @date_style.setDataFormat( JExcelFile::date_format )
70
-
71
- @current_sheet = 0
72
- sheet(@current_sheet)
73
- end
74
-
75
- # EXCEL ITEMS
76
-
77
- def create(sheet_name)
78
- @workbook = HSSFWorkbook.new() if @workbook.nil?
79
-
80
- acceptable_name = sheet_name.gsub(':', '').gsub(" ", '')
81
-
82
- # Double check sheet doesn't already exist
83
- if(@workbook.getSheetIndex(acceptable_name) < 0)
84
- sheet = @workbook.createSheet(acceptable_name.gsub(" ", ''))
85
-
86
- @patriarchs.store(acceptable_name, sheet.createDrawingPatriarch())
87
- end
88
- @current_sheet = @workbook.getSheetIndex(acceptable_name)
89
-
90
- @date_style = @workbook.createCellStyle
91
- @date_style.setDataFormat( JExcelFile::date_format )
92
-
93
- self.sheet()
94
- end
95
-
96
- alias_method(:create_sheet, :create)
97
-
98
- # Return the current or specified HSSFSheet
99
- def sheet(i = nil)
100
- @current_sheet = i if i
101
- @sheet = @workbook.getSheetAt(@current_sheet)
102
- end
103
-
104
- def activate_sheet(sheet)
105
- active_sheet = @current_sheet
106
- if(@workbook)
107
- i = sheet if sheet.kind_of?(Integer)
108
- i = @workbook.getSheetIndex(sheet) if sheet.kind_of?(String)
109
-
110
- if( i >= 0 )
111
- @workbook.setActiveSheet(i) unless @workbook.nil?
112
- active_sheet = @workbook.getSheetAt(i)
113
- active_sheet.setActive(true)
114
- end unless i.nil?
115
- end
116
- return active_sheet
117
- end
118
-
119
- def num_rows
120
- @sheet.getPhysicalNumberOfRows
121
- end
122
-
123
- # Process each row. (type is org.apache.poi.hssf.usermodel.HSSFRow)
124
-
125
- def each_row
126
- @sheet.rowIterator.each { |row| @row = row; yield row }
127
- end
128
-
129
- # Create new row, index with POI usage starts at 0
130
- def create_row(index)
131
- return if @sheet.nil?
132
- raise "BAD INDEX: Row indexing starts at 0" if(index < 0)
133
- @row = @sheet.createRow(index)
134
- @row
135
- end
136
-
137
- #############################
138
- # INSERTING DATA INTO EXCEL #
139
- #############################
140
-
141
- # Populate a single cell with data
142
- #
143
- def set_cell(row, column, datum)
144
- @row = @sheet.getRow(row) || create_row(row)
145
- @row.createCell(column, excel_cell_type(datum)).setCellValue(datum)
146
- end
147
-
148
- # Convert array into a header row
149
- def set_headers(headers, apply_style = nil)
150
- create_row(0)
151
- return if headers.empty?
152
-
153
- style = apply_style || header_style()
154
-
155
- headers.each_with_index do |datum, i|
156
- c = @row.createCell(i, excel_cell_type(datum))
157
- c.setCellValue(datum)
158
- c.setCellStyle(style)
159
- end
160
- end
161
-
162
- def header_style
163
- return @header_style if @header_style
164
- @header_style = @workbook.createCellStyle();
165
- @header_style.setBorderTop(6) # double lines border
166
- @header_style.setBorderBottom(1) # single line border
167
- @header_style.setFillBackgroundColor(HSSFColor::GREY_25_PERCENT.index)
168
-
169
- @header_style
170
- end
171
-
172
- # Populate a row of cells with data in an array
173
- # where the co-ordinates relate to row/column start position
174
- #
175
- def set_row( row, col, data, sheet_num = nil)
176
-
177
- sheet(sheet_num)
178
-
179
- create_row(row)
180
-
181
- column = col
182
- data.each do |datum|
183
- set_cell(row, column, datum)
184
- column += 1
185
- end
186
- end
187
-
188
- # Return the suitable type for a HSSFCell from a Ruby data type
189
-
190
- def excel_cell_type(data)
191
-
192
- if(data.kind_of?(Numeric))
193
- HSSFCell::CELL_TYPE_NUMERIC
194
- elsif(data.nil?)
195
- HSSFCell::CELL_TYPE_BLANK
196
- elsif(data.is_a?(TrueClass) || data.is_a?(FalseClass))
197
- HSSFCell::CELL_TYPE_BOOLEAN
198
- else
199
- HSSFCell::CELL_TYPE_STRING
200
- end
201
- # HSSFCell::CELL_TYPE_FORMULA
202
- end
203
-
204
-
205
- # Auto size either the given column index or all columns
206
- def autosize(column = nil)
207
- return if @sheet.nil?
208
- if (column.kind_of? Integer)
209
- @sheet.autoSizeColumn(column)
210
- else
211
- @sheet.getRow(0).cellIterator.each{|c| @sheet.autoSizeColumn(c.getColumnIndex)}
212
- end
213
- end
214
-
215
- # TODO - Move into an ActiveRecord helper module of it's own
216
- def ar_to_headers( records )
217
- return if( !records.first.is_a?(ActiveRecord::Base) || records.empty?)
218
-
219
- headers = records.first.class.columns.collect( &:name )
220
- set_headers( headers )
221
- end
222
-
223
-
224
- # Pass a set of AR records
225
- def ar_to_xls(records, options = {})
226
- return if( ! records.first.is_a?(ActiveRecord::Base) || records.empty?)
227
-
228
- row_index =
229
- if(options[:no_headers])
230
- 0
231
- else
232
- ar_to_headers( records )
233
- 1
234
- end
235
-
236
- records.each do |record|
237
- create_row(row_index)
238
-
239
- ar_to_xls_row(0, record)
240
-
241
- row_index += 1
242
- end
243
- end
244
-
245
- # Save data from an AR record to the current row, based on the record's columns [c1,c2,c3]
246
- # Returns the number of the final column written to
247
- def ar_to_xls_row(start_column, record)
248
- return unless( record.is_a?(ActiveRecord::Base))
249
-
250
- column = start_column
251
- record.class.columns.each do |connection_column|
252
- ar_to_xls_cell(column, record, connection_column)
253
- column += 1
254
- end
255
- column
256
- end
257
-
258
- def ar_to_xls_cell(column, record, connection_column)
259
- begin
260
- datum = record.send(connection_column.name)
261
-
262
- if(connection_column.sql_type =~ /date/)
263
- @row.createCell(column, HSSFCell::CELL_TYPE_STRING).setCellValue(datum.to_s)
264
-
265
- elsif(connection_column.type == :boolean || connection_column.sql_type =~ /tinyint/)
266
- @row.createCell(column, HSSFCell::CELL_TYPE_BOOLEAN).setCellValue(datum)
267
-
268
- elsif(connection_column.sql_type =~ /int/)
269
- @row.createCell(column, HSSFCell::CELL_TYPE_NUMERIC).setCellValue(datum.to_i)
270
- else
271
- @row.createCell(column, HSSFCell::CELL_TYPE_STRING).setCellValue( datum.to_s )
272
- end
273
-
274
- rescue => e
275
- puts "Failed to export #{datum} from #{connection_column.inspect} to column #{column}"
276
- puts e
277
- end
278
- end
279
-
280
- ##############################
281
- # RETRIEVING DATA FROM EXCEL #
282
- ##############################
283
-
284
- # Return the raw data of the requested cell by row/column
285
- def get_cell_value(row, column)
286
- raise TypeError, "Expect row argument of type HSSFRow" unless row.is_a?(Java::OrgApachePoiHssfUsermodel::HSSFRow)
287
- cell_value( row.getCell(column) )
288
- end
289
-
290
- # Return the raw data of an HSSFCell
291
- def cell_value(cell)
292
- return unless cell
293
- case (cell.getCellType())
294
- when HSSFCell::CELL_TYPE_FORMULA then return cell.getCellFormula()
295
- when HSSFCell::CELL_TYPE_NUMERIC then return cell.getNumericCellValue()
296
- when HSSFCell::CELL_TYPE_STRING then return cell.getStringCellValue()
297
- when HSSFCell::CELL_TYPE_BOOLEAN then return cell.getBooleanCellValue()
298
- when HSSFCell::CELL_TYPE_BLANK then return ""
299
- end
300
- end
301
-
302
- def save( filename = nil )
303
-
304
- filename.nil? ? file = @filepath : file = filename
305
- begin
306
- out = FileOutputStream.new(file)
307
- @workbook.write(out) unless @workbook.nil?
308
-
309
- out.close
310
- rescue => e
311
- puts e
312
- raise "Cannot write file - is file already open in Excel ?"
313
- end
314
- end
315
-
316
- def save_to_text( filename )
317
- File.open( filename, 'w') {|f| f.write(to_s) }
318
- end
319
-
320
-
321
- def add_comment( cell, text )
322
- raise "Please supply valid HSSFCell" unless cell.respond_to?('setCellComment')
323
- return if @sheet.nil?
324
-
325
- patriarch = @patriarchs[@sheet.getSheetName()]
326
-
327
- anchor = HSSFClientAnchor.new(100, 50, 100, 50, cell.getColumnIndex(), cell.getRowIndex(), cell.getColumnIndex()+3, cell.getRowIndex()+4)
328
- comment = patriarch.createCellComment(anchor)
329
-
330
- comment_text = HSSFRichTextString.new(text)
331
- comment.setString(comment_text)
332
- comment.setAuthor("Mapping")
333
-
334
- cell.setCellComment(comment)
335
- end
336
-
337
- # The internal representation of a Excel File
338
-
339
- # Get a percentage style
340
- def getPercentStyle()
341
- if (@percentCellStyle.nil? && @workbook)
342
- @percentCellStyle = @workbook.createCellStyle();
343
- @percentCellStyle.setDataFormat(HSSFDataFormat.getBuiltinFormat("0.00%"));
344
- end
345
- return @percentCellStyle
346
- end
347
-
348
-
349
- def to_s
350
- return "" unless @workbook
351
-
352
- outs = ByteArrayOutputStream.new
353
- @workbook.write(outs);
354
- outs.close();
355
- String.from_java_bytes(outs.toByteArray)
356
- end
357
-
358
- def createFreezePane(row=1, column=0)
359
- return if @sheet.nil?
360
- @sheet.createFreezePane(row, column)
361
- end
362
-
363
- # Use execute to run sql query provided
364
- # and write to a csv file (path required)
365
- # header row is optional but default is on
366
- # Auto mapping of specified columns is optional
367
- # @mappings is a hash{column => map} of columns to a map{old_value => new_value}
368
- def results_to_sheet( results, sheet, mappings=nil, header=true)
369
- numrows = results.length
370
- sheet_name = sheet
371
-
372
- if (numrows == 0)
373
- log :info, "WARNING - results are empty nothing written to sheet: #{sheet}"
374
- return
375
- end
376
-
377
- #Check if we need to split the results into seperate sheets
378
- if (numrows > @@maxrows )
379
- startrow = 0
380
- while (numrows > 0)
381
- # Split the results and write to a new sheet
382
- next_results = results.slice(startrow, @@maxrows > numrows ? numrows : @@maxrows)
383
- self.results_to_sheet(next_results, "#{sheet_name}", mappings, header) if next_results
384
-
385
- # Increase counters
386
- numrows -= next_results.length
387
- startrow += next_results.length
388
- sheet_name += 'I'
389
- end
390
- else
391
- # Create required sheet
392
- self.create(sheet)
393
-
394
- row_index = self.num_rows
395
- # write header line
396
- if (header && row_index==0 )
397
- header_row = @sheet.createRow(row_index)
398
- cell_index = 0
399
- results[0].keys.each{ |h|
400
- header_row.createCell(cell_index).setCellValue("#{h}")
401
- @sheet.setDefaultColumnStyle(cell_index, self.getPercentStyle) if "#{h}".include? '%'
402
- cell_index += 1
403
- }
404
- # Freeze the header row
405
- @sheet.createFreezePane( 0, 1, 0, 1 )
406
- row_index += 1
407
- end
408
-
409
- # write_results
410
- results.each{ |row|
411
- sheet_row = @sheet.createRow(row_index)
412
- cell_index = 0
413
- row.each{|k,v|
414
- celltype = v.kind_of?(Numeric) ? HSSFCell::CELL_TYPE_NUMERIC : HSSFCell::CELL_TYPE_STRING
415
- cell = sheet_row.createCell(cell_index, celltype)
416
-
417
- v.nil? ? value = "<NIL>" : value = v
418
-
419
- cell.setCellValue(value)
420
-
421
- cell_index +=1
422
- }
423
- #puts "#{sheet}: written row #{row_index}"
424
- row_index +=1
425
- }
426
- end
427
-
428
- end
429
-
430
- end # END JExcelFile
431
- else
432
- class JExcelFile
433
- def initialize
434
- raise DataShift::BadRuby, "Please install and use JRuby for working with .xls files"
435
- end
436
- end
437
- end