datashift 0.16.0 → 0.40.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/{LICENSE.txt → LICENSE} +0 -0
- data/Rakefile +1 -20
- data/datashift.thor +125 -0
- data/lib/applications/apache_poi_extensions.rb +21 -52
- data/lib/applications/excel.rb +64 -57
- data/lib/applications/hssf_row_extensions.rb +66 -0
- data/lib/applications/jexcel_file.rb +99 -95
- data/lib/applications/jexcel_file_extensions.rb +76 -83
- data/lib/applications/jruby/word.rb +36 -36
- data/lib/applications/ruby_poi_translations.rb +34 -32
- data/lib/applications/spreadsheet_extensions.rb +21 -19
- data/lib/datashift.rb +49 -59
- data/lib/datashift/binder.rb +217 -0
- data/lib/datashift/column_packer.rb +21 -72
- data/lib/datashift/configuration.rb +317 -0
- data/lib/datashift/context_factory.rb +88 -0
- data/lib/datashift/core_ext/array.rb +15 -0
- data/lib/datashift/core_ext/csv_ext.rb +46 -0
- data/lib/datashift/core_ext/string.rb +49 -0
- data/lib/datashift/core_ext/to_b.rb +11 -0
- data/lib/datashift/delimiters.rb +55 -61
- data/lib/datashift/doc_context.rb +137 -0
- data/lib/datashift/excel_base.rb +93 -81
- data/lib/datashift/exceptions.rb +30 -28
- data/lib/datashift/file_definitions.rb +44 -39
- data/lib/datashift/guards.rb +5 -5
- data/lib/datashift/header.rb +25 -0
- data/lib/datashift/headers.rb +94 -0
- data/lib/datashift/inbound_data/column.rb +44 -0
- data/lib/datashift/inbound_data/lookup_support.rb +33 -0
- data/lib/datashift/inbound_data/method_binding.rb +139 -0
- data/lib/datashift/load_object.rb +37 -12
- data/lib/datashift/logging.rb +54 -27
- data/lib/datashift/mandatory.rb +39 -0
- data/lib/datashift/mapping/data_flow_schema.rb +198 -0
- data/lib/datashift/{model_mapper.rb → mapping/mapper_utils.rb} +30 -10
- data/lib/datashift/model_methods/catalogue.rb +183 -0
- data/lib/datashift/model_methods/collection.rb +140 -0
- data/lib/datashift/model_methods/model_method.rb +162 -0
- data/lib/datashift/model_methods/model_methods_manager.rb +76 -0
- data/lib/datashift/model_methods/operator.rb +62 -0
- data/lib/datashift/node_collection.rb +26 -0
- data/lib/datashift/node_context.rb +68 -0
- data/lib/datashift/populator.rb +308 -282
- data/lib/datashift/progress_monitor.rb +91 -0
- data/lib/datashift/querying.rb +110 -52
- data/lib/datashift/templates/import_export_config.erb +55 -0
- data/lib/datashift/transformation/factory.rb +219 -0
- data/lib/datashift/transformation/remove.rb +44 -0
- data/lib/datashift/version.rb +3 -0
- data/lib/exporters/configuration.rb +84 -0
- data/lib/exporters/csv_exporter.rb +54 -52
- data/lib/exporters/excel_exporter.rb +80 -61
- data/lib/exporters/exporter_base.rb +8 -8
- data/lib/generators/config_generator.rb +218 -0
- data/lib/generators/csv_generator.rb +13 -70
- data/lib/generators/excel_generator.rb +23 -111
- data/lib/generators/generator_base.rb +15 -70
- data/lib/loaders/configuration.rb +90 -0
- data/lib/loaders/csv_loader.rb +63 -101
- data/lib/loaders/excel_loader.rb +71 -156
- data/lib/loaders/failure_data.rb +40 -0
- data/lib/loaders/file_loader.rb +16 -0
- data/lib/loaders/loader_base.rb +82 -410
- data/lib/loaders/loader_factory.rb +42 -0
- data/lib/loaders/paperclip/attachment_loader.rb +157 -140
- data/lib/loaders/paperclip/datashift_paperclip.rb +18 -35
- data/lib/loaders/paperclip/image_loading.rb +40 -35
- data/lib/loaders/reporters/basic_stdout_reporter.rb +40 -0
- data/lib/loaders/reporters/reporter.rb +26 -0
- data/lib/tasks/config.thor +65 -0
- data/{tasks → lib/tasks}/config/seed_fu_product_template.erb +0 -0
- data/{tasks → lib/tasks}/config/tidy_config.txt +0 -0
- data/lib/tasks/export.thor +192 -0
- data/lib/tasks/generate.thor +190 -0
- data/lib/tasks/import.thor +142 -0
- data/lib/{thor → tasks}/paperclip.thor +69 -69
- data/{tasks → lib/tasks/to_convert_to_thor}/db_tasks.rake +20 -20
- data/lib/tasks/tools.thor +109 -0
- data/spec/MissingAttachmentRecords/DEMO_001_ror_bag.jpeg +0 -0
- data/spec/MissingAttachmentRecords/DEMO_002_Powerstation.jpeg +0 -0
- data/spec/MissingAttachmentRecords/DEMO_003_ror_mug.jpeg +0 -0
- data/spec/MissingAttachmentRecords/DEMO_004_ror_ringer.jpeg +0 -0
- data/spec/datashift/binder_spec.rb +266 -0
- data/spec/datashift/config_generator_spec.rb +186 -0
- data/spec/datashift/configuration.rb +66 -0
- data/spec/datashift/context_factory_spec.rb +63 -0
- data/spec/datashift/data_flow_schema_spec.rb +150 -0
- data/spec/datashift/datashift_spec.rb +52 -0
- data/spec/datashift/excel_base_spec.rb +57 -0
- data/spec/datashift/excel_spec.rb +188 -0
- data/spec/datashift/failure_data_spec.rb +27 -0
- data/spec/{file_definitions.rb → datashift/file_definitions.rb} +9 -10
- data/spec/datashift/headers_spec.rb +56 -0
- data/spec/datashift/inbound_data_spec.rb +47 -0
- data/spec/datashift/mapper_utils_spec.rb +38 -0
- data/spec/datashift/method_binding_spec.rb +60 -0
- data/spec/datashift/model_method_spec.rb +109 -0
- data/spec/datashift/model_methods_catalogue.rb +111 -0
- data/spec/datashift/model_methods_collection_spec.rb +138 -0
- data/spec/datashift/model_methods_manager_spec.rb +329 -0
- data/spec/datashift/populator_spec.rb +117 -0
- data/spec/datashift/thor_spec.rb +314 -0
- data/spec/datashift/transformation/factory_spec.rb +195 -0
- data/spec/datashift/transformation/transformer_remove_spec.rb +43 -0
- data/spec/dummy/Gemfile +53 -0
- data/spec/dummy/Gemfile.lock +197 -0
- data/spec/dummy/README.rdoc +28 -0
- data/spec/dummy/Rakefile +6 -0
- data/spec/dummy/app/assets/javascripts/application.js +16 -0
- data/spec/dummy/app/assets/javascripts/categories.js +2 -0
- data/spec/dummy/app/assets/javascripts/digitals.js +2 -0
- data/spec/dummy/app/assets/javascripts/empties.js +2 -0
- data/spec/dummy/app/assets/javascripts/loader_releases.js +2 -0
- data/spec/dummy/app/assets/javascripts/long_and_complex_table_linked_to_versions.js +2 -0
- data/spec/dummy/app/assets/javascripts/milestones.js +2 -0
- data/spec/dummy/app/assets/javascripts/owners.js +2 -0
- data/spec/dummy/app/assets/javascripts/projects.js +2 -0
- data/spec/dummy/app/assets/javascripts/users.js +2 -0
- data/spec/dummy/app/assets/javascripts/versions.js +2 -0
- data/spec/dummy/app/assets/stylesheets/application.css +15 -0
- data/spec/dummy/app/assets/stylesheets/categories.css +4 -0
- data/spec/dummy/app/assets/stylesheets/digitals.css +4 -0
- data/spec/dummy/app/assets/stylesheets/empties.css +4 -0
- data/spec/dummy/app/assets/stylesheets/loader_releases.css +4 -0
- data/spec/dummy/app/assets/stylesheets/long_and_complex_table_linked_to_versions.css +4 -0
- data/spec/dummy/app/assets/stylesheets/milestones.css +4 -0
- data/spec/dummy/app/assets/stylesheets/owners.css +4 -0
- data/spec/dummy/app/assets/stylesheets/projects.css +4 -0
- data/spec/dummy/app/assets/stylesheets/scaffold.css +56 -0
- data/spec/dummy/app/assets/stylesheets/users.css +4 -0
- data/spec/dummy/app/assets/stylesheets/versions.css +4 -0
- data/spec/dummy/app/controllers/application_controller.rb +5 -0
- data/spec/dummy/app/controllers/categories_controller.rb +58 -0
- data/spec/dummy/app/controllers/digitals_controller.rb +58 -0
- data/spec/dummy/app/controllers/empties_controller.rb +58 -0
- data/spec/dummy/app/controllers/loader_releases_controller.rb +58 -0
- data/spec/dummy/app/controllers/long_and_complex_table_linked_to_versions_controller.rb +58 -0
- data/spec/dummy/app/controllers/milestones_controller.rb +58 -0
- data/spec/dummy/app/controllers/owners_controller.rb +58 -0
- data/spec/dummy/app/controllers/projects_controller.rb +58 -0
- data/spec/dummy/app/controllers/users_controller.rb +58 -0
- data/spec/dummy/app/controllers/versions_controller.rb +58 -0
- data/spec/dummy/app/helpers/application_helper.rb +2 -0
- data/spec/dummy/app/helpers/categories_helper.rb +2 -0
- data/spec/dummy/app/helpers/digitals_helper.rb +2 -0
- data/spec/dummy/app/helpers/empties_helper.rb +2 -0
- data/spec/dummy/app/helpers/loader_releases_helper.rb +2 -0
- data/spec/dummy/app/helpers/long_and_complex_table_linked_to_versions_helper.rb +2 -0
- data/spec/dummy/app/helpers/milestones_helper.rb +2 -0
- data/spec/dummy/app/helpers/owners_helper.rb +2 -0
- data/spec/dummy/app/helpers/projects_helper.rb +2 -0
- data/spec/dummy/app/helpers/users_helper.rb +2 -0
- data/spec/dummy/app/helpers/versions_helper.rb +2 -0
- data/spec/dummy/app/models/category.rb +6 -0
- data/spec/dummy/app/models/digital.rb +22 -0
- data/spec/dummy/app/models/empty.rb +2 -0
- data/spec/dummy/app/models/loader_release.rb +10 -0
- data/spec/dummy/app/models/long_and_complex_table_linked_to_version.rb +6 -0
- data/spec/dummy/app/models/milestone.rb +15 -0
- data/spec/dummy/app/models/owner.rb +13 -0
- data/spec/dummy/app/models/project.rb +53 -0
- data/spec/dummy/app/models/user.rb +5 -0
- data/spec/dummy/app/models/version.rb +7 -0
- data/spec/dummy/app/views/categories/_form.html.erb +17 -0
- data/spec/dummy/app/views/categories/edit.html.erb +6 -0
- data/spec/dummy/app/views/categories/index.html.erb +25 -0
- data/spec/dummy/app/views/categories/new.html.erb +5 -0
- data/spec/dummy/app/views/categories/show.html.erb +4 -0
- data/spec/dummy/app/views/digitals/_form.html.erb +17 -0
- data/spec/dummy/app/views/digitals/edit.html.erb +6 -0
- data/spec/dummy/app/views/digitals/index.html.erb +25 -0
- data/spec/dummy/app/views/digitals/new.html.erb +5 -0
- data/spec/dummy/app/views/digitals/show.html.erb +4 -0
- data/spec/dummy/app/views/empties/_form.html.erb +17 -0
- data/spec/dummy/app/views/empties/edit.html.erb +6 -0
- data/spec/dummy/app/views/empties/index.html.erb +25 -0
- data/spec/dummy/app/views/empties/new.html.erb +5 -0
- data/spec/dummy/app/views/empties/show.html.erb +4 -0
- data/spec/dummy/app/views/layouts/application.html.erb +14 -0
- data/spec/dummy/app/views/loader_releases/_form.html.erb +17 -0
- data/spec/dummy/app/views/loader_releases/edit.html.erb +6 -0
- data/spec/dummy/app/views/loader_releases/index.html.erb +25 -0
- data/spec/dummy/app/views/loader_releases/new.html.erb +5 -0
- data/spec/dummy/app/views/loader_releases/show.html.erb +4 -0
- data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/_form.html.erb +17 -0
- data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/edit.html.erb +6 -0
- data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/index.html.erb +25 -0
- data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/new.html.erb +5 -0
- data/spec/dummy/app/views/long_and_complex_table_linked_to_versions/show.html.erb +4 -0
- data/spec/dummy/app/views/milestones/_form.html.erb +17 -0
- data/spec/dummy/app/views/milestones/edit.html.erb +6 -0
- data/spec/dummy/app/views/milestones/index.html.erb +25 -0
- data/spec/dummy/app/views/milestones/new.html.erb +5 -0
- data/spec/dummy/app/views/milestones/show.html.erb +4 -0
- data/spec/dummy/app/views/owners/_form.html.erb +17 -0
- data/spec/dummy/app/views/owners/edit.html.erb +6 -0
- data/spec/dummy/app/views/owners/index.html.erb +25 -0
- data/spec/dummy/app/views/owners/new.html.erb +5 -0
- data/spec/dummy/app/views/owners/show.html.erb +4 -0
- data/spec/dummy/app/views/projects/_form.html.erb +17 -0
- data/spec/dummy/app/views/projects/edit.html.erb +6 -0
- data/spec/dummy/app/views/projects/index.html.erb +25 -0
- data/spec/dummy/app/views/projects/new.html.erb +5 -0
- data/spec/dummy/app/views/projects/show.html.erb +4 -0
- data/spec/dummy/app/views/users/_form.html.erb +17 -0
- data/spec/dummy/app/views/users/edit.html.erb +6 -0
- data/spec/dummy/app/views/users/index.html.erb +25 -0
- data/spec/dummy/app/views/users/new.html.erb +5 -0
- data/spec/dummy/app/views/users/show.html.erb +4 -0
- data/spec/dummy/app/views/versions/_form.html.erb +17 -0
- data/spec/dummy/app/views/versions/edit.html.erb +6 -0
- data/spec/dummy/app/views/versions/index.html.erb +25 -0
- data/spec/dummy/app/views/versions/new.html.erb +5 -0
- data/spec/dummy/app/views/versions/show.html.erb +4 -0
- data/spec/dummy/bin/bundle +3 -0
- data/spec/dummy/bin/rails +9 -0
- data/spec/dummy/bin/rake +9 -0
- data/spec/dummy/bin/setup +29 -0
- data/spec/dummy/bin/spring +16 -0
- data/spec/dummy/config.ru +4 -0
- data/spec/dummy/config/application.rb +26 -0
- data/spec/dummy/config/boot.rb +3 -0
- data/spec/dummy/config/database.yml +25 -0
- data/spec/dummy/config/environment.rb +5 -0
- data/spec/dummy/config/environments/development.rb +41 -0
- data/spec/dummy/config/environments/production.rb +79 -0
- data/spec/dummy/config/environments/test.rb +42 -0
- data/spec/dummy/config/initializers/assets.rb +11 -0
- data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
- data/spec/dummy/config/initializers/cookies_serializer.rb +3 -0
- data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
- data/spec/dummy/config/initializers/inflections.rb +16 -0
- data/spec/dummy/config/initializers/mime_types.rb +4 -0
- data/spec/dummy/config/initializers/session_store.rb +3 -0
- data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
- data/spec/dummy/config/locales/en.yml +23 -0
- data/spec/dummy/config/routes.rb +76 -0
- data/spec/dummy/config/secrets.yml +22 -0
- data/spec/dummy/db/development.sqlite3 +0 -0
- data/spec/dummy/db/migrate/20110803201325_create_test_bed.rb +98 -0
- data/spec/dummy/db/migrate/20121009161700_add_digitals.rb +24 -0
- data/spec/dummy/db/migrate/20161005123106_create_digitals.rb +8 -0
- data/spec/dummy/db/migrate/20161005123106_create_long_and_complex_table_linked_to_versions.rb +8 -0
- data/spec/dummy/db/migrate/20161005123107_create_loader_releases.rb +8 -0
- data/spec/dummy/db/migrate/20161005123108_create_owners.rb +8 -0
- data/spec/dummy/db/migrate/20161005123109_create_empties.rb +8 -0
- data/spec/dummy/db/migrate/20161005123110_create_projects.rb +8 -0
- data/spec/dummy/db/migrate/20161005123111_create_users.rb +8 -0
- data/spec/dummy/db/migrate/20161005123111_create_versions.rb +8 -0
- data/spec/dummy/db/migrate/20161005123112_create_milestones.rb +8 -0
- data/spec/dummy/db/migrate/20161005123113_create_categories.rb +8 -0
- data/spec/dummy/db/schema.rb +93 -0
- data/spec/dummy/db/seeds.rb +9 -0
- data/spec/dummy/db/test.sqlite3 +0 -0
- data/spec/dummy/log/test.log +69 -0
- data/spec/dummy/public/404.html +67 -0
- data/spec/dummy/public/422.html +67 -0
- data/spec/dummy/public/500.html +66 -0
- data/spec/dummy/public/favicon.ico +0 -0
- data/spec/dummy/public/robots.txt +5 -0
- data/spec/dummy/sandbox_example.thor +4 -0
- data/spec/dummy/test/controllers/categories_controller_test.rb +49 -0
- data/spec/dummy/test/controllers/digitals_controller_test.rb +49 -0
- data/spec/dummy/test/controllers/empties_controller_test.rb +49 -0
- data/spec/dummy/test/controllers/loader_releases_controller_test.rb +49 -0
- data/spec/dummy/test/controllers/long_and_complex_table_linked_to_versions_controller_test.rb +49 -0
- data/spec/dummy/test/controllers/milestones_controller_test.rb +49 -0
- data/spec/dummy/test/controllers/owners_controller_test.rb +49 -0
- data/spec/dummy/test/controllers/projects_controller_test.rb +49 -0
- data/spec/dummy/test/controllers/users_controller_test.rb +49 -0
- data/spec/dummy/test/controllers/versions_controller_test.rb +49 -0
- data/spec/dummy/test/factories/categories.rb +5 -0
- data/spec/dummy/test/factories/digitals.rb +5 -0
- data/spec/dummy/test/factories/empties.rb +5 -0
- data/spec/dummy/test/factories/loader_releases.rb +5 -0
- data/spec/dummy/test/factories/long_and_complex_table_linked_to_versions.rb +5 -0
- data/spec/dummy/test/factories/milestones.rb +5 -0
- data/spec/dummy/test/factories/owners.rb +5 -0
- data/spec/dummy/test/factories/projects.rb +5 -0
- data/spec/dummy/test/factories/users.rb +5 -0
- data/spec/dummy/test/factories/versions.rb +5 -0
- data/spec/dummy/test/models/category_test.rb +7 -0
- data/spec/dummy/test/models/digital_test.rb +7 -0
- data/spec/dummy/test/models/empty_test.rb +7 -0
- data/spec/dummy/test/models/loader_release_test.rb +7 -0
- data/spec/dummy/test/models/long_and_complex_table_linked_to_version_test.rb +7 -0
- data/spec/dummy/test/models/milestone_test.rb +7 -0
- data/spec/dummy/test/models/owner_test.rb +7 -0
- data/spec/dummy/test/models/project_test.rb +7 -0
- data/spec/dummy/test/models/user_test.rb +7 -0
- data/spec/dummy/test/models/version_test.rb +7 -0
- data/spec/dummy/test/test_helper.rb +10 -0
- data/spec/exporters/csv_exporter_spec.rb +240 -0
- data/spec/exporters/csv_generator_spec.rb +139 -0
- data/spec/exporters/excel_exporter_spec.rb +193 -0
- data/spec/exporters/excel_generator_spec.rb +181 -0
- data/spec/exporters/generator_base_spec.rb +45 -0
- data/spec/factories/categories.rb +7 -0
- data/spec/factories/factories.rb +18 -0
- data/spec/factories/milestone.rb +16 -0
- data/spec/factories/projects.rb +41 -0
- data/spec/fixtures/BadAssociationName.xls +0 -0
- data/spec/fixtures/DemoNegativeTesting.xls +0 -0
- data/spec/fixtures/ProjectConfiguration.yml +18 -0
- data/spec/fixtures/ProjectsMultiCategories.xls +0 -0
- data/spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls +0 -0
- data/spec/fixtures/ProjectsSingleCategories.xls +0 -0
- data/spec/fixtures/ProjectsSingleCategories.xlsx +0 -0
- data/spec/fixtures/SimpleProjects.xls +0 -0
- data/spec/fixtures/config/database.yml +28 -0
- data/spec/fixtures/csv/BadAssociationName.csv +6 -0
- data/spec/fixtures/csv/DemoNegativeTesting.csv +6 -0
- data/spec/fixtures/csv/ProjectsMultiCategories.csv +5 -0
- data/spec/fixtures/csv/ProjectsMultiCategoriesHeaderLookup.csv +5 -0
- data/spec/fixtures/csv/ProjectsSingleCategories.csv +5 -0
- data/spec/fixtures/csv/SimpleProjects.csv +4 -0
- data/spec/fixtures/db/migrate/20110803201325_create_test_bed.rb +98 -0
- data/spec/fixtures/db/migrate/20121009161700_add_digitals.rb +24 -0
- data/spec/fixtures/db/seeds.rb +9 -0
- data/spec/fixtures/images/DEMO_001_ror_bag.jpeg +0 -0
- data/spec/fixtures/images/DEMO_002_Powerstation.jpeg +0 -0
- data/spec/fixtures/images/DEMO_003_ror_mug.jpeg +0 -0
- data/spec/fixtures/images/DEMO_004_ror_ringer.jpeg +0 -0
- data/spec/fixtures/load_datashift.thor +3 -0
- data/spec/fixtures/models/category.rb +6 -0
- data/spec/fixtures/models/digital.rb +22 -0
- data/spec/fixtures/models/empty.rb +2 -0
- data/spec/fixtures/models/loader_release.rb +10 -0
- data/spec/fixtures/models/long_and_complex_table_linked_to_version.rb +6 -0
- data/spec/fixtures/models/milestone.rb +15 -0
- data/spec/fixtures/models/owner.rb +13 -0
- data/spec/fixtures/models/project.rb +53 -0
- data/spec/fixtures/models/user.rb +5 -0
- data/spec/fixtures/models/version.rb +7 -0
- data/spec/fixtures/results/exp_project_assoc_headers.xls +0 -0
- data/spec/fixtures/results/exp_project_collection_spec.csv +2 -0
- data/spec/fixtures/results/exp_project_export.xls +0 -0
- data/spec/fixtures/results/exp_project_first_export.xls +0 -0
- data/spec/fixtures/results/exp_project_plus_assoc.xls +0 -0
- data/spec/fixtures/results/exp_project_plus_assoc_export_spec.csv +9 -0
- data/spec/fixtures/results/gen_project_plus_assoc_template.xls +0 -0
- data/spec/fixtures/results/gen_project_plus_some_assoc_template.xls +0 -0
- data/spec/fixtures/results/gen_project_template.xls +0 -0
- data/spec/fixtures/results/project_and_assoc_in_hash_export.xls +0 -0
- data/spec/fixtures/results/project_and_assoc_in_json_export.csv +9 -0
- data/spec/fixtures/results/project_and_assoc_in_json_export.xls +0 -0
- data/spec/fixtures/results/project_export_spec_with_custom_delim_,.csv +2 -0
- data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/243.csv +2 -0
- data/spec/fixtures/results/project_export_spec_with_custom_delim_/302/247.csv +2 -0
- data/spec/fixtures/results/project_plus_assoc_template.csv +1 -0
- data/spec/fixtures/results/project_plus_some_assoc_template.csv +1 -0
- data/spec/fixtures/results/project_remove_export_spec.csv +2 -0
- data/spec/fixtures/results/project_template.csv +1 -0
- data/spec/fixtures/results/project_with_methods_export_spec.csv +2 -0
- data/spec/fixtures/results/thor_spec_gen_project.csv +1 -0
- data/spec/fixtures/sandbox_example.thor +4 -0
- data/spec/fixtures/simple_export_spec.xls +0 -0
- data/spec/fixtures/simple_template_spec.xls +0 -0
- data/spec/fixtures/test_model_defs.rb +7 -0
- data/spec/loaders/csv_loader_spec.rb +206 -0
- data/spec/loaders/data_flow_excel_loader_spec.rb +290 -0
- data/spec/loaders/excel_loader_failures_spec.rb +67 -0
- data/spec/loaders/excel_loader_spec.rb +294 -0
- data/spec/loaders/loader_base_spec.rb +29 -0
- data/spec/loaders/paperclip_loader_spec.rb +106 -0
- data/spec/log/datashift.log +14930 -0
- data/spec/private/digitals/1/DEMO_003_ror_mug.jpeg +0 -0
- data/spec/private/digitals/2/DEMO_002_Powerstation.jpeg +0 -0
- data/spec/private/digitals/3/DEMO_004_ror_ringer.jpeg +0 -0
- data/spec/private/digitals/4/DEMO_001_ror_bag.jpeg +0 -0
- data/spec/spec_helper.rb +26 -230
- data/spec/support/clear_and_manage_contexts.rb +25 -0
- data/spec/support/database_cleaner.rb +32 -0
- data/spec/support/datashift_test_helpers.rb +153 -0
- data/spec/support/files_paths_helper.rb +13 -0
- data/spec/support/fixtures/results/mapping_template.yaml +15 -0
- data/spec/support/sandbox.rb +136 -0
- metadata +804 -85
- data/README.markdown +0 -274
- data/README.rdoc +0 -19
- data/VERSION +0 -1
- data/datashift.gemspec +0 -48
- data/lib/applications/jruby/old_pre_proxy_jexcel_file.rb +0 -437
- data/lib/datashift/data_transforms.rb +0 -83
- data/lib/datashift/mapping_file_definitions.rb +0 -88
- data/lib/datashift/mapping_service.rb +0 -91
- data/lib/datashift/method_detail.rb +0 -165
- data/lib/datashift/method_details_manager.rb +0 -95
- data/lib/datashift/method_dictionary.rb +0 -281
- data/lib/datashift/method_mapper.rb +0 -174
- data/lib/datashift/thor_base.rb +0 -38
- data/lib/generators/mapping_generator.rb +0 -112
- data/lib/helpers/core_ext/csv_file.rb +0 -33
- data/lib/helpers/core_ext/to_b.rb +0 -24
- data/lib/loaders/reporter.rb +0 -58
- data/lib/thor/export.thor +0 -175
- data/lib/thor/generate.thor +0 -191
- data/lib/thor/import.thor +0 -110
- data/lib/thor/mapping.thor +0 -65
- data/lib/thor/tools.thor +0 -84
- data/spec/Gemfile +0 -31
- data/spec/Gemfile.lock +0 -134
- data/spec/csv_exporter_spec.rb +0 -144
- data/spec/csv_generator_spec.rb +0 -159
- data/spec/csv_loader_spec.rb +0 -212
- data/spec/datashift_spec.rb +0 -55
- data/spec/excel_exporter_spec.rb +0 -199
- data/spec/excel_generator_spec.rb +0 -203
- data/spec/excel_loader_spec.rb +0 -237
- data/spec/excel_spec.rb +0 -203
- data/spec/loader_base_spec.rb +0 -166
- data/spec/mapping_spec.rb +0 -117
- data/spec/method_dictionary_spec.rb +0 -300
- data/spec/method_mapper_spec.rb +0 -100
- data/spec/model_mapper_spec.rb +0 -41
- data/spec/paperclip_loader_spec.rb +0 -92
- data/spec/populator_spec.rb +0 -128
- data/spec/thor_spec.rb +0 -90
- data/tasks/file_tasks.rake +0 -37
- data/tasks/word_to_seedfu.rake +0 -167
data/README.markdown
DELETED
@@ -1,274 +0,0 @@
|
|
1
|
-
## DataShift
|
2
|
-
|
3
|
-
- [Installation](#Installation)
|
4
|
-
- [Features](#features)
|
5
|
-
- [Testing](#testing)
|
6
|
-
- [License](#license)
|
7
|
-
|
8
|
-
Shift data between Excel/CSV files and Rails or Ruby applications
|
9
|
-
|
10
|
-
Comprehensive Wiki here : **https://github.com/autotelik/datashift/wiki**
|
11
|
-
|
12
|
-
### <a name="Installation">Installation</a>
|
13
|
-
|
14
|
-
Add gem 'datashift' to your Gemfile/bundle or use ```gem install```
|
15
|
-
|
16
|
-
```ruby
|
17
|
-
gem 'datashift'
|
18
|
-
```
|
19
|
-
|
20
|
-
To use :
|
21
|
-
|
22
|
-
require 'datashift'
|
23
|
-
|
24
|
-
To use the Thor command line applications, pull in the tasks.
|
25
|
-
|
26
|
-
Generally the easiest way is to, create a high level .thor file in your Rails root directory
|
27
|
-
|
28
|
-
e.g mysite.thor
|
29
|
-
|
30
|
-
Edit the file and add the following to pull in the thor commands :
|
31
|
-
|
32
|
-
```ruby
|
33
|
-
require 'thor'
|
34
|
-
require 'datashift'
|
35
|
-
|
36
|
-
DataShift::load_commands
|
37
|
-
```
|
38
|
-
|
39
|
-
To keep the availability to only development mode use
|
40
|
-
|
41
|
-
```ruby DataShift::load_commands if(Rails.env.development?) ```
|
42
|
-
|
43
|
-
To check the available tasks run
|
44
|
-
|
45
|
-
bundle exec thor list datashift
|
46
|
-
|
47
|
-
To get usage information use thor help <command>, for example
|
48
|
-
|
49
|
-
bundle exec thor help datashift:generate:excel
|
50
|
-
|
51
|
-
To use Excel OLE and MS Excel are NOT required.
|
52
|
-
|
53
|
-
Specific tools for Spree E-Commerce now separate gem [datashift_spree](https://github.com/autotelik/datashift_spree "Datashift Spree")
|
54
|
-
|
55
|
-
|
56
|
-
#### <a name="Features">Features</a>
|
57
|
-
|
58
|
-
Import and Export ActiveRecord models direct to CSV or Excel/OpenOffice (.xls)
|
59
|
-
|
60
|
-
You can select which associations to include and for import, set configurable defaults or over rides.
|
61
|
-
|
62
|
-
Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby
|
63
|
-
|
64
|
-
Generate a sample template with headers only.
|
65
|
-
|
66
|
-
Export template and populate with model data
|
67
|
-
|
68
|
-
Bulk import tools for Paperclip attachments.
|
69
|
-
|
70
|
-
Easily extendable Loader functionality to deal with non trivial import cases, such
|
71
|
-
as complex association lookups.
|
72
|
-
|
73
|
-
High level rake and thor command line tasks for import/export provided.
|
74
|
-
|
75
|
-
|
76
|
-
Loaders can be configured via YAML with over ride values, default values and mandatory column settings.
|
77
|
-
|
78
|
-
Many example Spreadsheets/CSV files in spec/fixtures, fully documented with comments for each column.
|
79
|
-
|
80
|
-
Features a common Excel interface over both our own wrapper around Apache POI (JRuby) and spreadsheet gem (all main Rubies)
|
81
|
-
|
82
|
-
This means you can switch seamlessly between the two libraries, and if required drop down to make use of advanced
|
83
|
-
features in the brilliant Apache POI libraries for anyone using JRuby.
|
84
|
-
|
85
|
-
Guards are provided, and used internally, for mixed Ruby setups. Can be used like :
|
86
|
-
|
87
|
-
if(DataShift::Guards::jruby? )
|
88
|
-
..do something with Apache
|
89
|
-
else
|
90
|
-
..do something with speadsheet
|
91
|
-
end
|
92
|
-
|
93
|
-
#### Active Record - Import/Export
|
94
|
-
|
95
|
-
Provides high level tasks for importing data via ActiveRecord models into a DB,
|
96
|
-
from various sources, currently csv or .xls files (Excel/Open Office)
|
97
|
-
|
98
|
-
Please use thor list and thor help <xxx> to get latest command lines
|
99
|
-
|
100
|
-
bundle exec thor datashift:import:csv model=BlogPost input=BlogPostImport.csv verbose=true
|
101
|
-
|
102
|
-
|
103
|
-
Provides high level tasks for exporting data to various sources, currently .xls files (Excel/Open Office)
|
104
|
-
|
105
|
-
bundle exec thor datashift:export:excel model=BlogPost result=BlogExport.xls
|
106
|
-
|
107
|
-
|
108
|
-
Import based on column headings with *Semi-Smart Name Lookup*
|
109
|
-
|
110
|
-
On import, first a dictionary of all possible attributes and associations is created for the AR class.
|
111
|
-
|
112
|
-
This enables lookup, of a user supplied name (column heading), managing white space, pluralisation etc .
|
113
|
-
|
114
|
-
Example usage, load from a file or spreadsheet where the column names are only
|
115
|
-
an approximation of the actual associations, so given 'Product Properties' heading,
|
116
|
-
finds real association 'product_properties' to send or call on the AR object
|
117
|
-
|
118
|
-
|
119
|
-
Can import/export 'belongs_to, 'has_many' and 'has_one' associations, including assignment of multiple objects
|
120
|
-
via either multiple columns, or via a DSL for creating multiple entries in a single (column).
|
121
|
-
|
122
|
-
The DSL can also be used to define which fields to lookup associations, and assign values to other fields.
|
123
|
-
|
124
|
-
See Wiki for more details on DSL syntax.
|
125
|
-
|
126
|
-
Supports inclusion of delegated attributes and normal instance methods as column headings.
|
127
|
-
|
128
|
-
The library can be easily extended with Loaders to deal with non trivial cases,
|
129
|
-
for example when multiple lookups required to find right association.
|
130
|
-
|
131
|
-
Spree loaders are an example, these illustrate over riding processing for specific columns with
|
132
|
-
complicated lookup requirements. Spree is the prime Open Source e-commerce project for Rails,
|
133
|
-
and the specific loaders and tasks support loading Spree Products, and associated data such as Variants,
|
134
|
-
OptionTypes, Properties and Images.
|
135
|
-
|
136
|
-
#### Template Generation and Export
|
137
|
-
|
138
|
-
Template generation tasks can be used to export a model's definition as column headings to CSV or .xls.
|
139
|
-
These can be provided to developers or business users, as a template for data collection and then loading.
|
140
|
-
|
141
|
-
Export tasks can be used to export of a model's definition and any existing data stored in the database.
|
142
|
-
|
143
|
-
This data can be exported directly to CSV or Excel/OpenOffice spreadsheets.
|
144
|
-
|
145
|
-
|
146
|
-
#### Example Spreadsheets
|
147
|
-
|
148
|
-
A number of example Spreadsheets with headers and comments, can be found in the spec/fixtures directory.
|
149
|
-
|
150
|
-
Extensive Spree samples - including .xls and csv versions for simple Products or complex Products with multiple
|
151
|
-
taxons, variants properties etc - can be found in the spec/fixtures/spree subdirectory.
|
152
|
-
|
153
|
-
Column headings contain comments with full descriptions and instructions on syntax.
|
154
|
-
|
155
|
-
|
156
|
-
#### Excel
|
157
|
-
|
158
|
-
|
159
|
-
MS Excel itself does not need to be installed.
|
160
|
-
|
161
|
-
Our proxy for Excel allows seamless switching between 'spreadsheet' gem and datashift's own JRuby wrapper over Apache POI.
|
162
|
-
|
163
|
-
When using JRuby, Apache POI may offer advanced facilities not found in standard Ruby spreadsheet gem
|
164
|
-
|
165
|
-
The required POI jars are already included.
|
166
|
-
|
167
|
-
Excel/OpenOffice spreadsheets are heavily used in many sectors, so direct support makes it
|
168
|
-
easier and quicker to migrate your client's data into a Rails/ActiveRecord project,
|
169
|
-
without converting first to CSV or YAML.
|
170
|
-
|
171
|
-
|
172
|
-
#### Associations
|
173
|
-
|
174
|
-
To perform a lookup for an associated model, the primary column(s) must be supplied, along with required select values for those columns.
|
175
|
-
|
176
|
-
A single association column can contain multiple name/value sets, in string form :
|
177
|
-
|
178
|
-
column:lookup_key_1, lookup_key_2,...
|
179
|
-
|
180
|
-
So if our Project model has many Categories, we can supply a Category list, which is keyed on the column Category.reference with :
|
181
|
-
|
182
|
-
|Categories|
|
183
|
-
|
184
|
-
reference:category_001,category_002
|
185
|
-
|
186
|
-
During loading, a call to find_all_by_reference will be made, picking up the 2 categories with matching references,
|
187
|
-
and our Project model will contain those two i.e project.categories = [category_002,category_003]
|
188
|
-
|
189
|
-
|
190
|
-
### TODO
|
191
|
-
|
192
|
-
- Smart sorting of column processing order ....
|
193
|
-
|
194
|
-
- Does not currently ensure mandatory columns (for valid?) processed first.
|
195
|
-
|
196
|
-
- Look at implementing import/export API using something like https://github.com/ianwhite/orm_adapter
|
197
|
-
rather than active record, so we can support additional ORMs
|
198
|
-
|
199
|
-
|
200
|
-
### <a name="Testing">Testing</a>
|
201
|
-
Specs have own Gemfile, so you can specify versions of active record that you want specs to run against :
|
202
|
-
|
203
|
-
Edit
|
204
|
-
```ruby spec/Gemfile. ```
|
205
|
-
|
206
|
-
Then run :
|
207
|
-
|
208
|
-
```ruby
|
209
|
-
cd spec
|
210
|
-
bundle install
|
211
|
-
```
|
212
|
-
|
213
|
-
#### Changing Versions
|
214
|
-
|
215
|
-
A sandbox will be generated in spec/sandbox if no such directory exists.
|
216
|
-
|
217
|
-
**N.B Manual Step**
|
218
|
-
When changing versions you probably need to **delete this whole directory** spec/sandbox. Next time you run spree specs it will be auto generated using latest Rails versions
|
219
|
-
|
220
|
-
The database are created in sqlite3 and are stored in spec/fixtures. When switching versions, of say Spree,
|
221
|
-
you will probably want to and to clear out old versions and retrigger the migrations
|
222
|
-
|
223
|
-
rm spec/fixtures/*.sqlite
|
224
|
-
|
225
|
-
You will probably also want to remove lock file :
|
226
|
-
|
227
|
-
rm spec/Gemfile.lock
|
228
|
-
|
229
|
-
First time the sandbox is regenerated, alot of tests may fail,perhaps not everything loads correctly during regeneration process.
|
230
|
-
|
231
|
-
Invariably the next run, the specs pass, so a fix is low priority.
|
232
|
-
|
233
|
-
#### Run the Tests
|
234
|
-
|
235
|
-
** N.B You should run the specs from within the specs directory. **
|
236
|
-
```ruby
|
237
|
-
bundle exec rspec -c .
|
238
|
-
```
|
239
|
-
|
240
|
-
A datashift **log **will be written within **spec/logs**, which hooks into the standard active record logger
|
241
|
-
|
242
|
-
/log/datashift.log
|
243
|
-
spec/logs/datashift_spec.log
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
## License
|
249
|
-
|
250
|
-
Copyright:: (c) Autotelik Media Ltd 2015
|
251
|
-
|
252
|
-
Author :: Tom Statter
|
253
|
-
|
254
|
-
Date :: Dec 2015
|
255
|
-
|
256
|
-
The MIT License
|
257
|
-
|
258
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
259
|
-
of this software and associated documentation files (the "Software"), to deal
|
260
|
-
in the Software without restriction, including without limitation the rights
|
261
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
262
|
-
copies of the Software, and to permit persons to whom the Software is
|
263
|
-
furnished to do so, subject to the following conditions:
|
264
|
-
|
265
|
-
The above copyright notice and this permission notice shall be included in
|
266
|
-
all copies or substantial portions of the Software.
|
267
|
-
|
268
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
269
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
270
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
271
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
272
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
273
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
274
|
-
THE SOFTWARE.
|
data/README.rdoc
DELETED
@@ -1,19 +0,0 @@
|
|
1
|
-
= DataShift
|
2
|
-
|
3
|
-
Description goes here.
|
4
|
-
|
5
|
-
== Contributing to DataShift
|
6
|
-
|
7
|
-
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
|
8
|
-
* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
|
9
|
-
* Fork the project.
|
10
|
-
* Start a feature/bugfix branch.
|
11
|
-
* Commit and push until you are happy with your contribution.
|
12
|
-
* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
|
13
|
-
* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
|
14
|
-
|
15
|
-
== Copyright
|
16
|
-
|
17
|
-
Copyright (c) 2012 Thomas Statter. See LICENSE.txt for
|
18
|
-
further details.
|
19
|
-
|
data/VERSION
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
0.16.0
|
data/datashift.gemspec
DELETED
@@ -1,48 +0,0 @@
|
|
1
|
-
require 'rake'
|
2
|
-
|
3
|
-
#TODO version = File.read("VERSION").strip
|
4
|
-
|
5
|
-
Gem::Specification.new do |s|
|
6
|
-
s.name = "datashift"
|
7
|
-
s.version = "0.16.0"
|
8
|
-
|
9
|
-
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
10
|
-
|
11
|
-
s.authors = ["Thomas Statter"]
|
12
|
-
|
13
|
-
s.description = "Comprehensive import/export tools between Excel/CSV & ActiveRecord Databases, Rails apps, and any Ruby project."
|
14
|
-
s.email = "rubygems@autotelik.co.uk"
|
15
|
-
s.extra_rdoc_files = [
|
16
|
-
"LICENSE.txt",
|
17
|
-
"README.markdown",
|
18
|
-
"README.rdoc"
|
19
|
-
]
|
20
|
-
|
21
|
-
s.test_files = FileList["{spec}/*"]
|
22
|
-
|
23
|
-
s.files = FileList[
|
24
|
-
"LICENSE.txt",
|
25
|
-
"README.markdown",
|
26
|
-
"README.rdoc",
|
27
|
-
"Rakefile",
|
28
|
-
"VERSION",
|
29
|
-
"datashift.gemspec",
|
30
|
-
"tasks/config/seed_fu_product_template.erb",
|
31
|
-
"tasks/config/tidy_config.txt",
|
32
|
-
"tasks/db_tasks.rake",
|
33
|
-
"tasks/file_tasks.rake",
|
34
|
-
"tasks/word_to_seedfu.rake",
|
35
|
-
"{lib}/**/*"].exclude("rdoc").exclude("nbproject").exclude("fixtures").exclude(".log").exclude(".contrib").to_a
|
36
|
-
|
37
|
-
s.homepage = "http://github.com/autotelik/datashift"
|
38
|
-
s.licenses = ["MIT"]
|
39
|
-
s.require_paths = ["lib"]
|
40
|
-
|
41
|
-
s.summary = "Shift data betwen Excel/CSV and any Ruby app"
|
42
|
-
|
43
|
-
s.add_dependency 'spreadsheet'
|
44
|
-
s.add_dependency 'rubyzip'
|
45
|
-
|
46
|
-
|
47
|
-
end
|
48
|
-
|
@@ -1,437 +0,0 @@
|
|
1
|
-
# Copyright:: (c) Autotelik Media Ltd 2011
|
2
|
-
# Author :: Tom Statter
|
3
|
-
# Date :: Aug 2010
|
4
|
-
# License:: MIT
|
5
|
-
#
|
6
|
-
# An Excel file helper. Create and populate XSL files
|
7
|
-
#
|
8
|
-
# The maximum number of columns and rows in an Excel file is fixed at 256 Columns and 65536 Rows
|
9
|
-
#
|
10
|
-
# POI jar location needs to be added to class path.
|
11
|
-
#
|
12
|
-
# TODO - Check out http://poi.apache.org/poi-ruby.html
|
13
|
-
#
|
14
|
-
if(DataShift::Guards::jruby?)
|
15
|
-
|
16
|
-
require "poi-3.7-20101029.jar"
|
17
|
-
|
18
|
-
class JExcelFile
|
19
|
-
|
20
|
-
java_import org.apache.poi.poifs.filesystem.POIFSFileSystem
|
21
|
-
|
22
|
-
include_class 'org.apache.poi.hssf.usermodel.HSSFCell'
|
23
|
-
include_class 'org.apache.poi.hssf.usermodel.HSSFWorkbook'
|
24
|
-
include_class 'org.apache.poi.hssf.usermodel.HSSFCellStyle'
|
25
|
-
include_class 'org.apache.poi.hssf.usermodel.HSSFDataFormat'
|
26
|
-
include_class 'org.apache.poi.hssf.usermodel.HSSFClientAnchor'
|
27
|
-
include_class 'org.apache.poi.hssf.usermodel.HSSFRichTextString'
|
28
|
-
|
29
|
-
include_class 'org.apache.poi.hssf.util.HSSFColor'
|
30
|
-
|
31
|
-
include_class 'java.io.ByteArrayOutputStream'
|
32
|
-
include_class 'java.util.Date'
|
33
|
-
include_class 'java.io.FileInputStream'
|
34
|
-
include_class 'java.io.FileOutputStream'
|
35
|
-
|
36
|
-
attr_accessor :workbook, :row, :date_style
|
37
|
-
attr_reader :sheet
|
38
|
-
|
39
|
-
MAX_COLUMNS = 256.freeze unless defined?(MAX_COLUMNS)
|
40
|
-
|
41
|
-
def self.date_format
|
42
|
-
HSSFDataFormat.getBuiltinFormat("m/d/yy h:mm")
|
43
|
-
end
|
44
|
-
|
45
|
-
# NOTE: this is the POI 3.7 HSSF maximum rows
|
46
|
-
def self.maxrows
|
47
|
-
return 65535
|
48
|
-
end
|
49
|
-
|
50
|
-
# The HSSFWorkbook uses 0 based indexes,
|
51
|
-
|
52
|
-
def initialize(filename = nil)
|
53
|
-
@workbook = nil
|
54
|
-
# The @patriarchs hash is a workaround because HSSFSheet.getDrawingPatriarch()
|
55
|
-
# causes a lot of issues (if it doesn't throw an exception!)
|
56
|
-
@patriarchs = Hash.new
|
57
|
-
|
58
|
-
@date_style = nil
|
59
|
-
|
60
|
-
open(filename) if(filename)
|
61
|
-
end
|
62
|
-
|
63
|
-
def open(filename)
|
64
|
-
inp = FileInputStream.new(filename)
|
65
|
-
|
66
|
-
@workbook = HSSFWorkbook.new(inp)
|
67
|
-
|
68
|
-
@date_style = @workbook.createCellStyle
|
69
|
-
@date_style.setDataFormat( JExcelFile::date_format )
|
70
|
-
|
71
|
-
@current_sheet = 0
|
72
|
-
sheet(@current_sheet)
|
73
|
-
end
|
74
|
-
|
75
|
-
# EXCEL ITEMS
|
76
|
-
|
77
|
-
def create(sheet_name)
|
78
|
-
@workbook = HSSFWorkbook.new() if @workbook.nil?
|
79
|
-
|
80
|
-
acceptable_name = sheet_name.gsub(':', '').gsub(" ", '')
|
81
|
-
|
82
|
-
# Double check sheet doesn't already exist
|
83
|
-
if(@workbook.getSheetIndex(acceptable_name) < 0)
|
84
|
-
sheet = @workbook.createSheet(acceptable_name.gsub(" ", ''))
|
85
|
-
|
86
|
-
@patriarchs.store(acceptable_name, sheet.createDrawingPatriarch())
|
87
|
-
end
|
88
|
-
@current_sheet = @workbook.getSheetIndex(acceptable_name)
|
89
|
-
|
90
|
-
@date_style = @workbook.createCellStyle
|
91
|
-
@date_style.setDataFormat( JExcelFile::date_format )
|
92
|
-
|
93
|
-
self.sheet()
|
94
|
-
end
|
95
|
-
|
96
|
-
alias_method(:create_sheet, :create)
|
97
|
-
|
98
|
-
# Return the current or specified HSSFSheet
|
99
|
-
def sheet(i = nil)
|
100
|
-
@current_sheet = i if i
|
101
|
-
@sheet = @workbook.getSheetAt(@current_sheet)
|
102
|
-
end
|
103
|
-
|
104
|
-
def activate_sheet(sheet)
|
105
|
-
active_sheet = @current_sheet
|
106
|
-
if(@workbook)
|
107
|
-
i = sheet if sheet.kind_of?(Integer)
|
108
|
-
i = @workbook.getSheetIndex(sheet) if sheet.kind_of?(String)
|
109
|
-
|
110
|
-
if( i >= 0 )
|
111
|
-
@workbook.setActiveSheet(i) unless @workbook.nil?
|
112
|
-
active_sheet = @workbook.getSheetAt(i)
|
113
|
-
active_sheet.setActive(true)
|
114
|
-
end unless i.nil?
|
115
|
-
end
|
116
|
-
return active_sheet
|
117
|
-
end
|
118
|
-
|
119
|
-
def num_rows
|
120
|
-
@sheet.getPhysicalNumberOfRows
|
121
|
-
end
|
122
|
-
|
123
|
-
# Process each row. (type is org.apache.poi.hssf.usermodel.HSSFRow)
|
124
|
-
|
125
|
-
def each_row
|
126
|
-
@sheet.rowIterator.each { |row| @row = row; yield row }
|
127
|
-
end
|
128
|
-
|
129
|
-
# Create new row, index with POI usage starts at 0
|
130
|
-
def create_row(index)
|
131
|
-
return if @sheet.nil?
|
132
|
-
raise "BAD INDEX: Row indexing starts at 0" if(index < 0)
|
133
|
-
@row = @sheet.createRow(index)
|
134
|
-
@row
|
135
|
-
end
|
136
|
-
|
137
|
-
#############################
|
138
|
-
# INSERTING DATA INTO EXCEL #
|
139
|
-
#############################
|
140
|
-
|
141
|
-
# Populate a single cell with data
|
142
|
-
#
|
143
|
-
def set_cell(row, column, datum)
|
144
|
-
@row = @sheet.getRow(row) || create_row(row)
|
145
|
-
@row.createCell(column, excel_cell_type(datum)).setCellValue(datum)
|
146
|
-
end
|
147
|
-
|
148
|
-
# Convert array into a header row
|
149
|
-
def set_headers(headers, apply_style = nil)
|
150
|
-
create_row(0)
|
151
|
-
return if headers.empty?
|
152
|
-
|
153
|
-
style = apply_style || header_style()
|
154
|
-
|
155
|
-
headers.each_with_index do |datum, i|
|
156
|
-
c = @row.createCell(i, excel_cell_type(datum))
|
157
|
-
c.setCellValue(datum)
|
158
|
-
c.setCellStyle(style)
|
159
|
-
end
|
160
|
-
end
|
161
|
-
|
162
|
-
def header_style
|
163
|
-
return @header_style if @header_style
|
164
|
-
@header_style = @workbook.createCellStyle();
|
165
|
-
@header_style.setBorderTop(6) # double lines border
|
166
|
-
@header_style.setBorderBottom(1) # single line border
|
167
|
-
@header_style.setFillBackgroundColor(HSSFColor::GREY_25_PERCENT.index)
|
168
|
-
|
169
|
-
@header_style
|
170
|
-
end
|
171
|
-
|
172
|
-
# Populate a row of cells with data in an array
|
173
|
-
# where the co-ordinates relate to row/column start position
|
174
|
-
#
|
175
|
-
def set_row( row, col, data, sheet_num = nil)
|
176
|
-
|
177
|
-
sheet(sheet_num)
|
178
|
-
|
179
|
-
create_row(row)
|
180
|
-
|
181
|
-
column = col
|
182
|
-
data.each do |datum|
|
183
|
-
set_cell(row, column, datum)
|
184
|
-
column += 1
|
185
|
-
end
|
186
|
-
end
|
187
|
-
|
188
|
-
# Return the suitable type for a HSSFCell from a Ruby data type
|
189
|
-
|
190
|
-
def excel_cell_type(data)
|
191
|
-
|
192
|
-
if(data.kind_of?(Numeric))
|
193
|
-
HSSFCell::CELL_TYPE_NUMERIC
|
194
|
-
elsif(data.nil?)
|
195
|
-
HSSFCell::CELL_TYPE_BLANK
|
196
|
-
elsif(data.is_a?(TrueClass) || data.is_a?(FalseClass))
|
197
|
-
HSSFCell::CELL_TYPE_BOOLEAN
|
198
|
-
else
|
199
|
-
HSSFCell::CELL_TYPE_STRING
|
200
|
-
end
|
201
|
-
# HSSFCell::CELL_TYPE_FORMULA
|
202
|
-
end
|
203
|
-
|
204
|
-
|
205
|
-
# Auto size either the given column index or all columns
|
206
|
-
def autosize(column = nil)
|
207
|
-
return if @sheet.nil?
|
208
|
-
if (column.kind_of? Integer)
|
209
|
-
@sheet.autoSizeColumn(column)
|
210
|
-
else
|
211
|
-
@sheet.getRow(0).cellIterator.each{|c| @sheet.autoSizeColumn(c.getColumnIndex)}
|
212
|
-
end
|
213
|
-
end
|
214
|
-
|
215
|
-
# TODO - Move into an ActiveRecord helper module of it's own
|
216
|
-
def ar_to_headers( records )
|
217
|
-
return if( !records.first.is_a?(ActiveRecord::Base) || records.empty?)
|
218
|
-
|
219
|
-
headers = records.first.class.columns.collect( &:name )
|
220
|
-
set_headers( headers )
|
221
|
-
end
|
222
|
-
|
223
|
-
|
224
|
-
# Pass a set of AR records
|
225
|
-
def ar_to_xls(records, options = {})
|
226
|
-
return if( ! records.first.is_a?(ActiveRecord::Base) || records.empty?)
|
227
|
-
|
228
|
-
row_index =
|
229
|
-
if(options[:no_headers])
|
230
|
-
0
|
231
|
-
else
|
232
|
-
ar_to_headers( records )
|
233
|
-
1
|
234
|
-
end
|
235
|
-
|
236
|
-
records.each do |record|
|
237
|
-
create_row(row_index)
|
238
|
-
|
239
|
-
ar_to_xls_row(0, record)
|
240
|
-
|
241
|
-
row_index += 1
|
242
|
-
end
|
243
|
-
end
|
244
|
-
|
245
|
-
# Save data from an AR record to the current row, based on the record's columns [c1,c2,c3]
|
246
|
-
# Returns the number of the final column written to
|
247
|
-
def ar_to_xls_row(start_column, record)
|
248
|
-
return unless( record.is_a?(ActiveRecord::Base))
|
249
|
-
|
250
|
-
column = start_column
|
251
|
-
record.class.columns.each do |connection_column|
|
252
|
-
ar_to_xls_cell(column, record, connection_column)
|
253
|
-
column += 1
|
254
|
-
end
|
255
|
-
column
|
256
|
-
end
|
257
|
-
|
258
|
-
def ar_to_xls_cell(column, record, connection_column)
|
259
|
-
begin
|
260
|
-
datum = record.send(connection_column.name)
|
261
|
-
|
262
|
-
if(connection_column.sql_type =~ /date/)
|
263
|
-
@row.createCell(column, HSSFCell::CELL_TYPE_STRING).setCellValue(datum.to_s)
|
264
|
-
|
265
|
-
elsif(connection_column.type == :boolean || connection_column.sql_type =~ /tinyint/)
|
266
|
-
@row.createCell(column, HSSFCell::CELL_TYPE_BOOLEAN).setCellValue(datum)
|
267
|
-
|
268
|
-
elsif(connection_column.sql_type =~ /int/)
|
269
|
-
@row.createCell(column, HSSFCell::CELL_TYPE_NUMERIC).setCellValue(datum.to_i)
|
270
|
-
else
|
271
|
-
@row.createCell(column, HSSFCell::CELL_TYPE_STRING).setCellValue( datum.to_s )
|
272
|
-
end
|
273
|
-
|
274
|
-
rescue => e
|
275
|
-
puts "Failed to export #{datum} from #{connection_column.inspect} to column #{column}"
|
276
|
-
puts e
|
277
|
-
end
|
278
|
-
end
|
279
|
-
|
280
|
-
##############################
|
281
|
-
# RETRIEVING DATA FROM EXCEL #
|
282
|
-
##############################
|
283
|
-
|
284
|
-
# Return the raw data of the requested cell by row/column
|
285
|
-
def get_cell_value(row, column)
|
286
|
-
raise TypeError, "Expect row argument of type HSSFRow" unless row.is_a?(Java::OrgApachePoiHssfUsermodel::HSSFRow)
|
287
|
-
cell_value( row.getCell(column) )
|
288
|
-
end
|
289
|
-
|
290
|
-
# Return the raw data of an HSSFCell
|
291
|
-
def cell_value(cell)
|
292
|
-
return unless cell
|
293
|
-
case (cell.getCellType())
|
294
|
-
when HSSFCell::CELL_TYPE_FORMULA then return cell.getCellFormula()
|
295
|
-
when HSSFCell::CELL_TYPE_NUMERIC then return cell.getNumericCellValue()
|
296
|
-
when HSSFCell::CELL_TYPE_STRING then return cell.getStringCellValue()
|
297
|
-
when HSSFCell::CELL_TYPE_BOOLEAN then return cell.getBooleanCellValue()
|
298
|
-
when HSSFCell::CELL_TYPE_BLANK then return ""
|
299
|
-
end
|
300
|
-
end
|
301
|
-
|
302
|
-
def save( filename = nil )
|
303
|
-
|
304
|
-
filename.nil? ? file = @filepath : file = filename
|
305
|
-
begin
|
306
|
-
out = FileOutputStream.new(file)
|
307
|
-
@workbook.write(out) unless @workbook.nil?
|
308
|
-
|
309
|
-
out.close
|
310
|
-
rescue => e
|
311
|
-
puts e
|
312
|
-
raise "Cannot write file - is file already open in Excel ?"
|
313
|
-
end
|
314
|
-
end
|
315
|
-
|
316
|
-
def save_to_text( filename )
|
317
|
-
File.open( filename, 'w') {|f| f.write(to_s) }
|
318
|
-
end
|
319
|
-
|
320
|
-
|
321
|
-
def add_comment( cell, text )
|
322
|
-
raise "Please supply valid HSSFCell" unless cell.respond_to?('setCellComment')
|
323
|
-
return if @sheet.nil?
|
324
|
-
|
325
|
-
patriarch = @patriarchs[@sheet.getSheetName()]
|
326
|
-
|
327
|
-
anchor = HSSFClientAnchor.new(100, 50, 100, 50, cell.getColumnIndex(), cell.getRowIndex(), cell.getColumnIndex()+3, cell.getRowIndex()+4)
|
328
|
-
comment = patriarch.createCellComment(anchor)
|
329
|
-
|
330
|
-
comment_text = HSSFRichTextString.new(text)
|
331
|
-
comment.setString(comment_text)
|
332
|
-
comment.setAuthor("Mapping")
|
333
|
-
|
334
|
-
cell.setCellComment(comment)
|
335
|
-
end
|
336
|
-
|
337
|
-
# The internal representation of a Excel File
|
338
|
-
|
339
|
-
# Get a percentage style
|
340
|
-
def getPercentStyle()
|
341
|
-
if (@percentCellStyle.nil? && @workbook)
|
342
|
-
@percentCellStyle = @workbook.createCellStyle();
|
343
|
-
@percentCellStyle.setDataFormat(HSSFDataFormat.getBuiltinFormat("0.00%"));
|
344
|
-
end
|
345
|
-
return @percentCellStyle
|
346
|
-
end
|
347
|
-
|
348
|
-
|
349
|
-
def to_s
|
350
|
-
return "" unless @workbook
|
351
|
-
|
352
|
-
outs = ByteArrayOutputStream.new
|
353
|
-
@workbook.write(outs);
|
354
|
-
outs.close();
|
355
|
-
String.from_java_bytes(outs.toByteArray)
|
356
|
-
end
|
357
|
-
|
358
|
-
def createFreezePane(row=1, column=0)
|
359
|
-
return if @sheet.nil?
|
360
|
-
@sheet.createFreezePane(row, column)
|
361
|
-
end
|
362
|
-
|
363
|
-
# Use execute to run sql query provided
|
364
|
-
# and write to a csv file (path required)
|
365
|
-
# header row is optional but default is on
|
366
|
-
# Auto mapping of specified columns is optional
|
367
|
-
# @mappings is a hash{column => map} of columns to a map{old_value => new_value}
|
368
|
-
def results_to_sheet( results, sheet, mappings=nil, header=true)
|
369
|
-
numrows = results.length
|
370
|
-
sheet_name = sheet
|
371
|
-
|
372
|
-
if (numrows == 0)
|
373
|
-
log :info, "WARNING - results are empty nothing written to sheet: #{sheet}"
|
374
|
-
return
|
375
|
-
end
|
376
|
-
|
377
|
-
#Check if we need to split the results into seperate sheets
|
378
|
-
if (numrows > @@maxrows )
|
379
|
-
startrow = 0
|
380
|
-
while (numrows > 0)
|
381
|
-
# Split the results and write to a new sheet
|
382
|
-
next_results = results.slice(startrow, @@maxrows > numrows ? numrows : @@maxrows)
|
383
|
-
self.results_to_sheet(next_results, "#{sheet_name}", mappings, header) if next_results
|
384
|
-
|
385
|
-
# Increase counters
|
386
|
-
numrows -= next_results.length
|
387
|
-
startrow += next_results.length
|
388
|
-
sheet_name += 'I'
|
389
|
-
end
|
390
|
-
else
|
391
|
-
# Create required sheet
|
392
|
-
self.create(sheet)
|
393
|
-
|
394
|
-
row_index = self.num_rows
|
395
|
-
# write header line
|
396
|
-
if (header && row_index==0 )
|
397
|
-
header_row = @sheet.createRow(row_index)
|
398
|
-
cell_index = 0
|
399
|
-
results[0].keys.each{ |h|
|
400
|
-
header_row.createCell(cell_index).setCellValue("#{h}")
|
401
|
-
@sheet.setDefaultColumnStyle(cell_index, self.getPercentStyle) if "#{h}".include? '%'
|
402
|
-
cell_index += 1
|
403
|
-
}
|
404
|
-
# Freeze the header row
|
405
|
-
@sheet.createFreezePane( 0, 1, 0, 1 )
|
406
|
-
row_index += 1
|
407
|
-
end
|
408
|
-
|
409
|
-
# write_results
|
410
|
-
results.each{ |row|
|
411
|
-
sheet_row = @sheet.createRow(row_index)
|
412
|
-
cell_index = 0
|
413
|
-
row.each{|k,v|
|
414
|
-
celltype = v.kind_of?(Numeric) ? HSSFCell::CELL_TYPE_NUMERIC : HSSFCell::CELL_TYPE_STRING
|
415
|
-
cell = sheet_row.createCell(cell_index, celltype)
|
416
|
-
|
417
|
-
v.nil? ? value = "<NIL>" : value = v
|
418
|
-
|
419
|
-
cell.setCellValue(value)
|
420
|
-
|
421
|
-
cell_index +=1
|
422
|
-
}
|
423
|
-
#puts "#{sheet}: written row #{row_index}"
|
424
|
-
row_index +=1
|
425
|
-
}
|
426
|
-
end
|
427
|
-
|
428
|
-
end
|
429
|
-
|
430
|
-
end # END JExcelFile
|
431
|
-
else
|
432
|
-
class JExcelFile
|
433
|
-
def initialize
|
434
|
-
raise DataShift::BadRuby, "Please install and use JRuby for working with .xls files"
|
435
|
-
end
|
436
|
-
end
|
437
|
-
end
|