datashift 0.40.3 → 0.40.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +7 -2
- data/datashift.thor +28 -23
- data/lib/datashift.rb +6 -6
- data/lib/datashift/binder.rb +30 -11
- data/lib/datashift/configuration.rb +10 -2
- data/lib/datashift/core_ext/array.rb +7 -7
- data/lib/datashift/delimiters.rb +1 -1
- data/lib/datashift/doc_context.rb +1 -1
- data/lib/datashift/excel_base.rb +2 -2
- data/lib/datashift/exporters/csv_exporter.rb +0 -1
- data/lib/datashift/exporters/excel_exporter.rb +3 -4
- data/lib/datashift/file_definitions.rb +1 -3
- data/lib/datashift/inbound_data/method_binding.rb +5 -5
- data/lib/datashift/loaders/csv_loader.rb +2 -3
- data/lib/datashift/loaders/excel_loader.rb +8 -4
- data/lib/datashift/loaders/failure_data.rb +1 -3
- data/lib/datashift/loaders/loader_base.rb +2 -8
- data/lib/datashift/loaders/loader_factory.rb +6 -0
- data/lib/datashift/loaders/paperclip/attachment_loader.rb +1 -1
- data/lib/datashift/loaders/paperclip/datashift_paperclip.rb +2 -2
- data/lib/datashift/loaders/paperclip/image_loading.rb +2 -2
- data/lib/datashift/mapping/data_flow_schema.rb +40 -14
- data/lib/datashift/mapping/mapper_utils.rb +3 -3
- data/lib/datashift/model_methods/catalogue.rb +14 -14
- data/lib/datashift/model_methods/model_method.rb +5 -6
- data/lib/datashift/model_methods/operator.rb +1 -1
- data/lib/datashift/node_context.rb +2 -3
- data/lib/datashift/populators/has_many.rb +2 -2
- data/lib/datashift/populators/insistent_assignment.rb +4 -4
- data/lib/datashift/populators/populator.rb +21 -16
- data/lib/datashift/populators/populator_factory.rb +2 -4
- data/lib/datashift/querying.rb +4 -5
- data/lib/datashift/transformation/factory.rb +3 -3
- data/lib/datashift/version.rb +1 -1
- data/lib/generators/datashift/install_generator.rb +3 -3
- data/lib/generators/templates/datashift.rb +0 -4
- data/lib/tasks/config.thor +7 -9
- data/lib/tasks/export.thor +44 -45
- data/lib/tasks/generate.thor +43 -37
- data/lib/tasks/import.thor +20 -14
- data/lib/tasks/paperclip.thor +46 -48
- data/lib/tasks/thor_behaviour.rb +1 -1
- data/lib/tasks/to_convert_to_thor/db_tasks.rake +1 -3
- data/lib/tasks/tools.thor +37 -38
- metadata +45 -45
data/lib/tasks/generate.thor
CHANGED
@@ -25,32 +25,37 @@ module Datashift
|
|
25
25
|
include DataShift::ThorBehavior
|
26
26
|
|
27
27
|
class_option :associations, aliases: '-a',
|
28
|
-
|
29
|
-
|
28
|
+
type: :boolean,
|
29
|
+
desc: 'Include associations. Can be further refined by :with & :exclude'
|
30
30
|
|
31
31
|
class_option :expand_associations, type: :boolean,
|
32
|
-
|
32
|
+
desc: 'Expand association data to multiple columns i.e 1 column per attribute'
|
33
33
|
|
34
|
+
desc 'excel', 'generate a template from an active record model (with optional associations)'
|
34
35
|
|
35
|
-
|
36
|
+
method_option :model, aliases: '-m', required: true, desc: 'The active record model to export'
|
37
|
+
method_option :result, aliases: '-r', required: true, desc: 'Create template of model in supplied file'
|
36
38
|
|
37
|
-
method_option :
|
38
|
-
method_option :
|
39
|
+
method_option :include_rails, desc: 'Include Rails auto generated columns like :id, created_at, updated_at'
|
40
|
+
method_option :force, aliases: '-f', type: :array, desc: "Inform datashift of columns that are still call-able even though they're non model methods"
|
39
41
|
|
40
|
-
def excel
|
42
|
+
def excel
|
41
43
|
start_connections
|
42
44
|
|
43
|
-
|
45
|
+
export_options = { remove_rails: !options[:include_rails] }
|
46
|
+
|
47
|
+
DataShift::Exporters::Configuration.from_hash(export_options)
|
48
|
+
|
49
|
+
generate(DataShift::ExcelGenerator.new)
|
44
50
|
|
45
51
|
puts "Datashift: Excel Template COMPLETED to #{options[:result]}"
|
46
52
|
end
|
47
53
|
|
54
|
+
desc 'csv', 'generate a template from an active record model (with optional associations)'
|
55
|
+
method_option :model, aliases: '-m', required: true, desc: 'The active record model to export'
|
56
|
+
method_option :result, aliases: '-r', required: true, desc: 'Create template of model in supplied file'
|
48
57
|
|
49
|
-
|
50
|
-
method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
|
51
|
-
method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
|
52
|
-
|
53
|
-
def csv()
|
58
|
+
def csv
|
54
59
|
start_connections
|
55
60
|
|
56
61
|
generate( DataShift::CsvGenerator.new)
|
@@ -58,27 +63,27 @@ module Datashift
|
|
58
63
|
puts "Datashift: CSV Template COMPLETED to #{options[:result]}"
|
59
64
|
end
|
60
65
|
|
61
|
-
desc
|
66
|
+
desc 'db', 'Generate a template for every Active Record model'
|
62
67
|
|
63
|
-
method_option :path, :
|
64
|
-
method_option :csv, :
|
68
|
+
method_option :path, aliases: '-p', required: true, desc: 'Path in which to create export files'
|
69
|
+
method_option :csv, aliases: '-c', desc: 'Generate CSV template instead - Excel is default.'
|
65
70
|
|
66
|
-
method_option :prefix_map, :
|
67
|
-
|
71
|
+
method_option :prefix_map, aliases: '-x', type: :hash, default: {},
|
72
|
+
desc: 'For namespaced tables/models specify the table prefix to module map e.g spree_:Spree'
|
68
73
|
|
69
|
-
method_option :modules, :
|
70
|
-
|
74
|
+
method_option :modules, aliases: '-m', type: :array, default: [],
|
75
|
+
desc: 'List of Modules to search for namespaced models'
|
71
76
|
|
72
|
-
def db
|
77
|
+
def db
|
73
78
|
|
74
79
|
start_connections
|
75
80
|
|
76
81
|
unless File.directory?(options[:path])
|
77
82
|
puts "WARNING : No such PATH found #{options[:path]} - trying mkdir"
|
78
|
-
FileUtils
|
83
|
+
FileUtils.mkdir_p(options[:path])
|
79
84
|
end
|
80
85
|
|
81
|
-
generator = options[:csv] ?
|
86
|
+
generator = options[:csv] ? DataShift::CsvGenerator.new : DataShift::ExcelGenerator.new
|
82
87
|
|
83
88
|
DataShift::Exporters::Configuration.from_hash(options)
|
84
89
|
|
@@ -93,13 +98,15 @@ module Datashift
|
|
93
98
|
break if(@klass)
|
94
99
|
end
|
95
100
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
101
|
+
unless @klass
|
102
|
+
options[:prefix_map].each do |p, m|
|
103
|
+
@klass = DataShift::MapperUtils.table_to_arclass(table.gsub(p, ''), m)
|
104
|
+
break if(@klass)
|
105
|
+
end
|
106
|
+
end
|
100
107
|
|
101
108
|
if(@klass.nil?)
|
102
|
-
puts
|
109
|
+
puts "ERROR: No Model found for Table [#{table}] - perhaps a prefix map required?"
|
103
110
|
next
|
104
111
|
end
|
105
112
|
|
@@ -108,15 +115,15 @@ module Datashift
|
|
108
115
|
puts "Datashift: Start export to #{result} for [#{table}]"
|
109
116
|
begin
|
110
117
|
if(options[:associations])
|
111
|
-
logger.info(
|
118
|
+
logger.info('Datashift: Generating with associations')
|
112
119
|
generator.generate_with_associations(result, @klass)
|
113
120
|
else
|
114
121
|
generator.generate(result, @klass)
|
115
122
|
end
|
116
|
-
rescue => e
|
123
|
+
rescue StandardError => e
|
117
124
|
puts e
|
118
125
|
puts e.backtrace
|
119
|
-
puts
|
126
|
+
puts 'Warning: Error during export, data may be incomplete'
|
120
127
|
end
|
121
128
|
end
|
122
129
|
|
@@ -132,27 +139,26 @@ module Datashift
|
|
132
139
|
|
133
140
|
logger.info "Datashift: Starting template generation for #{generater.class.name} to #{result}"
|
134
141
|
|
135
|
-
klass = DataShift::MapperUtils
|
142
|
+
klass = DataShift::MapperUtils.class_from_string(model) # Kernel.const_get(model)
|
136
143
|
|
137
144
|
raise "ERROR: No such Model [#{model}] found - check valid model supplied via -model <Class>" if(klass.nil?)
|
138
145
|
|
139
146
|
begin
|
140
|
-
|
141
147
|
if(options[:associations])
|
142
|
-
logger.info(
|
148
|
+
logger.info('Datashift: Generating template including associations')
|
143
149
|
generater.generate_with_associations(result, klass)
|
144
150
|
else
|
145
151
|
generater.generate(result, klass)
|
146
152
|
end
|
147
|
-
rescue => e
|
153
|
+
rescue StandardError => e
|
148
154
|
puts e
|
149
155
|
puts e.backtrace
|
150
|
-
puts
|
156
|
+
puts 'Warning: Error during export, data may be incomplete'
|
151
157
|
end
|
152
158
|
|
153
159
|
end
|
154
160
|
|
155
|
-
end
|
161
|
+
end # no_commands
|
156
162
|
|
157
163
|
end
|
158
164
|
|
data/lib/tasks/import.thor
CHANGED
@@ -21,7 +21,6 @@ module Datashift
|
|
21
21
|
when we try
|
22
22
|
Datashift::Import.new
|
23
23
|
|
24
|
-
|
25
24
|
class_option :model, aliases: '-m', required: true, desc: 'The related active record model'
|
26
25
|
|
27
26
|
class_option :input, aliases: '-i', required: true, desc: 'The input file'
|
@@ -33,7 +32,7 @@ module Datashift
|
|
33
32
|
class_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
|
34
33
|
=end
|
35
34
|
|
36
|
-
desc
|
35
|
+
desc 'load', 'Import data from file for specific active record model'
|
37
36
|
|
38
37
|
method_option :model, aliases: '-m', required: true, desc: 'The related active record model'
|
39
38
|
|
@@ -45,22 +44,23 @@ module Datashift
|
|
45
44
|
|
46
45
|
method_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
|
47
46
|
|
48
|
-
|
47
|
+
method_option :force, aliases: '-f', type: :array, desc: "Call-able columns even though datashift does not recognise them as model methods. Space separated e.g : -f sku price"
|
48
|
+
|
49
|
+
def load
|
49
50
|
start_connections
|
50
51
|
|
51
52
|
importer = if(options[:loader])
|
52
53
|
logger.info("Attempting to use supplied Loader : #{options[:loader]}")
|
53
|
-
DataShift::MapperUtils
|
54
|
+
DataShift::MapperUtils.class_from_string(options[:loader]).new
|
54
55
|
else
|
55
|
-
logger.info(
|
56
|
+
logger.info('No Loader specified - finding appropriate Loader for file type')
|
56
57
|
DataShift::Loader::Factory.get_loader(options[:input])
|
57
58
|
end
|
58
59
|
|
59
60
|
import(importer)
|
60
61
|
end
|
61
62
|
|
62
|
-
|
63
|
-
desc "excel", "Import .xls file for specifiec active record model"
|
63
|
+
desc 'excel', 'Import .xls file for specifiec active record model'
|
64
64
|
|
65
65
|
method_option :model, aliases: '-m', required: true, desc: 'The related active record model'
|
66
66
|
|
@@ -72,22 +72,24 @@ module Datashift
|
|
72
72
|
|
73
73
|
method_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
|
74
74
|
|
75
|
-
|
75
|
+
method_option :force, aliases: '-f', type: :array, desc: "Call-able columns even though datashift does not recognise them as model methods. Space separated e.g : -f sku price"
|
76
|
+
|
77
|
+
def excel
|
76
78
|
|
77
79
|
start_connections
|
78
80
|
|
79
81
|
importer = if(options[:loader])
|
80
82
|
logger.info("Attempting to use supplied Loader : #{options[:loader]}")
|
81
|
-
DataShift::MapperUtils
|
83
|
+
DataShift::MapperUtils.class_from_string(options[:loader]).new
|
82
84
|
else
|
83
|
-
logger.info(
|
85
|
+
logger.info('No Loader specified - using standard Excel Loader')
|
84
86
|
DataShift::ExcelLoader.new
|
85
87
|
end
|
86
88
|
|
87
89
|
import(importer)
|
88
90
|
end
|
89
91
|
|
90
|
-
desc
|
92
|
+
desc 'csv', 'Import CSV file for specified active record model'
|
91
93
|
|
92
94
|
method_option :model, aliases: '-m', required: true, desc: 'The related active record model'
|
93
95
|
|
@@ -99,15 +101,17 @@ module Datashift
|
|
99
101
|
|
100
102
|
method_option :config, aliases: '-c', desc: 'YAML config file with defaults, over-rides etc'
|
101
103
|
|
102
|
-
|
104
|
+
method_option :force, aliases: '-f', type: :array, desc: "Call-able columns even though datashift does not recognise them as model methods. Space separated e.g : -f sku price"
|
105
|
+
|
106
|
+
def csv
|
103
107
|
|
104
108
|
start_connections
|
105
109
|
|
106
110
|
importer = if(options[:loader])
|
107
111
|
logger.info("Attempting to use supplied Loader : #{options[:loader]}")
|
108
|
-
DataShift::MapperUtils
|
112
|
+
DataShift::MapperUtils.class_from_string(options[:loader]).new
|
109
113
|
else
|
110
|
-
logger.info(
|
114
|
+
logger.info('No Loader specified - using standard Csv Loader')
|
111
115
|
DataShift::CsvLoader.new
|
112
116
|
end
|
113
117
|
|
@@ -121,6 +125,8 @@ module Datashift
|
|
121
125
|
|
122
126
|
importer.configure_from( options[:config] ) if(options[:config])
|
123
127
|
|
128
|
+
DataShift::Configuration.call.force_inclusion_of_columns = options[:force]
|
129
|
+
|
124
130
|
importer.run(options[:input], options[:model])
|
125
131
|
end
|
126
132
|
|
data/lib/tasks/paperclip.thor
CHANGED
@@ -12,23 +12,23 @@ require_relative 'thor_behaviour'
|
|
12
12
|
|
13
13
|
# Note, not DataShift, case sensitive, create namespace for command line : datashift
|
14
14
|
module Datashift
|
15
|
-
|
15
|
+
|
16
16
|
class Paperclip < Thor
|
17
17
|
|
18
18
|
include DataShift::ThorBehavior
|
19
19
|
|
20
|
-
desc
|
20
|
+
desc 'attach', "Create paperclip attachments and attach to a Model from files in a directory.
|
21
21
|
This is specifically for the use case where the paperclip attachments are stored in a class, such as Image, Icon, Asset,
|
22
22
|
and this class has a relationship, such as belongs_to, with another class, such as Product, User, Document.
|
23
|
-
|
23
|
+
|
24
24
|
Each matching file is used to create an instance of the paperclip attachment, given by :attachment_klass.
|
25
|
-
|
25
|
+
|
26
26
|
The class with the relationship, can be specified via :attach_to_klass
|
27
27
|
|
28
|
-
Examples
|
28
|
+
Examples
|
29
29
|
Owner has_many pdfs and mp3 files as Digitals .... :attachment_klass = Digital and :attach_to_klass = Owner
|
30
30
|
User has a single Image used as an avatar ... attachment_klass = Image and :attach_to_klass = User
|
31
|
-
|
31
|
+
|
32
32
|
The file name is used to lookup the instance of :attach_to_klass to assign the new attachment to, via :attach_to_find_by_field
|
33
33
|
|
34
34
|
So say we have a file called smithj_avatar.gif, and we want to lookup Users by login
|
@@ -37,60 +37,58 @@ module Datashift
|
|
37
37
|
|
38
38
|
Once instance of :attach_to_klass found, the new attachment is assigned.
|
39
39
|
|
40
|
-
The attribute to assign new attachment to is gtiven by :attach_to_field
|
41
|
-
|
40
|
+
The attribute to assign new attachment to is gtiven by :attach_to_field
|
41
|
+
|
42
42
|
Examples
|
43
43
|
:attach_to_field => digitals : Owner.digitals = attachment(Digital)
|
44
44
|
:attach_to_field => avatar : User.avatar = attachment(Image)"
|
45
|
-
|
46
45
|
|
47
46
|
# :dummy => dummy run without actual saving to DB
|
48
|
-
method_option :input, :
|
49
|
-
|
50
|
-
method_option :glob, :
|
51
|
-
method_option :recursive, :
|
52
|
-
|
53
|
-
method_option :attachment_klass, :
|
54
|
-
|
55
|
-
method_option :attach_to_klass, :
|
56
|
-
method_option :attach_to_find_by_field, :
|
57
|
-
method_option :attach_to_field, :
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
method_option :
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
def attach()
|
47
|
+
method_option :input, aliases: '-i', required: true, desc: 'The input path containing images '
|
48
|
+
|
49
|
+
method_option :glob, aliases: '-g', desc: 'The glob to use to find files e.g. \'{*.jpg,*.gif,*.png}\' '
|
50
|
+
method_option :recursive, aliases: '-r', type: :boolean, desc: 'Scan sub directories of input for images'
|
51
|
+
|
52
|
+
method_option :attachment_klass, required: true, aliases: '-a', desc: 'Ruby Class name of the Attachment e.g Image, Icon'
|
53
|
+
|
54
|
+
method_option :attach_to_klass, required: true, aliases: '-k', desc: 'A class that has a relationship with the attachment (has_many, has_one, belongs_to)'
|
55
|
+
method_option :attach_to_find_by_field, required: true, aliases: '-l', desc: 'The field to use to find the :attach_to_klass record'
|
56
|
+
method_option :attach_to_field, required: true, aliases: '-f', desc: 'Attachment belongs to field e.g Product.image, Blog.digital'
|
57
|
+
|
58
|
+
# => :attach_to_find_by_field
|
59
|
+
# For the :attach_to_klass, this is the field used to search for the parent
|
60
|
+
# object to assign the new attachment to.
|
61
|
+
# Examples
|
62
|
+
# Owner has a unique 'name' field ... :attach_to_find_by_field = :name
|
63
|
+
# User has a unique 'login' field ... :attach_to_klass = :login
|
64
|
+
#
|
65
|
+
# => :attach_to_field
|
66
|
+
# Attribute/association to assign attachment to on :attach_to_klass.
|
67
|
+
# Examples
|
68
|
+
# :attach_to_field => digitals : Owner.digitals = attachment
|
69
|
+
# :attach_to_field => avatar : User.avatar = attachment
|
70
|
+
|
71
|
+
method_option :split_file_name_on, type: :string,
|
72
|
+
desc: 'delimiter to progressivley split file_name for lookup', default: ' '
|
73
|
+
|
74
|
+
method_option :case_sensitive, type: :boolean, desc: 'Use case sensitive where clause to find :attach_to_klass'
|
75
|
+
method_option :use_like, type: :boolean, desc: "Use :lookup_field LIKE 'string%' instead of :lookup_field = 'string' in where clauses to find :attach_to_klass"
|
76
|
+
|
77
|
+
method_option :skip_when_assoc, aliases: '-x', type: :boolean, desc: 'Do not process if :attach_to_klass already has an attachment'
|
78
|
+
|
79
|
+
method_option :verbose, aliases: '-v', type: :boolean, desc: 'Verbose logging'
|
80
|
+
|
81
|
+
def attach
|
84
82
|
|
85
83
|
@attachment_path = options[:input]
|
86
|
-
|
84
|
+
|
87
85
|
unless(File.exist?(@attachment_path))
|
88
86
|
puts "ERROR: Supplied Path [#{@attachment_path}] not accesible"
|
89
87
|
exit(-1)
|
90
88
|
end
|
91
89
|
|
92
90
|
start_connections
|
93
|
-
|
91
|
+
|
94
92
|
require 'paperclip/attachment_loader'
|
95
93
|
|
96
94
|
puts "Using Field #{options[:attach_to_find_by_field]} to lookup matching [#{options[:attach_to_klass]}]"
|
@@ -104,5 +102,5 @@ module Datashift
|
|
104
102
|
loader.run(@attachment_path, options[:attachment_klass])
|
105
103
|
end
|
106
104
|
end
|
107
|
-
|
105
|
+
|
108
106
|
end
|
data/lib/tasks/thor_behaviour.rb
CHANGED
@@ -13,7 +13,7 @@ module DataShift
|
|
13
13
|
if File.exist?(File.expand_path('config/environment.rb'))
|
14
14
|
begin
|
15
15
|
require File.expand_path('config/environment.rb')
|
16
|
-
rescue => e
|
16
|
+
rescue StandardError => e
|
17
17
|
logger.error("Failed to initialise ActiveRecord : #{e.message}")
|
18
18
|
raise ConnectionError, "Failed to initialise ActiveRecord : #{e.message}"
|
19
19
|
end
|
@@ -8,9 +8,7 @@ namespace :datashift do
|
|
8
8
|
task :purge, [:exclude_system_tables] => [:environment] do |_t, args|
|
9
9
|
require 'highline/import'
|
10
10
|
|
11
|
-
if Rails.env.production?
|
12
|
-
agree('WARNING: In Production database, REALLY PURGE ? [y]:')
|
13
|
-
end
|
11
|
+
agree('WARNING: In Production database, REALLY PURGE ? [y]:') if Rails.env.production?
|
14
12
|
|
15
13
|
config = ActiveRecord::Base.configurations[Rails.env || 'development']
|
16
14
|
case config['adapter']
|
data/lib/tasks/tools.thor
CHANGED
@@ -5,22 +5,22 @@
|
|
5
5
|
#
|
6
6
|
#
|
7
7
|
require 'datashift'
|
8
|
-
|
8
|
+
|
9
9
|
# Note, not DataShift, case sensitive, create namespace for command line : datashift
|
10
10
|
module Datashift
|
11
11
|
|
12
|
-
class Tools < Thor
|
13
|
-
|
12
|
+
class Tools < Thor
|
13
|
+
|
14
14
|
include DataShift::Logging
|
15
15
|
|
16
16
|
desc 'file_rename', 'Clone a folder of files, consistently renaming each in the process'
|
17
17
|
|
18
|
-
method_option :path, :
|
19
|
-
method_option :output, :
|
20
|
-
method_option :offset,
|
21
|
-
method_option :width,
|
22
|
-
method_option :prefix, required: false, desc:
|
23
|
-
method_option :commit, required: false, type: :boolean, desc:
|
18
|
+
method_option :path, aliases: '-p', required: true, desc: 'The path to the original files'
|
19
|
+
method_option :output, aliases: '-o', required: true, desc: 'The resulting zip file name'
|
20
|
+
method_option :offset, required: false, type: :numeric, desc: 'A numeric offset to add to the file name'
|
21
|
+
method_option :width, required: false, type: :numeric, desc: 'A numeric width tp pad the file name'
|
22
|
+
method_option :prefix, required: false, desc: 'A strign prefix to add to file name'
|
23
|
+
method_option :commit, required: false, type: :boolean, desc: 'Actually perform copy'
|
24
24
|
|
25
25
|
def file_rename
|
26
26
|
|
@@ -33,7 +33,7 @@ module Datashift
|
|
33
33
|
id = base_name.slice!(/\w+/)
|
34
34
|
|
35
35
|
id = id.to_i + options[:offset].to_i if options[:offset]
|
36
|
-
id = "%0#{width}d"
|
36
|
+
id = format("%0#{width}d", id.to_i) if options[:width]
|
37
37
|
id = options[:prefix] + id.to_s if options[:prefix]
|
38
38
|
|
39
39
|
destination = File.join( options[:output], "#{id}#{base_name}")
|
@@ -44,13 +44,13 @@ module Datashift
|
|
44
44
|
end
|
45
45
|
end
|
46
46
|
|
47
|
-
desc
|
47
|
+
desc 'zip', 'Create zip of files'
|
48
|
+
|
49
|
+
method_option :path, aliases: '-p', required: true, desc: 'The path to the digital files'
|
50
|
+
method_option :output, aliases: '-o', required: true, desc: 'The resulting zip file name'
|
48
51
|
|
49
|
-
method_option :path, :aliases => '-p', required: true, desc: "The path to the digital files"
|
50
|
-
method_option :output, :aliases => '-o', required: true, desc: "The resulting zip file name"
|
51
|
-
|
52
52
|
def zip
|
53
|
-
|
53
|
+
|
54
54
|
require 'zip/zip'
|
55
55
|
require 'zip/zipfilesystem'
|
56
56
|
|
@@ -63,36 +63,36 @@ module Datashift
|
|
63
63
|
end
|
64
64
|
end
|
65
65
|
end
|
66
|
-
|
67
|
-
desc
|
68
|
-
|
69
|
-
method_option :path, :
|
70
|
-
method_option :results, :
|
71
|
-
|
72
|
-
def zip_matching
|
73
|
-
|
66
|
+
|
67
|
+
desc 'zip_matching', 'Create zip of matching digital files e.g zip up pdf, jpg and png versions of a file'
|
68
|
+
|
69
|
+
method_option :path, aliases: '-p', required: true, desc: 'The path to the digital files'
|
70
|
+
method_option :results, aliases: '-r', required: true, desc: 'The path to store resulting zip files'
|
71
|
+
|
72
|
+
def zip_matching
|
73
|
+
|
74
74
|
require 'zip/zip'
|
75
75
|
require 'zip/zipfilesystem'
|
76
76
|
|
77
77
|
ready_to_zip = {}
|
78
78
|
Dir[File.join(options[:path], '**', '*.*')].each do |p|
|
79
79
|
next if File.directory? p
|
80
|
-
|
80
|
+
|
81
81
|
basename = File.basename(p, '.*')
|
82
|
-
ready_to_zip[basename] ||= []
|
83
|
-
ready_to_zip[basename] << p
|
84
|
-
end
|
85
|
-
|
82
|
+
ready_to_zip[basename] ||= []
|
83
|
+
ready_to_zip[basename] << p
|
84
|
+
end
|
85
|
+
|
86
86
|
output = options[:results]
|
87
|
-
|
88
|
-
FileUtils
|
89
|
-
|
87
|
+
|
88
|
+
FileUtils.mkdir_p(output) unless File.exist?(output)
|
89
|
+
|
90
90
|
puts "Creating #{ready_to_zip.keys.size} new zips"
|
91
91
|
ready_to_zip.each do |basename, paths|
|
92
|
-
|
93
|
-
z= File.join(output, basename + '.zip')
|
92
|
+
|
93
|
+
z = File.join(output, basename + '.zip')
|
94
94
|
puts "zipping to #{z}"
|
95
|
-
|
95
|
+
|
96
96
|
Zip::ZipOutputStream.open(z) do |zos|
|
97
97
|
paths.each do |file|
|
98
98
|
zos.put_next_entry(File.basename(file))
|
@@ -100,10 +100,9 @@ module Datashift
|
|
100
100
|
end
|
101
101
|
end
|
102
102
|
end
|
103
|
-
|
104
|
-
end
|
105
|
-
|
103
|
+
|
104
|
+
end
|
105
|
+
|
106
106
|
end
|
107
107
|
|
108
108
|
end
|
109
|
-
|