lexicon-common 0.1.0 → 0.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lexicon-common.gemspec +2 -0
- data/lib/lexicon/common/database/database.rb +24 -2
- data/lib/lexicon/common/mixin/logger_aware.rb +2 -2
- data/lib/lexicon/common/mixin/nameable.rb +16 -0
- data/lib/lexicon/common/package/directory_package_loader.rb +56 -12
- data/lib/lexicon/common/package/package.rb +11 -43
- data/lib/lexicon/common/package/package_file.rb +50 -0
- data/lib/lexicon/common/package/v1/package.rb +83 -0
- data/lib/lexicon/common/package/v1/package_builder.rb +70 -0
- data/lib/lexicon/common/package/v1/source_file_set.rb +28 -0
- data/lib/lexicon/common/package/v2/package.rb +56 -0
- data/lib/lexicon/common/package/v2/package_builder.rb +72 -0
- data/lib/lexicon/common/package/v2/source_file_set.rb +26 -0
- data/lib/lexicon/common/production/datasource_loader.rb +119 -60
- data/lib/lexicon/common/production/table_locker.rb +58 -0
- data/lib/lexicon/common/psql.rb +47 -0
- data/lib/lexicon/common/remote/package_downloader.rb +61 -43
- data/lib/lexicon/common/remote/package_uploader.rb +37 -28
- data/lib/lexicon/common/remote/s3_client.rb +6 -0
- data/lib/lexicon/common/schema/validator_factory.rb +1 -1
- data/lib/lexicon/common/shell_executor.rb +0 -3
- data/lib/lexicon/common/version.rb +1 -1
- data/lib/lexicon/common.rb +1 -0
- data/lib/lexicon-common.rb +3 -0
- data/resources/lexicon.schema.json +116 -31
- metadata +45 -9
- data/lib/lexicon/common/package/package_builder.rb +0 -68
- data/lib/lexicon/common/package/source_file_set.rb +0 -24
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5b24914e5f9dbd3d4261ec1b6c04741958a5d9fcfccb9dde8fd14b92b5c19b57
|
4
|
+
data.tar.gz: 7b2a40b0e4e033dbc111804a5c6a6a7105fdd2ac991d72baeb7022e66f585913
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0b7e024f856d40571826aa8237cfbd0b1ea55be19bf31f997ce5f65cf04c561ffaa43d4b5c23b5fcdda05ae7f3b48629ae0ca68c0c91903fa362e4674e3cdbd4
|
7
|
+
data.tar.gz: 7c225eb0b67789074fa76e5a50bcc61439503b5fb579d811077732b7f9763efc822f7f4c12d2aa314d5b0719ef3820d0eff097e08aaa3c167d442f48665dbe64
|
data/lexicon-common.gemspec
CHANGED
@@ -20,6 +20,8 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.add_dependency 'aws-sdk-s3', '~> 1.84'
|
22
22
|
spec.add_dependency 'colored', '~> 1.2'
|
23
|
+
spec.add_dependency 'concurrent-ruby', '~> 1.1'
|
24
|
+
spec.add_dependency 'corindon', '~> 0.8.0'
|
23
25
|
spec.add_dependency 'json_schemer', '~> 0.2.16'
|
24
26
|
spec.add_dependency 'pg', '~> 1.2'
|
25
27
|
spec.add_dependency 'semantic', '~> 1.6'
|
@@ -50,7 +50,7 @@ module Lexicon
|
|
50
50
|
drop_schema(schema, cascade: true)
|
51
51
|
end
|
52
52
|
|
53
|
-
def drop_schema(name, cascade: false)
|
53
|
+
def drop_schema(name, cascade: false, if_exists: false)
|
54
54
|
cascade = if cascade
|
55
55
|
' CASCADE'
|
56
56
|
else
|
@@ -58,7 +58,7 @@ module Lexicon
|
|
58
58
|
end
|
59
59
|
|
60
60
|
query <<~SQL
|
61
|
-
DROP SCHEMA "#{name}"#{cascade};
|
61
|
+
DROP SCHEMA #{if_exists ? 'IF EXISTS ' : ''}"#{name}"#{cascade};
|
62
62
|
SQL
|
63
63
|
end
|
64
64
|
|
@@ -95,6 +95,28 @@ module Lexicon
|
|
95
95
|
@connection.copy_data(sql) { block.call(put_data) }
|
96
96
|
end
|
97
97
|
|
98
|
+
# @param [#to_s] table
|
99
|
+
# @param [#to_s | nil] schema
|
100
|
+
# @return [Boolean]
|
101
|
+
def table_exists?(table, schema: nil)
|
102
|
+
schema = search_path.first if schema.nil?
|
103
|
+
|
104
|
+
query(<<~SQL, table, schema).any?
|
105
|
+
SELECT table_name FROM information_schema.tables
|
106
|
+
WHERE table_name = $1 AND table_schema = $2
|
107
|
+
SQL
|
108
|
+
end
|
109
|
+
|
110
|
+
# @param [#to_s] schema_name
|
111
|
+
# @return [Boolean]
|
112
|
+
def schema_exists?(schema_name)
|
113
|
+
query(<<~SQL, schema_name).count > 0
|
114
|
+
SELECT "schema_name"
|
115
|
+
FROM "information_schema"."schemata"
|
116
|
+
WHERE "schema_name" = $1
|
117
|
+
SQL
|
118
|
+
end
|
119
|
+
|
98
120
|
private
|
99
121
|
|
100
122
|
# @return [PG::Connection]
|
@@ -17,7 +17,7 @@ module Lexicon
|
|
17
17
|
end
|
18
18
|
|
19
19
|
# @param [String] name
|
20
|
-
# @return [Package, nil]
|
20
|
+
# @return [Package::Package, nil]
|
21
21
|
def load_package(name)
|
22
22
|
package_dir = root_dir.join(name.to_s)
|
23
23
|
|
@@ -40,18 +40,17 @@ module Lexicon
|
|
40
40
|
json = JSON.parse(spec_file.read)
|
41
41
|
|
42
42
|
if @schema_validator.valid?(json)
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
)
|
52
|
-
end
|
43
|
+
package_version = json.fetch('schema_version', 1)
|
44
|
+
case package_version
|
45
|
+
when 1
|
46
|
+
load_v1(dir: dir, spec_file: spec_file, checksum_file: checksum_file, json: json)
|
47
|
+
when 2
|
48
|
+
load_v2(dir: dir, spec_file: spec_file, checksum_file: checksum_file, json: json)
|
49
|
+
else
|
50
|
+
log("Package version #{package_version} is not supported")
|
53
51
|
|
54
|
-
|
52
|
+
nil
|
53
|
+
end
|
55
54
|
else
|
56
55
|
log("Package at path #{dir} has invalid manifest")
|
57
56
|
|
@@ -61,6 +60,51 @@ module Lexicon
|
|
61
60
|
nil
|
62
61
|
end
|
63
62
|
end
|
63
|
+
|
64
|
+
# @param [Pathname] checksum_file
|
65
|
+
# @param [Pathname] dir
|
66
|
+
# @param [Hash] json
|
67
|
+
# @param [Pathname] spec_file
|
68
|
+
# @return [V1::Package]
|
69
|
+
def load_v1(dir:, spec_file:, checksum_file:, json:)
|
70
|
+
version = Semantic::Version.new(json.fetch('version'))
|
71
|
+
file_sets = json.fetch('content').map do |id, values|
|
72
|
+
V1::SourceFileSet.new(
|
73
|
+
id: id,
|
74
|
+
name: values.fetch('name'),
|
75
|
+
structure: values.fetch('structure'),
|
76
|
+
data: values.fetch('data', nil),
|
77
|
+
tables: values.fetch('tables', [])
|
78
|
+
)
|
79
|
+
end
|
80
|
+
|
81
|
+
V1::Package.new(file_sets: file_sets, version: version, dir: dir, checksum_file: checksum_file, spec_file: spec_file)
|
82
|
+
end
|
83
|
+
|
84
|
+
# @param [Pathname] checksum_file
|
85
|
+
# @param [Pathname] dir
|
86
|
+
# @param [Hash] json
|
87
|
+
# @param [Pathname] spec_file
|
88
|
+
# @return [V2::Package]
|
89
|
+
def load_v2(dir:, spec_file:, checksum_file:, json:)
|
90
|
+
version = Semantic::Version.new(json.fetch('version'))
|
91
|
+
file_sets = json.fetch('content').map do |id, values|
|
92
|
+
V2::SourceFileSet.new(
|
93
|
+
id: id,
|
94
|
+
name: values.fetch('name'),
|
95
|
+
structure: values.fetch('structure'),
|
96
|
+
tables: values.fetch('tables', {})
|
97
|
+
)
|
98
|
+
end
|
99
|
+
|
100
|
+
V2::Package.new(
|
101
|
+
file_sets: file_sets,
|
102
|
+
version: version,
|
103
|
+
dir: dir,
|
104
|
+
checksum_file: checksum_file,
|
105
|
+
spec_file: spec_file
|
106
|
+
)
|
107
|
+
end
|
64
108
|
end
|
65
109
|
end
|
66
110
|
end
|
@@ -11,65 +11,33 @@ module Lexicon
|
|
11
11
|
attr_reader :checksum_file, :spec_file
|
12
12
|
# @return [Pathname]
|
13
13
|
attr_reader :dir
|
14
|
-
# @return [Array<SourceFileSet>]
|
15
|
-
attr_reader :file_sets
|
16
14
|
# @return [Semantic::Version]
|
17
15
|
attr_reader :version
|
16
|
+
# @return [Integer]
|
17
|
+
attr_reader :schema_version
|
18
18
|
|
19
|
-
# @param [Array<SourceFileSet>] file_sets
|
20
|
-
# @param [Pathname] dir
|
21
19
|
# @param [Pathname] checksum_file
|
20
|
+
# @param [Pathname] dir
|
21
|
+
# @param [Integer] schema_version
|
22
|
+
# @param [Pathname] spec_file
|
22
23
|
# @param [Semantic::Version] version
|
23
|
-
def initialize(
|
24
|
+
def initialize(checksum_file:, dir:, schema_version:, spec_file:, version:)
|
24
25
|
@checksum_file = checksum_file
|
25
26
|
@dir = dir
|
26
|
-
@
|
27
|
+
@schema_version = schema_version
|
27
28
|
@spec_file = spec_file
|
28
29
|
@version = version
|
29
30
|
end
|
30
31
|
|
31
32
|
# @return [Boolean]
|
32
33
|
def valid?
|
33
|
-
checksum_file.exist? && dir.directory? &&
|
34
|
-
end
|
35
|
-
|
36
|
-
# @return [Array<Pathname>]
|
37
|
-
def structure_files
|
38
|
-
file_sets.map { |fs| dir.join(relative_structure_path(fs)) }
|
34
|
+
checksum_file.exist? && dir.directory? && files.all? { |f| f.path.exist? rescue false }
|
39
35
|
end
|
40
36
|
|
41
|
-
# @
|
42
|
-
|
43
|
-
|
44
|
-
dir.join(relative_data_path(file_set))
|
45
|
-
end
|
46
|
-
|
47
|
-
# @return [Pathname]
|
48
|
-
def data_dir
|
49
|
-
dir.join('data')
|
37
|
+
# @return [Array<PackageFile>] Array of File of the package
|
38
|
+
def files
|
39
|
+
[]
|
50
40
|
end
|
51
|
-
|
52
|
-
# @return [Pathname, nil]
|
53
|
-
def relative_data_path(file_set)
|
54
|
-
if file_set.data_path.nil?
|
55
|
-
nil
|
56
|
-
else
|
57
|
-
data_dir.basename.join(file_set.data_path)
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
# @return [Pathname]
|
62
|
-
def relative_structure_path(file_set)
|
63
|
-
data_dir.basename.join(file_set.structure_path)
|
64
|
-
end
|
65
|
-
|
66
|
-
private
|
67
|
-
|
68
|
-
def all_sets_valid?
|
69
|
-
file_sets.all? do |set|
|
70
|
-
data_dir.join(set.structure_path).exist? && !set.data_path.nil? && data_dir.join(set.data_path).exist?
|
71
|
-
end
|
72
|
-
end
|
73
41
|
end
|
74
42
|
end
|
75
43
|
end
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Lexicon
|
4
|
+
module Common
|
5
|
+
module Package
|
6
|
+
class PackageFile
|
7
|
+
class << self
|
8
|
+
def new_structure(path)
|
9
|
+
new(path, type: STRUCTURE)
|
10
|
+
end
|
11
|
+
|
12
|
+
def new_data(path)
|
13
|
+
new(path, type: DATA)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
# @return [Pathname]
|
18
|
+
attr_reader :path
|
19
|
+
|
20
|
+
# @return [String]
|
21
|
+
def to_s
|
22
|
+
path.to_s
|
23
|
+
end
|
24
|
+
|
25
|
+
# @return [Boolean]
|
26
|
+
def data?
|
27
|
+
type == DATA
|
28
|
+
end
|
29
|
+
|
30
|
+
# @return [Boolean]
|
31
|
+
def structure?
|
32
|
+
type == STRUCTURE
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
attr_reader :type
|
38
|
+
|
39
|
+
DATA = 'data'
|
40
|
+
STRUCTURE = 'structure'
|
41
|
+
|
42
|
+
def initialize(path, type:)
|
43
|
+
@path = path
|
44
|
+
@type = type
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Lexicon
|
4
|
+
module Common
|
5
|
+
module Package
|
6
|
+
module V1
|
7
|
+
class Package < Common::Package::Package
|
8
|
+
# @return [Array<SourceFileSet>]
|
9
|
+
attr_reader :file_sets
|
10
|
+
|
11
|
+
# @param [Array<SourceFileSet>] file_sets
|
12
|
+
# @param [Pathname] dir
|
13
|
+
# @param [Pathname] checksum_file
|
14
|
+
# @param [Semantic::Version] version
|
15
|
+
def initialize(version:, spec_file:, checksum_file:, dir:, file_sets:)
|
16
|
+
super(
|
17
|
+
checksum_file: checksum_file,
|
18
|
+
dir: dir,
|
19
|
+
spec_file: spec_file,
|
20
|
+
schema_version: 1,
|
21
|
+
version: version,
|
22
|
+
)
|
23
|
+
|
24
|
+
@file_sets = file_sets
|
25
|
+
end
|
26
|
+
|
27
|
+
# @return [Boolean]
|
28
|
+
def valid?
|
29
|
+
super && data_dir.directory? && all_sets_valid?
|
30
|
+
end
|
31
|
+
|
32
|
+
def files
|
33
|
+
structures = file_sets.map { |fs| PackageFile.new_structure(relative_structure_path(fs)) }
|
34
|
+
data = file_sets.flat_map do |fs|
|
35
|
+
data_path = relative_data_path(fs)
|
36
|
+
|
37
|
+
if data_path.nil?
|
38
|
+
[]
|
39
|
+
else
|
40
|
+
[PackageFile.new_data(data_path)]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
[*structures, *data]
|
45
|
+
end
|
46
|
+
|
47
|
+
# @param [SourceFileSet] file_set
|
48
|
+
# @return [Pathname]
|
49
|
+
def data_path(file_set)
|
50
|
+
dir.join(relative_data_path(file_set))
|
51
|
+
end
|
52
|
+
|
53
|
+
# @return [Pathname]
|
54
|
+
def data_dir
|
55
|
+
dir.join('data')
|
56
|
+
end
|
57
|
+
|
58
|
+
# @return [Pathname, nil]
|
59
|
+
def relative_data_path(file_set)
|
60
|
+
if file_set.data_path.nil?
|
61
|
+
nil
|
62
|
+
else
|
63
|
+
data_dir.basename.join(file_set.data_path)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
# @return [Pathname]
|
68
|
+
def relative_structure_path(file_set)
|
69
|
+
data_dir.basename.join(file_set.structure_path)
|
70
|
+
end
|
71
|
+
|
72
|
+
private
|
73
|
+
|
74
|
+
def all_sets_valid?
|
75
|
+
file_sets.all? do |set|
|
76
|
+
data_dir.join(set.structure_path).exist? && !set.data_path.nil? && data_dir.join(set.data_path).exist?
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Lexicon
|
4
|
+
module Common
|
5
|
+
module Package
|
6
|
+
module V1
|
7
|
+
class PackageBuilder < Package
|
8
|
+
def initialize(version:, dir:)
|
9
|
+
super(
|
10
|
+
file_sets: [],
|
11
|
+
version: version,
|
12
|
+
dir: dir,
|
13
|
+
checksum_file: dir.join(CHECKSUM_FILE_NAME),
|
14
|
+
spec_file: dir.join(SPEC_FILE_NAME)
|
15
|
+
)
|
16
|
+
|
17
|
+
FileUtils.mkdir_p(data_dir)
|
18
|
+
end
|
19
|
+
|
20
|
+
# @param [String] id
|
21
|
+
# @param [String] name
|
22
|
+
# @param [Pathname] structure
|
23
|
+
# Takes ownership of the file (moves it to the correct folder)
|
24
|
+
# @param [Array<String>] tables
|
25
|
+
# @param [Pathname] data
|
26
|
+
# Takes ownership of the file (moves it to the correct folder)
|
27
|
+
# @param [String] data_ext
|
28
|
+
def add_file_set(id, name:, structure:, tables:, data: nil, data_ext: '.sql')
|
29
|
+
# @type [Pathname] structure_file_path
|
30
|
+
structure_file_path = data_dir.join(structure_file_name(id))
|
31
|
+
FileUtils.mv(structure.to_s, structure_file_path.to_s)
|
32
|
+
|
33
|
+
# @type [Pathname] data_file_path
|
34
|
+
data_name = if data.nil?
|
35
|
+
nil
|
36
|
+
else
|
37
|
+
dname = data_file_name(id, data_ext)
|
38
|
+
path = data_dir.join(dname)
|
39
|
+
FileUtils.mv(data, path)
|
40
|
+
|
41
|
+
dname
|
42
|
+
end
|
43
|
+
|
44
|
+
file_sets << SourceFileSet.new(
|
45
|
+
id: id,
|
46
|
+
name: name,
|
47
|
+
structure: structure_file_name(id),
|
48
|
+
data: data.nil? ? nil : data_name,
|
49
|
+
tables: tables
|
50
|
+
)
|
51
|
+
end
|
52
|
+
|
53
|
+
def as_package
|
54
|
+
Package.new(version: version, dir: dir, file_sets: file_sets, checksum_file: checksum_file, spec_file: spec_file)
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
def data_file_name(id, ext)
|
60
|
+
"#{id}#{ext}"
|
61
|
+
end
|
62
|
+
|
63
|
+
def structure_file_name(id)
|
64
|
+
"#{id}__structure.sql"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Lexicon
|
4
|
+
module Common
|
5
|
+
module Package
|
6
|
+
module V1
|
7
|
+
class SourceFileSet
|
8
|
+
include Mixin::Nameable
|
9
|
+
|
10
|
+
attr_reader :id, :name, :structure_path, :data_path, :tables
|
11
|
+
|
12
|
+
# @param [String] id
|
13
|
+
# @param [String] name
|
14
|
+
# @param [String] structure
|
15
|
+
# @param [String] data
|
16
|
+
# @param [Array<String>] tables
|
17
|
+
def initialize(id:, name:, structure:, data:, tables:)
|
18
|
+
@id = id
|
19
|
+
@name = name
|
20
|
+
@structure_path = structure
|
21
|
+
@data_path = data
|
22
|
+
@tables = tables
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Lexicon
|
4
|
+
module Common
|
5
|
+
module Package
|
6
|
+
module V2
|
7
|
+
class Package < Common::Package::Package
|
8
|
+
# @param [Pathname] dir
|
9
|
+
# @param [Pathname] checksum_file
|
10
|
+
# @param [Semantic::Version] version
|
11
|
+
def initialize(version:, spec_file:, checksum_file:, dir:, file_sets:)
|
12
|
+
super(
|
13
|
+
checksum_file: checksum_file,
|
14
|
+
dir: dir,
|
15
|
+
spec_file: spec_file,
|
16
|
+
schema_version: 2,
|
17
|
+
version: version,
|
18
|
+
)
|
19
|
+
|
20
|
+
@file_sets = file_sets
|
21
|
+
end
|
22
|
+
|
23
|
+
def valid?
|
24
|
+
super
|
25
|
+
end
|
26
|
+
|
27
|
+
def files
|
28
|
+
file_sets.flat_map { |fs| file_set_files(fs) }
|
29
|
+
end
|
30
|
+
|
31
|
+
# @return [SourceFileSet]
|
32
|
+
attr_reader :file_sets
|
33
|
+
|
34
|
+
def data_dir
|
35
|
+
dir.join('data')
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
# @param [SourceFileSet] file_set
|
41
|
+
# @return [Array<PackageFile>]
|
42
|
+
def file_set_files(file_set)
|
43
|
+
relative_data_dir = data_dir.basename
|
44
|
+
|
45
|
+
structure_file = PackageFile.new_structure(relative_data_dir.join(file_set.structure))
|
46
|
+
table_files = file_set.tables
|
47
|
+
.values.flatten(1)
|
48
|
+
.map { |table_file| PackageFile.new_data(relative_data_dir.join(table_file)) }
|
49
|
+
|
50
|
+
[structure_file, *table_files]
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Lexicon
|
4
|
+
module Common
|
5
|
+
module Package
|
6
|
+
module V2
|
7
|
+
class PackageBuilder < Package
|
8
|
+
def initialize(version:, dir:)
|
9
|
+
super(
|
10
|
+
file_sets: [],
|
11
|
+
version: version,
|
12
|
+
dir: dir,
|
13
|
+
checksum_file: dir.join(CHECKSUM_FILE_NAME),
|
14
|
+
spec_file: dir.join(SPEC_FILE_NAME),
|
15
|
+
)
|
16
|
+
|
17
|
+
FileUtils.mkdir_p(data_dir)
|
18
|
+
end
|
19
|
+
|
20
|
+
# @param [String] id
|
21
|
+
# @param [String] name
|
22
|
+
# @param [Pathname] structure
|
23
|
+
# Takes ownership of the file (moves it to the correct folder)
|
24
|
+
# @param [Hash{String=>Array<Pathname>}] tables
|
25
|
+
# Takes ownership of the files (moves them to the correct folder)
|
26
|
+
def add_file_set(id, name:, structure:, tables:)
|
27
|
+
# @type [Pathname] structure_file_path
|
28
|
+
structure_file_path = data_dir.join(structure_file_name(id))
|
29
|
+
FileUtils.mv(structure.to_s, structure_file_path.to_s)
|
30
|
+
|
31
|
+
table_data = tables.map do |table_name, files|
|
32
|
+
index = 0
|
33
|
+
|
34
|
+
file_names = files.map do |file|
|
35
|
+
file_name = "#{table_name}_#{index}.csv.gz"
|
36
|
+
FileUtils.mv(file.to_s, data_dir.join(file_name))
|
37
|
+
index += 1
|
38
|
+
|
39
|
+
file_name
|
40
|
+
end
|
41
|
+
|
42
|
+
[table_name, file_names]
|
43
|
+
end
|
44
|
+
|
45
|
+
file_sets << SourceFileSet.new(
|
46
|
+
id: id,
|
47
|
+
name: name,
|
48
|
+
structure: structure_file_name(id),
|
49
|
+
tables: table_data.to_h
|
50
|
+
)
|
51
|
+
end
|
52
|
+
|
53
|
+
def as_package
|
54
|
+
Package.new(
|
55
|
+
checksum_file: checksum_file,
|
56
|
+
dir: dir,
|
57
|
+
file_sets: file_sets,
|
58
|
+
spec_file: spec_file,
|
59
|
+
version: version,
|
60
|
+
)
|
61
|
+
end
|
62
|
+
|
63
|
+
private
|
64
|
+
|
65
|
+
def structure_file_name(id)
|
66
|
+
"#{id}__structure.sql"
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Lexicon
|
4
|
+
module Common
|
5
|
+
module Package
|
6
|
+
module V2
|
7
|
+
class SourceFileSet
|
8
|
+
include Mixin::Nameable
|
9
|
+
|
10
|
+
attr_reader :id, :name, :structure, :tables
|
11
|
+
|
12
|
+
# @param [String] id
|
13
|
+
# @param [String] name
|
14
|
+
# @param [String] structure
|
15
|
+
# @param [Hash{String=>Array<String>}] tables
|
16
|
+
def initialize(id:, name:, structure:, tables:)
|
17
|
+
@id = id
|
18
|
+
@name = name
|
19
|
+
@structure = structure
|
20
|
+
@tables = tables.freeze
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|