linecook-gem 0.0.3 → 0.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6f3d23161644b325da20922d5a50a9915344b80d
4
- data.tar.gz: bafc625040aa05abc282487ab7ae5f61c7206f3e
3
+ metadata.gz: e6a9328562fa6cf23fc9c9d97500f929b07299e3
4
+ data.tar.gz: 70c988ff74e983fed3ae8e74c20c3cd59ea6baaa
5
5
  SHA512:
6
- metadata.gz: d06ea091b4b2df1f79e5f61f75b232fa29af7230ad06bb60daa2caf41e497fa549f3de272e1d990e0143f578ebcedc4675d77e0678ec225da9cab9f6386294db
7
- data.tar.gz: cbd070f9ce4eabaf39d9be7bcb03c66254e02dbc1e4591368950722f4653e3fea85abef8bc41d3041ff86d1e4c9257a17172cda0e0e054eaf45d70c01407b326
6
+ metadata.gz: c7d3c9ce2562d88673b7bed970b8411c1a560d394e8c5401f982b5608464ca9a2c38026fbf825bf55b6097c5145c305bc3489cd21ac9e2a934d56770d513a9ea
7
+ data.tar.gz: 5856d130e60480ca800ed090ef5e59b016e627225ca4f4ba6b00a9d9e6b6498218f0fd3737ea026802cb1b38efb2b73f60810199a521632d8189929043309542
@@ -0,0 +1,28 @@
1
+ require 'forwardable'
2
+ require 'linecook/builder/manager'
3
+
4
+ module Linecook
5
+ class Build
6
+ extend Forwardable
7
+
8
+ def_instance_delegators :@container, :stop, :start, :ip, :info
9
+
10
+ def initialize(name, image: nil)
11
+ Linecook::Builder.start
12
+ @name = name
13
+ @image = image || Linecook::Config.load_config[:provisioner][:default_image]
14
+ @container = Linecook::Lxc::Container.new(name: @name, image: @image, remote: Linecook::Builder.ssh)
15
+ end
16
+
17
+ def ssh
18
+ @ssh ||= Linecook::SSH.new(@container.ip, username: 'ubuntu', password: 'ubuntu', proxy: Linecook::Builder.ssh, keyfile: Linecook::SSH.private_key)
19
+ end
20
+
21
+ def snapshot(save: false)
22
+ path = "/tmp/#{@name}-#{Time.now.to_i}.squashfs"
23
+ Linecook::Builder.ssh.run("sudo mksquashfs #{@container.root} #{path} -wildcards -e 'usr/src' 'var/lib/apt/lists/archive*' 'var/cache/apt/archives'") # FIXME make these excludes dynamic based on OS
24
+ Linecook::Builder.ssh.download(path, local: File.join(Linecook::ImageManager::IMAGE_PATH, File.basename(path))) if save
25
+ path
26
+ end
27
+ end
28
+ end
@@ -1,6 +1,9 @@
1
1
  require 'securerandom'
2
- require 'linecook/image'
2
+ require 'linecook/image/manager'
3
3
  # FIXME: read config values from config file
4
+ # - create cache loopback image
5
+ # - dd, based on config file.
6
+
4
7
 
5
8
  module Linecook
6
9
  module OSXBuilder
@@ -1,13 +1,13 @@
1
1
  module Linecook
2
2
  module LinuxBuilder
3
3
  extend self
4
- LXC_MIN_VERSION = '1.0.7'
4
+ LXC_MIN_VERSION = '1.1.4'
5
5
 
6
6
  def backend
7
7
  check_lxc_version
8
8
  config = Linecook::Config.load_config[:builder]
9
- images = Linecook::Config.load_config[:images]
10
- Linecook::Lxc::Container.new(name: config[:name], home: config[:home], image: images[config[:image]])
9
+ images = Linecook::Config.load_config[:image][:images]
10
+ Linecook::Lxc::Container.new(name: config[:name], home: config[:home], image: config[:image], bridge: true)
11
11
  end
12
12
 
13
13
  private
@@ -1,34 +1,37 @@
1
- require 'linecook/ssh'
2
- require 'linecook/image'
3
- require 'linecook/config'
1
+ require 'linecook/image/manager'
2
+ require 'linecook/util/ssh'
3
+ require 'linecook/util/config'
4
+ require 'linecook/util/executor'
4
5
  require 'tempfile'
5
6
  require 'ipaddress'
6
7
 
7
8
  module Linecook
8
9
  module Lxc
9
10
  class Container
11
+
12
+ include Executor
10
13
  MAX_WAIT = 60
11
- attr_reader :config
12
- def initialize(name: 'linecook', home: '/u/lxc', image: nil, remote: :local)
13
- @remote = remote == :local ? false : remote
14
- config = { utsname: name, rootfs: File.join(home, name, 'rootfs') }
15
- config.merge!(network: { type: 'veth', flags: 'up', link: 'lxcbr0' }) # FIXME
16
- @config = Linecook::Lxc::Config.generate(config) # FIXME read link from config
17
- @source_image = image || Linecook::Config.load_config[:images][:base_image]
14
+ attr_reader :config, :root
15
+ def initialize(name: 'linecook', home: '/u/lxc', image: nil, remote: :local, bridge: false)
18
16
  @name = name
19
17
  @home = home
18
+ @bridge = bridge
19
+ @remote = remote == :local ? false : remote
20
+ @root = File.join(@home, @name, 'rootfs')
21
+ config = { utsname: name, rootfs: @root }
22
+ @config = Linecook::Lxc::Config.generate(config) # FIXME read link from config
23
+ @source_image = image || :base_image
20
24
  end
21
25
 
22
26
  def start
27
+ return if running?
23
28
  setup_image
24
29
  setup_dirs
25
30
  mount_all
26
31
  write_config
27
32
  execute("lxc-start #{container_str} -d")
28
33
  wait_running
29
- # Don't start a cgmanager if we're already in a container
30
- execute('[ -f /etc/init/cgmanager.conf ] && sudo status cgmanager | grep -q running && sudo stop cgmanager || true') if lxc?
31
- setup_bridge unless @remote
34
+ setup_bridge if @bridge
32
35
  wait_ssh
33
36
  unmount unless running?
34
37
  end
@@ -59,7 +62,7 @@ module Linecook
59
62
 
60
63
  def info
61
64
  @info = {}
62
- capture("lxc-info #{container_str}").lines.each do |line|
65
+ capture("lxc-info #{container_str} || true").lines.each do |line|
63
66
  k, v = line.strip.split(/:\s+/)
64
67
  key = k.downcase.to_sym
65
68
  @info[key] = @info[key] ? [@info[key]].flatten << v : v
@@ -95,9 +98,8 @@ module Linecook
95
98
  @upper_base = tmpdir(label: 'upper')
96
99
  @upper_dir = File.join(@upper_base, '/upper')
97
100
  @work_dir = File.join(@upper_base, '/work')
98
- @overlay = File.join(@home, @name, '/rootfs')
99
101
  @socket_dirs = []
100
- (Linecook::Config.load_config[:socket_dirs] ||[]).each{ |sock| @socket_dirs << File.join(@overlay, sock) }
102
+ (Linecook::Config.load_config[:socket_dirs] ||[]).each{ |sock| @socket_dirs << File.join(@root, sock) }
101
103
  end
102
104
 
103
105
  def write_config
@@ -115,14 +117,14 @@ module Linecook
115
117
 
116
118
  def mount_all
117
119
  # Prepare an overlayfs
118
- execute("mkdir -p #{@overlay}")
120
+ execute("mkdir -p #{@root}")
119
121
  execute("mkdir -p #{@lower_dir}")
120
122
  execute("mkdir -p #{@upper_base}")
121
123
  mount(@image_path, @lower_dir, options: '-o loop')
122
124
  mount('tmpfs', @upper_base, type: '-t tmpfs', options:'-o noatime') # FIXME: - don't always be tmpfs
123
125
  execute("mkdir -p #{@work_dir}")
124
126
  execute("mkdir -p #{@upper_dir}")
125
- mount('overlay', @overlay, type: '-t overlay', options: "-o lowerdir=#{@lower_dir},upperdir=#{@upper_dir},workdir=#{@work_dir}")
127
+ mount('overlay', @root, type: '-t overlay', options: "-o lowerdir=#{@lower_dir},upperdir=#{@upper_dir},workdir=#{@work_dir}")
126
128
  # Overlayfs doesn't support unix domain sockets
127
129
  @socket_dirs.each do |sock|
128
130
  execute("mkdir -p #{sock}")
@@ -136,7 +138,7 @@ module Linecook
136
138
 
137
139
  def unmount
138
140
  @socket_dirs.each { |sock| execute("umount #{sock}") }
139
- execute("umount #{@overlay}")
141
+ execute("umount #{@root}")
140
142
  execute("umount #{@upper_base}")
141
143
  execute("umount #{@lower_dir}")
142
144
  execute("rmdir #{@lower_dir}")
@@ -164,17 +166,17 @@ eos
164
166
  interfaces = Tempfile.new('interfaces')
165
167
  interfaces.write(bridge_config)
166
168
  interfaces.close
167
- execute("mv #{interfaces.path} #{File.join(@home, @name, 'rootfs', 'etc', 'network', 'interfaces')}")
169
+ execute("mv #{interfaces.path} #{File.join(@root, 'etc', 'network', 'interfaces')}")
168
170
 
169
171
  execute("lxc-attach -n #{@name} -P #{@home} ifup lxcbr0")
170
172
  end
171
173
 
172
174
 
173
175
  def setup_image
174
- @source_path = Linecook::ImageFetcher.fetch(@source_image)
176
+ @source_path = Linecook::ImageManager.fetch(@source_image)
175
177
  if @remote
176
178
  dest = "#{File.basename(@source_path)}"
177
- @remote.upload(@source_path, dest) unless @remote.test("[ -f #{dest} ]")
179
+ @remote.upload(@source_path, dest) unless test("[ -f #{dest} ]")
178
180
  @image_path = dest
179
181
  else
180
182
  @image_path = @source_path
@@ -189,26 +191,6 @@ eos
189
191
  "-n #{@name} -P #{@home}"
190
192
  end
191
193
 
192
- def capture(command, sudo: true)
193
- execute(command, sudo: sudo, capture: true)
194
- end
195
-
196
- def execute(command, sudo: true, capture: false)
197
- command = "sudo #{command}" if sudo
198
- if @remote
199
- if capture
200
- return @remote.capture(command)
201
- else
202
- @remote.run(command)
203
- end
204
- else
205
- if capture
206
- return `#{command}`
207
- else
208
- system(command)
209
- end
210
- end
211
- end
212
194
  end
213
195
 
214
196
  module Config
@@ -222,7 +204,7 @@ eos
222
204
  network: {
223
205
  type: 'veth',
224
206
  flags: 'up',
225
- link: 'br0'
207
+ link: 'lxcbr0'
226
208
  },
227
209
  mount: {
228
210
  auto: 'cgroup'
@@ -0,0 +1,56 @@
1
+ require 'forwardable'
2
+ require 'sshkey'
3
+
4
+ require 'linecook/builder/lxc'
5
+ require 'linecook/builder/darwin_backend'
6
+ require 'linecook/builder/linux_backend'
7
+ require 'linecook/builder/build'
8
+
9
+ module Linecook
10
+ module Builder
11
+ extend self
12
+ extend Forwardable
13
+ BUILD_HOME = '/u/lxc'
14
+
15
+ def_instance_delegators :backend, :stop, :ip, :info, :running?
16
+
17
+ def backend
18
+ @backend ||= backend_for_platform
19
+ end
20
+
21
+ def start
22
+ return if running?
23
+ backend.start
24
+ end
25
+
26
+ def ssh
27
+ config = Linecook::Config.load_config[:builder]
28
+ @ssh ||= SSH.new(ip, username: config[:username], password: config[:password], keyfile: Linecook::SSH.private_key)
29
+ end
30
+
31
+ def builds
32
+ ssh.test("[ -d #{BUILD_HOME} ]") ? ssh.capture("find #{BUILD_HOME} -maxdepth 1 -mindepth 1 -type d -printf \"%f\n\"").delete(';').lines : []
33
+ end
34
+
35
+ def build_info
36
+ info = {}
37
+ builds.each do |build|
38
+ info[build] = Linecook::Build.new(build, nil).info
39
+ end
40
+ info
41
+ end
42
+
43
+ private
44
+
45
+ def backend_for_platform
46
+ case Config.platform
47
+ when 'linux'
48
+ LinuxBuilder.backend
49
+ when 'darwin'
50
+ OSXBuilder.backend
51
+ else
52
+ fail "Cannot find supported backend for #{Config.platform}"
53
+ end
54
+ end
55
+ end
56
+ end
data/lib/linecook/cli.rb CHANGED
@@ -1,6 +1,56 @@
1
1
  require 'thor'
2
2
  require 'linecook'
3
3
 
4
+ class Crypto < Thor
5
+ desc 'keygen', 'Generate AES key for securing images'
6
+ def keygen
7
+ puts Linecook::Crypto.keygen
8
+ end
9
+
10
+ desc 'decrypt IMAGE', ''
11
+ def decrypt(image)
12
+ puts Linecook::Crypto.new.decrypt_file(image)
13
+ end
14
+
15
+ desc 'encrypt IMAGE', ''
16
+ def encrypt(image)
17
+ puts Linecook::Crypto.new.encrypt_file(image)
18
+ end
19
+ end
20
+
21
+ class Image < Thor
22
+ desc 'crypto SUBCOMMAND', 'Manage image encryption'
23
+ subcommand 'crypto', Crypto
24
+
25
+ desc 'list', 'List images' # add remote flag
26
+ def list
27
+ end
28
+
29
+ desc 'fetch IMAGE_NAME', 'Fetch an image by name'
30
+ method_options name: :string
31
+ def fetch(name)
32
+ Linecook::ImageManager.fetch(name)
33
+ end
34
+
35
+ desc 'upload IMAGE', 'Upload an image'
36
+ method_options name: :string
37
+ def upload(image)
38
+ Linecook::ImageManager.upload(image)
39
+ end
40
+
41
+ desc 'url IMAGE', 'Get URL for image'
42
+ method_options image: :string
43
+ def url(image)
44
+ puts Linecook::ImageManager.url(image)
45
+ end
46
+
47
+ desc 'package IMAGE', 'Package image'
48
+ method_options image: :string
49
+ def package(image)
50
+ Linecook::Packager.package(image)
51
+ end
52
+ end
53
+
4
54
  class Builder < Thor
5
55
  desc 'info', 'Show builder info'
6
56
  def info
@@ -29,27 +79,42 @@ class Build < Thor
29
79
  puts Linecook::Builder.builds
30
80
  end
31
81
 
32
- desc 'info', 'Show build info' # FIXME: accept the build name
82
+ desc 'info', 'Show build info'
33
83
  def info
34
- puts Linecook::Builder.build_info
84
+ end
85
+
86
+ desc 'ip', 'Show IP address for build'
87
+ def ip
88
+ end
89
+
90
+ desc 'snapshot', 'Take a snapshot of the build with NAME'
91
+ method_option :name, type: :string, required: true, banner: 'ROLE_NAME', desc: 'Name of the role to build', aliases: '-n'
92
+ def snapshot
93
+ build = Linecook::Build.new(name, '')
94
+ build.snapshot(save: true)
35
95
  end
36
96
  end
37
97
 
38
98
  class Linecook::CLI < Thor
39
- desc 'builder SUBCOMMAND', 'Manage builders'
99
+
100
+ desc 'image SUBCOMMAND', 'Manage linecook images.'
101
+ subcommand 'image', Image
102
+
103
+ desc 'builder SUBCOMMAND', 'Manage the builder.'
40
104
  subcommand 'builder', Builder
41
105
 
42
- desc 'build SUBCOMMAND', 'Manage builds'
106
+ desc 'build SUBCOMMAND', 'Manage running and completed builds.'
43
107
  subcommand 'build', Build
44
108
 
45
- desc 'bake', 'Bake a new image'
109
+ desc 'bake', 'Bake a new image.'
110
+ method_option :name, type: :string, required: true, banner: 'ROLE_NAME', desc: 'Name of the role to build', aliases: '-n'
111
+ method_option :image, type: :string, banner: 'SOURCE_IMAGE', desc: 'Source image to seed the build.', aliases: '-i'
112
+ method_option :build, type: :boolean, default: true, desc: 'Build the image', aliases: '-b'
113
+ method_option :snapshot, type: :boolean, default: false, desc: 'Snapshot the resulting build to create an image', aliases: '-s'
114
+ method_option :upload, type: :boolean, default: false, desc: 'Upload the resulting build. Implies --snapshot and --encrypt.', aliases: '-u'
115
+ method_option :package, type: :boolean, default: false, desc: 'Package the resulting image. Implies --upload, --snapshot and --encrypt.', aliases: '-p'
46
116
  def bake
47
- Linecook::Baker.bake
48
- end
49
-
50
- desc 'fetch IMAGE_NAME', 'Fetch an image by name'
51
- method_options name: :string
52
- def fetch(name)
53
- Linecook::ImageFetcher.fetch(name)
117
+ opts = options.symbolize_keys
118
+ Linecook::Baker.bake(**opts)
54
119
  end
55
120
  end
@@ -0,0 +1,49 @@
1
+ require 'base64'
2
+ require 'encryptor'
3
+
4
+ require 'linecook/image/manager'
5
+ require 'linecook/util/executor'
6
+ require 'linecook/util/config'
7
+
8
+ module Linecook
9
+ class Crypto
10
+ include Executor
11
+ CIPHER = 'aes-256-cbc'
12
+ KEY_BYTES = 32 # 256 bits
13
+ attr_reader :iv, :secret_key
14
+
15
+ def initialize(remote: nil)
16
+ @remote = remote
17
+ load_key
18
+ end
19
+
20
+ def encrypt_image(image)
21
+ image_path = File.join(Linecook::ImageManager::IMAGE_PATH,File.basename(image))
22
+ encrypt_file(image_path)
23
+ end
24
+
25
+ def encrypt_file(source, dest: nil, keypath: nil)
26
+ dest ||= "/tmp/#{File.basename(source)}"
27
+ capture("openssl enc -#{CIPHER} -out #{dest} -in #{source} -K #{@secret_key} -iv #{@iv}")
28
+ dest
29
+ end
30
+
31
+ def decrypt_file(source, dest: nil, keypath: nil)
32
+ dest ||= "/tmp/#{File.basename(source)}-decrypted"
33
+ capture("openssl enc -#{CIPHER} -out #{dest} -in #{source} -K #{@secret_key} -iv #{@iv} -d")
34
+ dest
35
+ end
36
+
37
+ def self.keygen
38
+ iv = OpenSSL::Cipher::Cipher.new(CIPHER).random_iv.unpack('H*').first
39
+ secret_key = Base64.encode64(OpenSSL::Random.random_bytes(KEY_BYTES)).unpack('H*').first
40
+ "[:IV:#{iv}:KY:#{secret_key}]"
41
+ end
42
+
43
+ private
44
+
45
+ def load_key
46
+ @iv, @secret_key = Linecook::Config.secrets['aeskey'].match(/\[:IV:(.+):KY:(.+)\]/m).captures
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,27 @@
1
+ require 'octokit'
2
+
3
+ require 'linecook/util/config'
4
+
5
+ module Linecook
6
+ module GithubManager
7
+ extend self
8
+
9
+ def url(name)
10
+ latest[:assets].find { |a| a[:name] =~ /#{name}/ }[:browser_download_url]
11
+ end
12
+
13
+ private
14
+
15
+ def client
16
+ @client ||= Octokit::Client.new
17
+ end
18
+
19
+ def source
20
+ @source ||= (Config.load_config['source_repo'] || 'dalehamel/lxb')
21
+ end
22
+
23
+ def latest
24
+ client.releases(source).sort_by { |r| r[:published_at] }.last
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,43 @@
1
+ require 'linecook/image/crypt'
2
+ require 'linecook/image/github'
3
+ require 'linecook/image/s3'
4
+ require 'linecook/util/downloader'
5
+
6
+ module Linecook
7
+ module ImageManager
8
+ IMAGE_PATH = File.join(Config::LINECOOK_HOME, 'images').freeze
9
+ extend self
10
+
11
+ def fetch(name, upgrade:false, profile: :private)
12
+ image_name = Linecook::Config.load_config[:image][:images][name][:name]
13
+ path = File.join(IMAGE_PATH, image_name)
14
+ url = provider(profile).url(name) unless File.exist?(path) || upgrade# FIXME
15
+ Linecook::Downloader.download(url, path) unless File.exist?(path) || upgrade
16
+ path
17
+ end
18
+
19
+ def upload(image, profile: :private)
20
+ path = File.join(IMAGE_PATH, File.basename(image))
21
+ puts "Encrypting and uploading image #{path}"
22
+ provider(profile).upload(Linecook::Crypto.new.encrypt_file(path))
23
+ end
24
+
25
+ def url(image, profile: :private)
26
+ provider(profile).url("builds/#{image}")
27
+ end
28
+
29
+ private
30
+
31
+ def provider(image_profile)
32
+ profile = Linecook::Config.load_config[:image][:provider][image_profile]
33
+ case profile
34
+ when :s3
35
+ S3Manager
36
+ when :github
37
+ GithubManager
38
+ else
39
+ fail "No provider implemented for for #{profile}"
40
+ end
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,47 @@
1
+ require 'ruby-progressbar'
2
+ require 'aws-sdk'
3
+
4
+ module Linecook
5
+ module S3Manager
6
+ extend self
7
+ EXPIRY = 20
8
+
9
+ def url(name)
10
+ client
11
+ s3 = Aws::S3::Resource.new
12
+ obj = s3.bucket(Linecook::Config.secrets['bucket']).object(name)
13
+ obj.presigned_url(:get, expires_in: EXPIRY * 60)
14
+ end
15
+
16
+ def upload(path)
17
+ File.open(path, 'rb') do |file|
18
+ fname = File.basename(path)
19
+ pbar = ProgressBar.create(title: fname, total: file.size)
20
+ common_opts = { bucket: Linecook::Config.secrets['bucket'], key: File.join('builds', fname) }
21
+ resp = client.create_multipart_upload(storage_class: 'REDUCED_REDUNDANCY', server_side_encryption: 'AES256', **common_opts)
22
+ id = resp.upload_id
23
+ part = 0
24
+ total = 0
25
+ parts = []
26
+ while content = file.read(1048576 * 20)
27
+ part += 1
28
+ resp = client.upload_part(body: content, content_length: content.length, part_number: part, upload_id: id, **common_opts)
29
+ parts << { etag: resp.etag, part_number: part }
30
+ total += content.length
31
+ pbar.progress = total
32
+ pbar.title = "#{fname} - (#{((total.to_f/file.size.to_f)*100.0).round(2)}%)"
33
+ end
34
+ client.complete_multipart_upload(upload_id: id, multipart_upload: { parts: parts }, **common_opts)
35
+ end
36
+ end
37
+
38
+ private
39
+ def client
40
+ @client ||= begin
41
+ Aws.config[:credentials] = Aws::Credentials.new(Linecook::Config.secrets['aws_access_key'], Linecook::Config.secrets['aws_secret_key'])
42
+ Aws.config[:region] = 'us-east-1'
43
+ Aws::S3::Client.new
44
+ end
45
+ end
46
+ end
47
+ end