docker-delta 2.2.12 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/rsync.js CHANGED
@@ -1,56 +1,71 @@
1
- // Generated by CoffeeScript 1.12.7
2
- (function() {
3
- var Promise, fs, mkfifoSync, path, ref, spawn, tmp;
1
+ 'use strict';
4
2
 
5
- Promise = require('bluebird');
3
+ const Bluebird = require('bluebird');
4
+ const fs = require('fs');
5
+ const tmp = require('tmp');
6
+ const path = require('path');
7
+ const { spawn, mkfifoSync } = require('./utils');
6
8
 
7
- fs = require('fs');
8
-
9
- tmp = require('tmp');
10
-
11
- path = require('path');
12
-
13
- ref = require('./utils'), spawn = ref.spawn, mkfifoSync = ref.mkfifoSync;
14
-
15
- exports.createRsyncStream = function(src, dest, ioTimeout, log) {
16
- return new Promise(function(resolve, reject) {
17
- return tmp.dir({
18
- unsafeCleanup: true
19
- }, function(err, tmpDirPath, cleanup) {
20
- var pipePath, ps, rsyncArgs;
21
- if (err) {
22
- log('Failed to create temporary directory');
23
- return reject(err);
24
- }
25
- pipePath = path.join(tmpDirPath, 'rsync.pipe');
26
- mkfifoSync(pipePath);
27
- rsyncArgs = ['--archive', '--compress', '--checksum', '--no-i-r', '--delete', '--hard-links', '--timeout', ioTimeout, '--compress-level=9', '--one-file-system', '--only-write-batch', pipePath, dest, src];
28
- log('Invoking rsync:', 'rsync ' + rsyncArgs.join(' '));
29
- ps = spawn('rsync', rsyncArgs, {
30
- stdio: 'ignore'
31
- });
32
- return fs.open(pipePath, 'r', function(err, fd) {
33
- var stream;
34
- if (err) {
35
- log('Failed to open pipe for reading. Killing rsync...');
36
- ps.kill('SIGUSR1');
37
- return ps.waitAsync().tap(function() {
38
- return log('rsync exited');
39
- }).tapCatch(function(e) {
40
- return log("rsync exited with error: " + e);
41
- })["finally"](function() {
42
- cleanup();
43
- return reject(err);
44
- });
45
- } else {
46
- stream = fs.createReadStream(void 0, {
47
- fd: fd
48
- }).on('close', cleanup);
49
- return resolve([ps, stream]);
50
- }
51
- });
52
- });
53
- });
54
- };
55
-
56
- }).call(this);
9
+ exports.createRsyncStream = function (src, dest, ioTimeout, log) {
10
+ return new Bluebird(function (resolve, reject) {
11
+ return tmp.dir(
12
+ {
13
+ unsafeCleanup: true,
14
+ },
15
+ function (err, tmpDirPath, cleanup) {
16
+ if (err) {
17
+ log('Failed to create temporary directory');
18
+ return reject(err);
19
+ }
20
+ const pipePath = path.join(tmpDirPath, 'rsync.pipe');
21
+ mkfifoSync(pipePath);
22
+ const rsyncArgs = [
23
+ '--archive',
24
+ '--compress',
25
+ '--checksum',
26
+ '--no-i-r',
27
+ '--delete',
28
+ '--hard-links',
29
+ '--timeout',
30
+ ioTimeout,
31
+ '--compress-level=9',
32
+ '--one-file-system',
33
+ '--only-write-batch',
34
+ pipePath,
35
+ dest,
36
+ src,
37
+ ];
38
+ log('Invoking rsync:', 'rsync ' + rsyncArgs.join(' '));
39
+ const ps = spawn('rsync', rsyncArgs, {
40
+ stdio: 'ignore',
41
+ });
42
+ fs.open(pipePath, 'r', function (error, fd) {
43
+ var stream;
44
+ if (error) {
45
+ log('Failed to open pipe for reading. Killing rsync...');
46
+ ps.kill('SIGUSR1');
47
+ return ps
48
+ .waitAsync()
49
+ .tap(function () {
50
+ return log('rsync exited');
51
+ })
52
+ .tapCatch(function (e) {
53
+ return log('rsync exited with error: ' + e);
54
+ })
55
+ ['finally'](function () {
56
+ cleanup();
57
+ return reject(error);
58
+ });
59
+ } else {
60
+ stream = fs
61
+ .createReadStream(void 0, {
62
+ fd: fd,
63
+ })
64
+ .on('close', cleanup);
65
+ return resolve([ps, stream]);
66
+ }
67
+ });
68
+ },
69
+ );
70
+ });
71
+ };
package/lib/utils.js CHANGED
@@ -1,60 +1,55 @@
1
- // Generated by CoffeeScript 1.12.7
2
- (function() {
3
- var Promise, execSync, ref, spawn, waitPidAsync;
1
+ 'use strict';
4
2
 
5
- ref = require('child_process'), spawn = ref.spawn, execSync = ref.execSync;
3
+ const { spawn, execSync } = require('child_process');
6
4
 
7
- Promise = require('bluebird');
5
+ const Bluebird = require('bluebird');
8
6
 
9
- waitPidAsync = function(cmd, ps, status) {
10
- return new Promise(function(resolve, reject) {
11
- var handleExit;
12
- handleExit = function(code, signal) {
13
- var error;
14
- if (code !== 0) {
15
- error = new Error(cmd + " exited. code: " + code + " signal: " + signal);
16
- error.code = code;
17
- error.signal = signal;
18
- return reject(error);
19
- } else {
20
- return resolve();
21
- }
22
- };
23
- if (status.exited) {
24
- return handleExit(status.code, status.signal);
25
- } else if (status.error != null) {
26
- return reject(status.error);
27
- } else {
28
- return ps.on('error', reject).on('exit', handleExit);
29
- }
30
- });
31
- };
7
+ function waitPidAsync(cmd, ps, status) {
8
+ return new Bluebird(function (resolve, reject) {
9
+ function handleExit(code, signal) {
10
+ if (code !== 0) {
11
+ const error = new Error(
12
+ cmd + ' exited. code: ' + code + ' signal: ' + signal,
13
+ );
14
+ error.code = code;
15
+ error.signal = signal;
16
+ return reject(error);
17
+ } else {
18
+ return resolve();
19
+ }
20
+ }
21
+ if (status.exited) {
22
+ return handleExit(status.code, status.signal);
23
+ } else if (status.error != null) {
24
+ return reject(status.error);
25
+ } else {
26
+ return ps.on('error', reject).on('exit', handleExit);
27
+ }
28
+ });
29
+ }
32
30
 
33
- exports.spawn = function(cmd, args, opts) {
34
- var processStatus, ps;
35
- ps = spawn(cmd, args, opts);
36
- processStatus = {
37
- exited: false,
38
- code: null,
39
- signal: null,
40
- error: null
41
- };
42
- ps.on('error', function(e) {
43
- return processStatus.error = e;
44
- });
45
- ps.on('exit', function(code, signal) {
46
- processStatus.exited = true;
47
- processStatus.code = code;
48
- return processStatus.signal = signal;
49
- });
50
- ps.waitAsync = function() {
51
- return waitPidAsync(cmd, ps, processStatus);
52
- };
53
- return ps;
54
- };
31
+ exports.spawn = function (cmd, args, opts) {
32
+ const ps = spawn(cmd, args, opts);
33
+ const processStatus = {
34
+ exited: false,
35
+ code: null,
36
+ signal: null,
37
+ error: null,
38
+ };
39
+ ps.on('error', function (e) {
40
+ processStatus.error = e;
41
+ });
42
+ ps.on('exit', function (code, signal) {
43
+ processStatus.exited = true;
44
+ processStatus.code = code;
45
+ processStatus.signal = signal;
46
+ });
47
+ ps.waitAsync = function () {
48
+ return waitPidAsync(cmd, ps, processStatus);
49
+ };
50
+ return ps;
51
+ };
55
52
 
56
- exports.mkfifoSync = function(fifoPath) {
57
- return execSync("mkfifo -m 0600 " + fifoPath);
58
- };
59
-
60
- }).call(this);
53
+ exports.mkfifoSync = function (fifoPath) {
54
+ return execSync('mkfifo -m 0600 ' + fifoPath);
55
+ };
package/package.json CHANGED
@@ -1,30 +1,26 @@
1
1
  {
2
2
  "name": "docker-delta",
3
- "version": "2.2.12",
3
+ "version": "3.0.0",
4
4
  "description": "Generate binary filesystem diffs between docker images to speed up distribution",
5
5
  "main": "lib/docker-delta",
6
6
  "scripts": {
7
- "prepublishOnly": "npm run build",
8
- "build": "coffee -c lib",
9
- "lint": "resin-lint lib",
7
+ "prettify": "balena-lint --fix -e js -e ts lib/ test/",
8
+ "lint": "balena-lint -e js -e ts lib/ test/",
10
9
  "pretest": "npm run lint",
11
10
  "test": "bash ./test.sh",
12
11
  "versionist": "versionist"
13
12
  },
14
- "author": "Petros Angelatos",
13
+ "author": "Balena Ltd.",
15
14
  "license": "Apache-2.0",
16
15
  "devDependencies": {
17
- "JSONStream": "^1.3.5",
16
+ "@balena/lint": "^6.2.2",
18
17
  "chai-events": "0.0.1",
19
18
  "chai-stream": "0.0.0",
20
19
  "coffee-script": "^1.10.0",
21
20
  "event-stream": "3.3.5",
22
- "gulp": "^3.9.0",
23
- "gulp-coffee": "^2.2.0",
24
- "gulp-util": "^3.0.6",
21
+ "JSONStream": "^1.3.5",
25
22
  "mocha": "^3.4.2",
26
23
  "mochainon": "^2.0.0",
27
- "resin-lint": "^1.2.2",
28
24
  "versionist": "^2.7.0"
29
25
  },
30
26
  "dependencies": {
@@ -36,6 +32,6 @@
36
32
  "typed-error": "^2.0.0"
37
33
  },
38
34
  "versionist": {
39
- "publishedAt": "2023-06-06T14:05:16.179Z"
35
+ "publishedAt": "2023-06-06T16:35:43.646Z"
40
36
  }
41
37
  }
@@ -0,0 +1,88 @@
1
+ 'use strict';
2
+
3
+ const m = require('mochainon');
4
+
5
+ m.chai.use(require('chai-events'));
6
+ m.chai.use(require('chai-stream'));
7
+
8
+ const es = require('event-stream');
9
+ const JSONStream = require('JSONStream');
10
+ const Bluebird = require('bluebird');
11
+ const stream = require('stream');
12
+ const Docker = require('dockerode');
13
+
14
+ const docker = new Docker();
15
+
16
+ const dockerDelta = require('../lib/docker-delta');
17
+
18
+ const expect = m.chai.expect;
19
+
20
+ function buildImg(name, dockerfile) {
21
+ return docker
22
+ .buildImage(
23
+ {
24
+ context: './test',
25
+ src: [dockerfile],
26
+ },
27
+ {
28
+ t: name,
29
+ dockerfile: dockerfile,
30
+ },
31
+ )
32
+ .then(function (res) {
33
+ return new Bluebird(function (resolve, reject) {
34
+ var outputStream;
35
+ return (outputStream = res
36
+ .pipe(JSONStream.parse())
37
+ .pipe(
38
+ es.through(function (data) {
39
+ if (data.error) {
40
+ return reject(data.error);
41
+ } else {
42
+ return console.log(data);
43
+ }
44
+ }),
45
+ )
46
+ .on('end', resolve)
47
+ .on('error', reject));
48
+ });
49
+ });
50
+ }
51
+
52
+ describe('docker-delta', function () {
53
+ this.timeout(120000);
54
+
55
+ before(function () {
56
+ return buildImg('source-image', 'Dockerfile.src').then(function () {
57
+ return buildImg('dest-image', 'Dockerfile.dst');
58
+ });
59
+ });
60
+
61
+ it('creates a delta between two images and applies it', function () {
62
+ const deltaStream = dockerDelta.createDelta(
63
+ 'source-image',
64
+ 'dest-image',
65
+ true,
66
+ );
67
+ expect(deltaStream).to.be.a.Stream;
68
+ const str = deltaStream.pipe(dockerDelta.applyDelta('source-image')).on(
69
+ 'id',
70
+ (function (_this) {
71
+ return function (id) {
72
+ return (_this.imageId = id);
73
+ };
74
+ })(this),
75
+ );
76
+ return expect(str).to.emit('id');
77
+ });
78
+
79
+ it('produces a valid docker image', function () {
80
+ expect(this.imageId).to.be.a.string;
81
+ const pt = new stream.PassThrough();
82
+ pt.setEncoding('utf8');
83
+ return docker.run(this.imageId, void 0, pt).then(function (container) {
84
+ expect(container.output.StatusCode).to.equal(0);
85
+ return expect(pt.read()).to.equal('HeXXo from the image\r\n');
86
+ });
87
+ });
88
+ });
package/test.sh CHANGED
@@ -13,6 +13,6 @@ docker run --privileged --name deltatest -d docker-delta-test
13
13
  # give docker daemon a few moments to finish starting up...
14
14
  sleep 5
15
15
 
16
- docker exec deltatest /bin/sh -c "npm run lint && ./node_modules/.bin/mocha --compilers coffee:coffee-script/register"
16
+ docker exec deltatest /bin/sh -c "npm run lint && ./node_modules/.bin/mocha"
17
17
 
18
18
  cleanup
package/lib/btrfs.coffee DELETED
@@ -1,11 +0,0 @@
1
- { spawn } = require './utils'
2
-
3
- # Returns a promise that settles after the subvolume `subvol` has been deleted
4
- exports.deleteSubvolAsync = (subvol) ->
5
- spawn('btrfs', [ 'subvolume', 'delete', subvol ]).waitAsync()
6
-
7
- # Returns a promise that settles after a snapshot of the subvolume `src` has
8
- # been created at `dest`
9
- exports.snapshotSubvolAsync = (src, dest) ->
10
- # console.log('btrfs subvolume snapshot', src, dest)
11
- spawn('btrfs', [ 'subvolume', 'snapshot', src, dest ]).waitAsync()
@@ -1,261 +0,0 @@
1
- path = require 'path'
2
-
3
- Promise = require 'bluebird'
4
- stream = require 'readable-stream'
5
- TypedError = require 'typed-error'
6
-
7
- rsync = require './rsync'
8
- btrfs = require './btrfs'
9
- { spawn } = require './utils'
10
- Docker = require 'docker-toolbelt'
11
-
12
- docker = new Docker()
13
-
14
- # code 19 isn't found in rsync's man pages.
15
- DELTA_OUT_OF_SYNC_CODES = [ 19, 23, 24 ]
16
-
17
- # We set a large timeout (10 minutes) for rsync to exit after applying the batch.
18
- # For most cases 5 seconds is more than enough, but in systems with slow IO
19
- # it can be a bit longer.
20
- RSYNC_EXIT_TIMEOUT = 10 * 60 * 1000
21
-
22
- exports.OutOfSyncError = class OutOfSyncError extends TypedError
23
-
24
- # Takes two strings `srcImage` and `destImage` which represent docker images
25
- # that are already present in the docker daemon and returns a Readable stream
26
- # of the binary diff between the two images.
27
- #
28
- # The stream format is the following where || means concatenation:
29
- #
30
- # result := jsonMetadata || 0x00 || rsyncData
31
- exports.createDelta = (srcImage, destImage, v2 = true, { log, ioTimeout = 0 } = {}) ->
32
- log ?= -> # noop
33
-
34
- # We need a passthrough stream so that we can return it immediately while the
35
- # promises are in progress
36
- deltaStream = new stream.PassThrough()
37
-
38
- # We retrieve the container config for the image
39
- config = docker.getImage(destImage).inspect().get('Config')
40
-
41
- # We then get the two root directories and then apply rsync on them
42
- rootDirFunc = docker.imageRootDirMounted.bind(docker)
43
- rootDirDisposers = [ srcImage, destImage ].map(rootDirFunc)
44
- Promise.using rootDirDisposers, (rootDirs) ->
45
- [ srcDir, destDir ] = rootDirs.map (rootDir) -> path.join(rootDir, path.sep)
46
- rsyncPromise = rsync.createRsyncStream(srcDir, destDir, ioTimeout, log)
47
-
48
- Promise.join config, rsyncPromise, (config, [ rsyncExit, rsyncStream ]) ->
49
- if v2
50
- metadata =
51
- version: 2
52
- dockerConfig: config
53
-
54
- # Write the header of the delta format which is the serialised metadata
55
- deltaStream.write(JSON.stringify(metadata))
56
- # Write the NUL byte separator for the rsync binary stream
57
- deltaStream.write(new Buffer([ 0x00 ]))
58
- # Write the rsync binary stream
59
- new Promise (resolve, reject) ->
60
- rsyncStream
61
- .on 'error', reject
62
- .on 'close', resolve
63
- .pipe(deltaStream)
64
- .finally ->
65
- rsyncExit.waitAsync()
66
- .catch (e) ->
67
- deltaStream.emit('error', e)
68
- .finally ->
69
- log('rsync exited')
70
- deltaStream.emit('close')
71
-
72
- return deltaStream
73
-
74
- bufIndexOfByte = (buf, byte) ->
75
- for b, i in buf when b is byte
76
- return i
77
- return -1
78
-
79
- # Parses the input stream `input` and returns a promise that will resolve to
80
- # the parsed JSON metadata of the delta stream. The input stream is consumed
81
- # exactly up to and including the separator so that it can be piped directly
82
- # to rsync after the promise resolves.
83
- parseDeltaStream = (input) ->
84
- new Promise (resolve, reject) ->
85
- buf = new Buffer(0)
86
-
87
- parser = ->
88
- # Read all available data
89
- chunks = [ buf ]
90
- while chunk = input.read()
91
- chunks.push(chunk)
92
-
93
- # FIXME: Implement a sensible upper bound on the size of metadata
94
- # and reject with an error if we get above that
95
- buf = Buffer.concat(chunks)
96
-
97
- sep = bufIndexOfByte(buf, 0x00)
98
-
99
- if sep isnt -1
100
- # We no longer have to parse the input
101
- input.removeListener('readable', parser)
102
-
103
- # The data after the separator are rsync binary data so we put
104
- # them back for the next consumer to process
105
- input.unshift(buf[sep + 1...])
106
-
107
- # Parse JSON up until the separator
108
- try
109
- metadata = JSON.parse(buf[...sep])
110
- catch e
111
- return reject(e)
112
-
113
- # Sanity check
114
- if metadata.version is 2
115
- resolve(metadata)
116
- else
117
- reject(new Error('Uknown version: ' + metadata.version))
118
-
119
- input.on('readable', parser)
120
-
121
- nullDisposer = ->
122
- Promise.resolve(null)
123
-
124
- hardlinkCopy = (srcRoot, dstRoot, linkDests) ->
125
- rsyncArgs = [
126
- '--archive'
127
- '--delete'
128
- ]
129
- rsyncArgs.push('--link-dest', dest) for dest in linkDests
130
- rsyncArgs.push(srcRoot, dstRoot)
131
- rsync = spawn('rsync', rsyncArgs)
132
- rsync.waitAsync()
133
-
134
- applyBatch = (rsync, batch, timeout, log) ->
135
- p = new Promise (resolve, reject) ->
136
- batch
137
- .on('error', reject)
138
- .on 'finish', ->
139
- # There seems to be a race condition between
140
- # rsync exiting and it's stdin pipe being closed. If
141
- # the stdin is not closed, the delta application never
142
- # finishes. Due to this, once the batch has fully
143
- # drained, we force close stdin. We did have this
144
- # change in the rsync.on('close') handler, but that
145
- # only worked some of the time. Anecdotal evidence
146
- # says that this works all of the time, and it's also
147
- # technically correct, the best kind. Be careful
148
- # removing this, as it seems to be required post node8.
149
- rsync.stdin.end()
150
- resolve()
151
- .pipe(rsync.stdin)
152
- .on('error', reject)
153
- .on('finish', resolve)
154
-
155
- if timeout isnt 0
156
- p = p.timeout(timeout)
157
-
158
- return p.then ->
159
- log('Batch input stream ended; waiting for rsync...')
160
- # `p` is resolved on 'finish' but that doesn't guarantee rsync exited cleanly :/
161
- # normally we'd end up here on rsync exiting with 0, but there are cases
162
- # our pipe to its stdin is broken with no error and rsync still is up.
163
- # so to find the reason rsync exited, we "poll" it once here via `waitAsync`.
164
- # if rsync exited cleanly, `waitAsync` will resolve and the chain continues.
165
- # for any other case, a short timeout will ensure we don't wait forever and
166
- # manually kill rsync below.
167
- rsync.waitAsync().timeout(RSYNC_EXIT_TIMEOUT)
168
- .tap ->
169
- log('rsync exited cleanly')
170
- .tapCatch (err) ->
171
- log("Error waiting for rsync to exit: #{err}")
172
- .catch (err) ->
173
- log("Killing rsync with force due to error: #{err}")
174
- rsync.kill('SIGUSR1')
175
- log('Waiting for rsync to exit...')
176
- rsync.waitAsync().throw(err)
177
-
178
- exports.applyDelta = (srcImage, { timeout = 0, log } = {}) ->
179
- log ?= -> # noop
180
-
181
- deltaStream = new stream.PassThrough()
182
- rootDirFunc = nullDisposer
183
- if srcImage?
184
- rootDirFunc = docker.imageRootDirMounted.bind(docker)
185
-
186
- dstIdPromise = parseDeltaStream(deltaStream)
187
- .get('dockerConfig')
188
- .tap -> log('Extracted image config')
189
- .bind(docker)
190
- .then(docker.createEmptyImage)
191
- .tap (id) -> log("Created empty image #{id}")
192
-
193
- Promise.using rootDirFunc(srcImage), (srcRoot) ->
194
- srcRoot = path.join(srcRoot, '/') if srcRoot?
195
-
196
- Promise.join(
197
- docker.info().get('Driver')
198
- dstIdPromise
199
- dstIdPromise.then(docker.imageRootDir)
200
- (dockerDriver, dstId, dstRoot) ->
201
- # trailing slashes are significant for rsync
202
- dstRoot = path.join(dstRoot, '/')
203
-
204
- Promise.try ->
205
- switch dockerDriver
206
- when 'btrfs'
207
- if srcRoot?
208
- btrfs.deleteSubvolAsync(dstRoot)
209
- .then ->
210
- btrfs.snapshotSubvolAsync(srcRoot, dstRoot)
211
- when 'overlay'
212
- if srcRoot?
213
- hardlinkCopy(srcRoot, dstRoot, [ srcRoot ])
214
- when 'aufs', 'overlay2'
215
- if srcRoot?
216
- docker.diffPaths(srcImage)
217
- .then (diffPaths) ->
218
- hardlinkCopy(srcRoot, dstRoot, diffPaths)
219
- else
220
- throw new Error("Unsupported driver #{dockerDriver}")
221
- .then ->
222
- log("Hard-linked files from '#{srcRoot}' to '#{dstRoot}'")
223
-
224
- rsyncArgs = [
225
- '--archive'
226
- '--delete'
227
- '--read-batch', '-'
228
- dstRoot
229
- ]
230
- opts = {
231
- # pipe stdin, ignore stdout/stderr
232
- stdio: [ 'pipe', 'ignore', 'ignore' ]
233
- }
234
-
235
- log("Spawning rsync with arguments #{rsyncArgs.join(' ')}")
236
-
237
- rsync = spawn('rsync', rsyncArgs, opts)
238
- applyBatch(rsync, deltaStream, timeout, log)
239
- .tap ->
240
- log('rsync exited successfully')
241
- .then ->
242
- log('fsync\'ing...')
243
- # rsync doesn't fsync by itself
244
- spawn('sync').waitAsync()
245
- .then ->
246
- log("All done. Image ID: #{dstId}")
247
- deltaStream.emit('id', dstId)
248
- )
249
- .catch (e) ->
250
- log("Error: #{e}")
251
- if e?.code in DELTA_OUT_OF_SYNC_CODES
252
- deltaStream.emit('error', new OutOfSyncError('Incompatible image'))
253
- else
254
- deltaStream.emit('error', e)
255
- # If the process failed for whatever reason, cleanup the empty image
256
- dstIdPromise.then (dstId) ->
257
- docker.getImage(dstId).remove()
258
- .catch (e) ->
259
- deltaStream.emit('error', e)
260
-
261
- return deltaStream