docker-delta 2.2.12 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.versionbot/CHANGELOG.yml +12 -1
- package/CHANGELOG.md +5 -0
- package/Dockerfile.test +0 -1
- package/lib/btrfs.js +8 -12
- package/lib/docker-delta.js +306 -255
- package/lib/rsync.js +69 -54
- package/lib/utils.js +50 -55
- package/package.json +7 -11
- package/test/docker-delta.spec.js +88 -0
- package/test.sh +1 -1
- package/lib/btrfs.coffee +0 -11
- package/lib/docker-delta.coffee +0 -261
- package/lib/rsync.coffee +0 -56
- package/lib/utils.coffee +0 -43
- package/test/docker-delta.spec.coffee +0 -55
package/lib/rsync.js
CHANGED
|
@@ -1,56 +1,71 @@
|
|
|
1
|
-
|
|
2
|
-
(function() {
|
|
3
|
-
var Promise, fs, mkfifoSync, path, ref, spawn, tmp;
|
|
1
|
+
'use strict';
|
|
4
2
|
|
|
5
|
-
|
|
3
|
+
const Bluebird = require('bluebird');
|
|
4
|
+
const fs = require('fs');
|
|
5
|
+
const tmp = require('tmp');
|
|
6
|
+
const path = require('path');
|
|
7
|
+
const { spawn, mkfifoSync } = require('./utils');
|
|
6
8
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
})
|
|
9
|
+
exports.createRsyncStream = function (src, dest, ioTimeout, log) {
|
|
10
|
+
return new Bluebird(function (resolve, reject) {
|
|
11
|
+
return tmp.dir(
|
|
12
|
+
{
|
|
13
|
+
unsafeCleanup: true,
|
|
14
|
+
},
|
|
15
|
+
function (err, tmpDirPath, cleanup) {
|
|
16
|
+
if (err) {
|
|
17
|
+
log('Failed to create temporary directory');
|
|
18
|
+
return reject(err);
|
|
19
|
+
}
|
|
20
|
+
const pipePath = path.join(tmpDirPath, 'rsync.pipe');
|
|
21
|
+
mkfifoSync(pipePath);
|
|
22
|
+
const rsyncArgs = [
|
|
23
|
+
'--archive',
|
|
24
|
+
'--compress',
|
|
25
|
+
'--checksum',
|
|
26
|
+
'--no-i-r',
|
|
27
|
+
'--delete',
|
|
28
|
+
'--hard-links',
|
|
29
|
+
'--timeout',
|
|
30
|
+
ioTimeout,
|
|
31
|
+
'--compress-level=9',
|
|
32
|
+
'--one-file-system',
|
|
33
|
+
'--only-write-batch',
|
|
34
|
+
pipePath,
|
|
35
|
+
dest,
|
|
36
|
+
src,
|
|
37
|
+
];
|
|
38
|
+
log('Invoking rsync:', 'rsync ' + rsyncArgs.join(' '));
|
|
39
|
+
const ps = spawn('rsync', rsyncArgs, {
|
|
40
|
+
stdio: 'ignore',
|
|
41
|
+
});
|
|
42
|
+
fs.open(pipePath, 'r', function (error, fd) {
|
|
43
|
+
var stream;
|
|
44
|
+
if (error) {
|
|
45
|
+
log('Failed to open pipe for reading. Killing rsync...');
|
|
46
|
+
ps.kill('SIGUSR1');
|
|
47
|
+
return ps
|
|
48
|
+
.waitAsync()
|
|
49
|
+
.tap(function () {
|
|
50
|
+
return log('rsync exited');
|
|
51
|
+
})
|
|
52
|
+
.tapCatch(function (e) {
|
|
53
|
+
return log('rsync exited with error: ' + e);
|
|
54
|
+
})
|
|
55
|
+
['finally'](function () {
|
|
56
|
+
cleanup();
|
|
57
|
+
return reject(error);
|
|
58
|
+
});
|
|
59
|
+
} else {
|
|
60
|
+
stream = fs
|
|
61
|
+
.createReadStream(void 0, {
|
|
62
|
+
fd: fd,
|
|
63
|
+
})
|
|
64
|
+
.on('close', cleanup);
|
|
65
|
+
return resolve([ps, stream]);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
},
|
|
69
|
+
);
|
|
70
|
+
});
|
|
71
|
+
};
|
package/lib/utils.js
CHANGED
|
@@ -1,60 +1,55 @@
|
|
|
1
|
-
|
|
2
|
-
(function() {
|
|
3
|
-
var Promise, execSync, ref, spawn, waitPidAsync;
|
|
1
|
+
'use strict';
|
|
4
2
|
|
|
5
|
-
|
|
3
|
+
const { spawn, execSync } = require('child_process');
|
|
6
4
|
|
|
7
|
-
|
|
5
|
+
const Bluebird = require('bluebird');
|
|
8
6
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
7
|
+
function waitPidAsync(cmd, ps, status) {
|
|
8
|
+
return new Bluebird(function (resolve, reject) {
|
|
9
|
+
function handleExit(code, signal) {
|
|
10
|
+
if (code !== 0) {
|
|
11
|
+
const error = new Error(
|
|
12
|
+
cmd + ' exited. code: ' + code + ' signal: ' + signal,
|
|
13
|
+
);
|
|
14
|
+
error.code = code;
|
|
15
|
+
error.signal = signal;
|
|
16
|
+
return reject(error);
|
|
17
|
+
} else {
|
|
18
|
+
return resolve();
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
if (status.exited) {
|
|
22
|
+
return handleExit(status.code, status.signal);
|
|
23
|
+
} else if (status.error != null) {
|
|
24
|
+
return reject(status.error);
|
|
25
|
+
} else {
|
|
26
|
+
return ps.on('error', reject).on('exit', handleExit);
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
}
|
|
32
30
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
};
|
|
31
|
+
exports.spawn = function (cmd, args, opts) {
|
|
32
|
+
const ps = spawn(cmd, args, opts);
|
|
33
|
+
const processStatus = {
|
|
34
|
+
exited: false,
|
|
35
|
+
code: null,
|
|
36
|
+
signal: null,
|
|
37
|
+
error: null,
|
|
38
|
+
};
|
|
39
|
+
ps.on('error', function (e) {
|
|
40
|
+
processStatus.error = e;
|
|
41
|
+
});
|
|
42
|
+
ps.on('exit', function (code, signal) {
|
|
43
|
+
processStatus.exited = true;
|
|
44
|
+
processStatus.code = code;
|
|
45
|
+
processStatus.signal = signal;
|
|
46
|
+
});
|
|
47
|
+
ps.waitAsync = function () {
|
|
48
|
+
return waitPidAsync(cmd, ps, processStatus);
|
|
49
|
+
};
|
|
50
|
+
return ps;
|
|
51
|
+
};
|
|
55
52
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
}).call(this);
|
|
53
|
+
exports.mkfifoSync = function (fifoPath) {
|
|
54
|
+
return execSync('mkfifo -m 0600 ' + fifoPath);
|
|
55
|
+
};
|
package/package.json
CHANGED
|
@@ -1,30 +1,26 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "docker-delta",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.0",
|
|
4
4
|
"description": "Generate binary filesystem diffs between docker images to speed up distribution",
|
|
5
5
|
"main": "lib/docker-delta",
|
|
6
6
|
"scripts": {
|
|
7
|
-
"
|
|
8
|
-
"
|
|
9
|
-
"lint": "resin-lint lib",
|
|
7
|
+
"prettify": "balena-lint --fix -e js -e ts lib/ test/",
|
|
8
|
+
"lint": "balena-lint -e js -e ts lib/ test/",
|
|
10
9
|
"pretest": "npm run lint",
|
|
11
10
|
"test": "bash ./test.sh",
|
|
12
11
|
"versionist": "versionist"
|
|
13
12
|
},
|
|
14
|
-
"author": "
|
|
13
|
+
"author": "Balena Ltd.",
|
|
15
14
|
"license": "Apache-2.0",
|
|
16
15
|
"devDependencies": {
|
|
17
|
-
"
|
|
16
|
+
"@balena/lint": "^6.2.2",
|
|
18
17
|
"chai-events": "0.0.1",
|
|
19
18
|
"chai-stream": "0.0.0",
|
|
20
19
|
"coffee-script": "^1.10.0",
|
|
21
20
|
"event-stream": "3.3.5",
|
|
22
|
-
"
|
|
23
|
-
"gulp-coffee": "^2.2.0",
|
|
24
|
-
"gulp-util": "^3.0.6",
|
|
21
|
+
"JSONStream": "^1.3.5",
|
|
25
22
|
"mocha": "^3.4.2",
|
|
26
23
|
"mochainon": "^2.0.0",
|
|
27
|
-
"resin-lint": "^1.2.2",
|
|
28
24
|
"versionist": "^2.7.0"
|
|
29
25
|
},
|
|
30
26
|
"dependencies": {
|
|
@@ -36,6 +32,6 @@
|
|
|
36
32
|
"typed-error": "^2.0.0"
|
|
37
33
|
},
|
|
38
34
|
"versionist": {
|
|
39
|
-
"publishedAt": "2023-06-
|
|
35
|
+
"publishedAt": "2023-06-06T16:35:43.646Z"
|
|
40
36
|
}
|
|
41
37
|
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const m = require('mochainon');
|
|
4
|
+
|
|
5
|
+
m.chai.use(require('chai-events'));
|
|
6
|
+
m.chai.use(require('chai-stream'));
|
|
7
|
+
|
|
8
|
+
const es = require('event-stream');
|
|
9
|
+
const JSONStream = require('JSONStream');
|
|
10
|
+
const Bluebird = require('bluebird');
|
|
11
|
+
const stream = require('stream');
|
|
12
|
+
const Docker = require('dockerode');
|
|
13
|
+
|
|
14
|
+
const docker = new Docker();
|
|
15
|
+
|
|
16
|
+
const dockerDelta = require('../lib/docker-delta');
|
|
17
|
+
|
|
18
|
+
const expect = m.chai.expect;
|
|
19
|
+
|
|
20
|
+
function buildImg(name, dockerfile) {
|
|
21
|
+
return docker
|
|
22
|
+
.buildImage(
|
|
23
|
+
{
|
|
24
|
+
context: './test',
|
|
25
|
+
src: [dockerfile],
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
t: name,
|
|
29
|
+
dockerfile: dockerfile,
|
|
30
|
+
},
|
|
31
|
+
)
|
|
32
|
+
.then(function (res) {
|
|
33
|
+
return new Bluebird(function (resolve, reject) {
|
|
34
|
+
var outputStream;
|
|
35
|
+
return (outputStream = res
|
|
36
|
+
.pipe(JSONStream.parse())
|
|
37
|
+
.pipe(
|
|
38
|
+
es.through(function (data) {
|
|
39
|
+
if (data.error) {
|
|
40
|
+
return reject(data.error);
|
|
41
|
+
} else {
|
|
42
|
+
return console.log(data);
|
|
43
|
+
}
|
|
44
|
+
}),
|
|
45
|
+
)
|
|
46
|
+
.on('end', resolve)
|
|
47
|
+
.on('error', reject));
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
describe('docker-delta', function () {
|
|
53
|
+
this.timeout(120000);
|
|
54
|
+
|
|
55
|
+
before(function () {
|
|
56
|
+
return buildImg('source-image', 'Dockerfile.src').then(function () {
|
|
57
|
+
return buildImg('dest-image', 'Dockerfile.dst');
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it('creates a delta between two images and applies it', function () {
|
|
62
|
+
const deltaStream = dockerDelta.createDelta(
|
|
63
|
+
'source-image',
|
|
64
|
+
'dest-image',
|
|
65
|
+
true,
|
|
66
|
+
);
|
|
67
|
+
expect(deltaStream).to.be.a.Stream;
|
|
68
|
+
const str = deltaStream.pipe(dockerDelta.applyDelta('source-image')).on(
|
|
69
|
+
'id',
|
|
70
|
+
(function (_this) {
|
|
71
|
+
return function (id) {
|
|
72
|
+
return (_this.imageId = id);
|
|
73
|
+
};
|
|
74
|
+
})(this),
|
|
75
|
+
);
|
|
76
|
+
return expect(str).to.emit('id');
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
it('produces a valid docker image', function () {
|
|
80
|
+
expect(this.imageId).to.be.a.string;
|
|
81
|
+
const pt = new stream.PassThrough();
|
|
82
|
+
pt.setEncoding('utf8');
|
|
83
|
+
return docker.run(this.imageId, void 0, pt).then(function (container) {
|
|
84
|
+
expect(container.output.StatusCode).to.equal(0);
|
|
85
|
+
return expect(pt.read()).to.equal('HeXXo from the image\r\n');
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
});
|
package/test.sh
CHANGED
|
@@ -13,6 +13,6 @@ docker run --privileged --name deltatest -d docker-delta-test
|
|
|
13
13
|
# give docker daemon a few moments to finish starting up...
|
|
14
14
|
sleep 5
|
|
15
15
|
|
|
16
|
-
docker exec deltatest /bin/sh -c "npm run lint && ./node_modules/.bin/mocha
|
|
16
|
+
docker exec deltatest /bin/sh -c "npm run lint && ./node_modules/.bin/mocha"
|
|
17
17
|
|
|
18
18
|
cleanup
|
package/lib/btrfs.coffee
DELETED
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
{ spawn } = require './utils'
|
|
2
|
-
|
|
3
|
-
# Returns a promise that settles after the subvolume `subvol` has been deleted
|
|
4
|
-
exports.deleteSubvolAsync = (subvol) ->
|
|
5
|
-
spawn('btrfs', [ 'subvolume', 'delete', subvol ]).waitAsync()
|
|
6
|
-
|
|
7
|
-
# Returns a promise that settles after a snapshot of the subvolume `src` has
|
|
8
|
-
# been created at `dest`
|
|
9
|
-
exports.snapshotSubvolAsync = (src, dest) ->
|
|
10
|
-
# console.log('btrfs subvolume snapshot', src, dest)
|
|
11
|
-
spawn('btrfs', [ 'subvolume', 'snapshot', src, dest ]).waitAsync()
|
package/lib/docker-delta.coffee
DELETED
|
@@ -1,261 +0,0 @@
|
|
|
1
|
-
path = require 'path'
|
|
2
|
-
|
|
3
|
-
Promise = require 'bluebird'
|
|
4
|
-
stream = require 'readable-stream'
|
|
5
|
-
TypedError = require 'typed-error'
|
|
6
|
-
|
|
7
|
-
rsync = require './rsync'
|
|
8
|
-
btrfs = require './btrfs'
|
|
9
|
-
{ spawn } = require './utils'
|
|
10
|
-
Docker = require 'docker-toolbelt'
|
|
11
|
-
|
|
12
|
-
docker = new Docker()
|
|
13
|
-
|
|
14
|
-
# code 19 isn't found in rsync's man pages.
|
|
15
|
-
DELTA_OUT_OF_SYNC_CODES = [ 19, 23, 24 ]
|
|
16
|
-
|
|
17
|
-
# We set a large timeout (10 minutes) for rsync to exit after applying the batch.
|
|
18
|
-
# For most cases 5 seconds is more than enough, but in systems with slow IO
|
|
19
|
-
# it can be a bit longer.
|
|
20
|
-
RSYNC_EXIT_TIMEOUT = 10 * 60 * 1000
|
|
21
|
-
|
|
22
|
-
exports.OutOfSyncError = class OutOfSyncError extends TypedError
|
|
23
|
-
|
|
24
|
-
# Takes two strings `srcImage` and `destImage` which represent docker images
|
|
25
|
-
# that are already present in the docker daemon and returns a Readable stream
|
|
26
|
-
# of the binary diff between the two images.
|
|
27
|
-
#
|
|
28
|
-
# The stream format is the following where || means concatenation:
|
|
29
|
-
#
|
|
30
|
-
# result := jsonMetadata || 0x00 || rsyncData
|
|
31
|
-
exports.createDelta = (srcImage, destImage, v2 = true, { log, ioTimeout = 0 } = {}) ->
|
|
32
|
-
log ?= -> # noop
|
|
33
|
-
|
|
34
|
-
# We need a passthrough stream so that we can return it immediately while the
|
|
35
|
-
# promises are in progress
|
|
36
|
-
deltaStream = new stream.PassThrough()
|
|
37
|
-
|
|
38
|
-
# We retrieve the container config for the image
|
|
39
|
-
config = docker.getImage(destImage).inspect().get('Config')
|
|
40
|
-
|
|
41
|
-
# We then get the two root directories and then apply rsync on them
|
|
42
|
-
rootDirFunc = docker.imageRootDirMounted.bind(docker)
|
|
43
|
-
rootDirDisposers = [ srcImage, destImage ].map(rootDirFunc)
|
|
44
|
-
Promise.using rootDirDisposers, (rootDirs) ->
|
|
45
|
-
[ srcDir, destDir ] = rootDirs.map (rootDir) -> path.join(rootDir, path.sep)
|
|
46
|
-
rsyncPromise = rsync.createRsyncStream(srcDir, destDir, ioTimeout, log)
|
|
47
|
-
|
|
48
|
-
Promise.join config, rsyncPromise, (config, [ rsyncExit, rsyncStream ]) ->
|
|
49
|
-
if v2
|
|
50
|
-
metadata =
|
|
51
|
-
version: 2
|
|
52
|
-
dockerConfig: config
|
|
53
|
-
|
|
54
|
-
# Write the header of the delta format which is the serialised metadata
|
|
55
|
-
deltaStream.write(JSON.stringify(metadata))
|
|
56
|
-
# Write the NUL byte separator for the rsync binary stream
|
|
57
|
-
deltaStream.write(new Buffer([ 0x00 ]))
|
|
58
|
-
# Write the rsync binary stream
|
|
59
|
-
new Promise (resolve, reject) ->
|
|
60
|
-
rsyncStream
|
|
61
|
-
.on 'error', reject
|
|
62
|
-
.on 'close', resolve
|
|
63
|
-
.pipe(deltaStream)
|
|
64
|
-
.finally ->
|
|
65
|
-
rsyncExit.waitAsync()
|
|
66
|
-
.catch (e) ->
|
|
67
|
-
deltaStream.emit('error', e)
|
|
68
|
-
.finally ->
|
|
69
|
-
log('rsync exited')
|
|
70
|
-
deltaStream.emit('close')
|
|
71
|
-
|
|
72
|
-
return deltaStream
|
|
73
|
-
|
|
74
|
-
bufIndexOfByte = (buf, byte) ->
|
|
75
|
-
for b, i in buf when b is byte
|
|
76
|
-
return i
|
|
77
|
-
return -1
|
|
78
|
-
|
|
79
|
-
# Parses the input stream `input` and returns a promise that will resolve to
|
|
80
|
-
# the parsed JSON metadata of the delta stream. The input stream is consumed
|
|
81
|
-
# exactly up to and including the separator so that it can be piped directly
|
|
82
|
-
# to rsync after the promise resolves.
|
|
83
|
-
parseDeltaStream = (input) ->
|
|
84
|
-
new Promise (resolve, reject) ->
|
|
85
|
-
buf = new Buffer(0)
|
|
86
|
-
|
|
87
|
-
parser = ->
|
|
88
|
-
# Read all available data
|
|
89
|
-
chunks = [ buf ]
|
|
90
|
-
while chunk = input.read()
|
|
91
|
-
chunks.push(chunk)
|
|
92
|
-
|
|
93
|
-
# FIXME: Implement a sensible upper bound on the size of metadata
|
|
94
|
-
# and reject with an error if we get above that
|
|
95
|
-
buf = Buffer.concat(chunks)
|
|
96
|
-
|
|
97
|
-
sep = bufIndexOfByte(buf, 0x00)
|
|
98
|
-
|
|
99
|
-
if sep isnt -1
|
|
100
|
-
# We no longer have to parse the input
|
|
101
|
-
input.removeListener('readable', parser)
|
|
102
|
-
|
|
103
|
-
# The data after the separator are rsync binary data so we put
|
|
104
|
-
# them back for the next consumer to process
|
|
105
|
-
input.unshift(buf[sep + 1...])
|
|
106
|
-
|
|
107
|
-
# Parse JSON up until the separator
|
|
108
|
-
try
|
|
109
|
-
metadata = JSON.parse(buf[...sep])
|
|
110
|
-
catch e
|
|
111
|
-
return reject(e)
|
|
112
|
-
|
|
113
|
-
# Sanity check
|
|
114
|
-
if metadata.version is 2
|
|
115
|
-
resolve(metadata)
|
|
116
|
-
else
|
|
117
|
-
reject(new Error('Uknown version: ' + metadata.version))
|
|
118
|
-
|
|
119
|
-
input.on('readable', parser)
|
|
120
|
-
|
|
121
|
-
nullDisposer = ->
|
|
122
|
-
Promise.resolve(null)
|
|
123
|
-
|
|
124
|
-
hardlinkCopy = (srcRoot, dstRoot, linkDests) ->
|
|
125
|
-
rsyncArgs = [
|
|
126
|
-
'--archive'
|
|
127
|
-
'--delete'
|
|
128
|
-
]
|
|
129
|
-
rsyncArgs.push('--link-dest', dest) for dest in linkDests
|
|
130
|
-
rsyncArgs.push(srcRoot, dstRoot)
|
|
131
|
-
rsync = spawn('rsync', rsyncArgs)
|
|
132
|
-
rsync.waitAsync()
|
|
133
|
-
|
|
134
|
-
applyBatch = (rsync, batch, timeout, log) ->
|
|
135
|
-
p = new Promise (resolve, reject) ->
|
|
136
|
-
batch
|
|
137
|
-
.on('error', reject)
|
|
138
|
-
.on 'finish', ->
|
|
139
|
-
# There seems to be a race condition between
|
|
140
|
-
# rsync exiting and it's stdin pipe being closed. If
|
|
141
|
-
# the stdin is not closed, the delta application never
|
|
142
|
-
# finishes. Due to this, once the batch has fully
|
|
143
|
-
# drained, we force close stdin. We did have this
|
|
144
|
-
# change in the rsync.on('close') handler, but that
|
|
145
|
-
# only worked some of the time. Anecdotal evidence
|
|
146
|
-
# says that this works all of the time, and it's also
|
|
147
|
-
# technically correct, the best kind. Be careful
|
|
148
|
-
# removing this, as it seems to be required post node8.
|
|
149
|
-
rsync.stdin.end()
|
|
150
|
-
resolve()
|
|
151
|
-
.pipe(rsync.stdin)
|
|
152
|
-
.on('error', reject)
|
|
153
|
-
.on('finish', resolve)
|
|
154
|
-
|
|
155
|
-
if timeout isnt 0
|
|
156
|
-
p = p.timeout(timeout)
|
|
157
|
-
|
|
158
|
-
return p.then ->
|
|
159
|
-
log('Batch input stream ended; waiting for rsync...')
|
|
160
|
-
# `p` is resolved on 'finish' but that doesn't guarantee rsync exited cleanly :/
|
|
161
|
-
# normally we'd end up here on rsync exiting with 0, but there are cases
|
|
162
|
-
# our pipe to its stdin is broken with no error and rsync still is up.
|
|
163
|
-
# so to find the reason rsync exited, we "poll" it once here via `waitAsync`.
|
|
164
|
-
# if rsync exited cleanly, `waitAsync` will resolve and the chain continues.
|
|
165
|
-
# for any other case, a short timeout will ensure we don't wait forever and
|
|
166
|
-
# manually kill rsync below.
|
|
167
|
-
rsync.waitAsync().timeout(RSYNC_EXIT_TIMEOUT)
|
|
168
|
-
.tap ->
|
|
169
|
-
log('rsync exited cleanly')
|
|
170
|
-
.tapCatch (err) ->
|
|
171
|
-
log("Error waiting for rsync to exit: #{err}")
|
|
172
|
-
.catch (err) ->
|
|
173
|
-
log("Killing rsync with force due to error: #{err}")
|
|
174
|
-
rsync.kill('SIGUSR1')
|
|
175
|
-
log('Waiting for rsync to exit...')
|
|
176
|
-
rsync.waitAsync().throw(err)
|
|
177
|
-
|
|
178
|
-
exports.applyDelta = (srcImage, { timeout = 0, log } = {}) ->
|
|
179
|
-
log ?= -> # noop
|
|
180
|
-
|
|
181
|
-
deltaStream = new stream.PassThrough()
|
|
182
|
-
rootDirFunc = nullDisposer
|
|
183
|
-
if srcImage?
|
|
184
|
-
rootDirFunc = docker.imageRootDirMounted.bind(docker)
|
|
185
|
-
|
|
186
|
-
dstIdPromise = parseDeltaStream(deltaStream)
|
|
187
|
-
.get('dockerConfig')
|
|
188
|
-
.tap -> log('Extracted image config')
|
|
189
|
-
.bind(docker)
|
|
190
|
-
.then(docker.createEmptyImage)
|
|
191
|
-
.tap (id) -> log("Created empty image #{id}")
|
|
192
|
-
|
|
193
|
-
Promise.using rootDirFunc(srcImage), (srcRoot) ->
|
|
194
|
-
srcRoot = path.join(srcRoot, '/') if srcRoot?
|
|
195
|
-
|
|
196
|
-
Promise.join(
|
|
197
|
-
docker.info().get('Driver')
|
|
198
|
-
dstIdPromise
|
|
199
|
-
dstIdPromise.then(docker.imageRootDir)
|
|
200
|
-
(dockerDriver, dstId, dstRoot) ->
|
|
201
|
-
# trailing slashes are significant for rsync
|
|
202
|
-
dstRoot = path.join(dstRoot, '/')
|
|
203
|
-
|
|
204
|
-
Promise.try ->
|
|
205
|
-
switch dockerDriver
|
|
206
|
-
when 'btrfs'
|
|
207
|
-
if srcRoot?
|
|
208
|
-
btrfs.deleteSubvolAsync(dstRoot)
|
|
209
|
-
.then ->
|
|
210
|
-
btrfs.snapshotSubvolAsync(srcRoot, dstRoot)
|
|
211
|
-
when 'overlay'
|
|
212
|
-
if srcRoot?
|
|
213
|
-
hardlinkCopy(srcRoot, dstRoot, [ srcRoot ])
|
|
214
|
-
when 'aufs', 'overlay2'
|
|
215
|
-
if srcRoot?
|
|
216
|
-
docker.diffPaths(srcImage)
|
|
217
|
-
.then (diffPaths) ->
|
|
218
|
-
hardlinkCopy(srcRoot, dstRoot, diffPaths)
|
|
219
|
-
else
|
|
220
|
-
throw new Error("Unsupported driver #{dockerDriver}")
|
|
221
|
-
.then ->
|
|
222
|
-
log("Hard-linked files from '#{srcRoot}' to '#{dstRoot}'")
|
|
223
|
-
|
|
224
|
-
rsyncArgs = [
|
|
225
|
-
'--archive'
|
|
226
|
-
'--delete'
|
|
227
|
-
'--read-batch', '-'
|
|
228
|
-
dstRoot
|
|
229
|
-
]
|
|
230
|
-
opts = {
|
|
231
|
-
# pipe stdin, ignore stdout/stderr
|
|
232
|
-
stdio: [ 'pipe', 'ignore', 'ignore' ]
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
log("Spawning rsync with arguments #{rsyncArgs.join(' ')}")
|
|
236
|
-
|
|
237
|
-
rsync = spawn('rsync', rsyncArgs, opts)
|
|
238
|
-
applyBatch(rsync, deltaStream, timeout, log)
|
|
239
|
-
.tap ->
|
|
240
|
-
log('rsync exited successfully')
|
|
241
|
-
.then ->
|
|
242
|
-
log('fsync\'ing...')
|
|
243
|
-
# rsync doesn't fsync by itself
|
|
244
|
-
spawn('sync').waitAsync()
|
|
245
|
-
.then ->
|
|
246
|
-
log("All done. Image ID: #{dstId}")
|
|
247
|
-
deltaStream.emit('id', dstId)
|
|
248
|
-
)
|
|
249
|
-
.catch (e) ->
|
|
250
|
-
log("Error: #{e}")
|
|
251
|
-
if e?.code in DELTA_OUT_OF_SYNC_CODES
|
|
252
|
-
deltaStream.emit('error', new OutOfSyncError('Incompatible image'))
|
|
253
|
-
else
|
|
254
|
-
deltaStream.emit('error', e)
|
|
255
|
-
# If the process failed for whatever reason, cleanup the empty image
|
|
256
|
-
dstIdPromise.then (dstId) ->
|
|
257
|
-
docker.getImage(dstId).remove()
|
|
258
|
-
.catch (e) ->
|
|
259
|
-
deltaStream.emit('error', e)
|
|
260
|
-
|
|
261
|
-
return deltaStream
|