docker-storage-gc 3.5.12 → 4.0.0

package/test/index.ts

@@ -0,0 +1,228 @@
+import type Docker from 'dockerode';
+import Bluebird from 'bluebird';
+import { expect } from 'chai';
+import DockerGC from '../build/index';
+import { getDocker } from '../build/docker';
+
+const SKIP_GC_TEST = process.env.SKIP_GC_TEST === '1' || false;
+const IMAGES = ['alpine:3.1', 'debian:squeeze', 'ubuntu:lucid'];
+
+// TODO: Move it to a proper repo
+// Same image (same id), different repo, different digest
+const NONE_TAG_IMAGES = [
+	'hello-world@sha256:8e3114318a995a1ee497790535e7b88365222a21771ae7e53687ad76563e8e76',
+	'balenaplayground/hello-world@sha256:90659bf80b44ce6be8234e6ff90a1ac34acbeb826903b02cfa0da11c82cbc042',
+];
+
+const promiseToBool = (p: Promise<unknown>): Promise<boolean> =>
+	p.then(() => true).catch(() => false);
+
+const pullAsync = function (docker: Docker, tag: string) {
+	console.log(`[TEST] Pulling ${tag}`);
+	return docker.pull(tag).then(
+		(stream) =>
+			new Promise(function (resolve, reject) {
+				stream.resume();
+				stream.once('error', reject);
+				stream.once('end', resolve);
+			}),
+	);
+};
+
+// This test case is a little weird, it requires that no other images are present on
+// the system to ensure that the correct one is being removed. Because of this, you
+// can use the SKIP_GC_TEST env var to inform the test suite not to run this test
+describe('Garbage collection', function () {
+	let dockerStorage: DockerGC;
+	let docker: Docker;
+	beforeEach(function () {
+		dockerStorage = new DockerGC();
+		// Use either local or CI docker
+		return Bluebird.join(
+			getDocker({
+				socketPath: '/tmp/dind/docker.sock',
+			}),
+			dockerStorage.setDocker({
+				socketPath: '/tmp/dind/docker.sock',
+			}),
+			($docker) => {
+				docker = $docker;
+				return dockerStorage.setupMtimeStream();
+			},
+		);
+	});
+
+	afterEach(function () {
+		console.log('[afterEach] Cleaning up...');
+		return IMAGES.concat(NONE_TAG_IMAGES).map((image) =>
+			docker
+				.getImage(image)
+				.remove()
+				.catch(function () {
+					// Ignore
+				}),
+		);
+	});
+
+	it('should remove a image by tag', function () {
+		this.timeout(600000);
+
+		return pullAsync(docker, IMAGES[0])
+			.then(() => docker.getImage(IMAGES[0]).inspect())
+			.then(() => dockerStorage.garbageCollect(1))
+			.then(() => promiseToBool(docker.getImage(IMAGES[0]).inspect()))
+			.then((imageFound) => expect(imageFound).to.be.false);
+	});
+
+	it('should remove a image by digest if its tag == none', function () {
+		this.timeout(600000);
+
+		return pullAsync(docker, NONE_TAG_IMAGES[0])
+			.then(() => docker.getImage(NONE_TAG_IMAGES[0]).inspect())
+			.then(() => dockerStorage.garbageCollect(1))
+			.then(() => promiseToBool(docker.getImage(NONE_TAG_IMAGES[0]).inspect()))
+			.then((imageFound) => expect(imageFound).to.be.false);
+	});
+
+	it('should remove a image with tag == none even if it is in several repos', function () {
+		this.timeout(600000);
+
+		return Bluebird.each(NONE_TAG_IMAGES, (image) => pullAsync(docker, image))
+			.then(() => docker.getImage(NONE_TAG_IMAGES[0]).inspect())
+			.then(() => dockerStorage.garbageCollect(1))
+			.then(() =>
+				Bluebird.map(NONE_TAG_IMAGES, (image) =>
+					promiseToBool(docker.getImage(image).inspect()),
+				),
+			)
+			.then((imagesFound) => expect(imagesFound).to.deep.equal([false, false]));
+	});
+
+	it('should remove all tags of an image', function () {
+		this.timeout(600000);
+		if (SKIP_GC_TEST) {
+			return Promise.resolve();
+		}
+
+		// first pull some images, so we know in which order they are referenced
+		return pullAsync(docker, IMAGES[0])
+			.then(() =>
+				docker.getImage(IMAGES[0]).tag({ repo: 'some-repo', tag: 'some-tag' }),
+			)
+			.then(() => {
+				return dockerStorage.garbageCollect(1);
+			})
+			.then(() => promiseToBool(docker.getImage(IMAGES[0]).inspect()))
+			.then((imagesFound) => expect(imagesFound).to.be.false);
+	});
+
+	it('should remove the LRU image', function () {
+		this.timeout(600000);
+		if (SKIP_GC_TEST) {
+			return Promise.resolve();
+		}
+
+		// first pull some images, so we know in which order they are referenced
+		return pullAsync(docker, IMAGES[0])
+			.then(() =>
+				docker.getImage(IMAGES[0]).tag({ repo: 'some-repo', tag: 'some-tag' }),
+			)
+			.then(() =>
+				Bluebird.each(IMAGES.slice(1), (image) => pullAsync(docker, image)),
+			)
+			.then(() => {
+				// Attempt to remove a single byte, which will remove the LRU image,
+				// which should be alpine
+				return dockerStorage.garbageCollect(1);
+			})
+			.then(() =>
+				Bluebird.map(IMAGES, (image) =>
+					promiseToBool(docker.getImage(image).inspect()),
+				),
+			)
+			.then((imagesFound) =>
+				expect(imagesFound).to.deep.equal([false, true, true]),
+			);
+	});
+
+	it('should remove more than one image if necessary', function () {
+		this.timeout(600000);
+		if (SKIP_GC_TEST) {
+			return Promise.resolve();
+		}
+
+		return Bluebird.each(IMAGES, (image) => pullAsync(docker, image))
+			.then(() =>
+				// Get the size of the first image, so we can add one to it to remove
+				// the next one in addition
+				docker
+					.getImage(IMAGES[0])
+					.inspect()
+					.then((i) => i.Size),
+			)
+			.then((size) => {
+				return dockerStorage.garbageCollect(size + 1);
+			})
+			.then(() =>
+				Bluebird.map(IMAGES, (image) =>
+					promiseToBool(docker.getImage(image).inspect()),
+				),
+			)
+			.then((imagesFound) =>
+				expect(imagesFound).to.deep.equal([false, false, true]),
+			);
+	});
+
+	it('should not consider images in use', function () {
+		this.timeout(600000);
+		const containerName = 'dont-consider-images-in-use-test';
+		if (SKIP_GC_TEST) {
+			return Promise.resolve();
+		}
+
+		return pullAsync(docker, IMAGES[0])
+			.then(() =>
+				docker.createContainer({
+					Image: IMAGES[0],
+					Tty: true,
+					Cmd: ['sh', '-c', 'while true; do echo test; sleep 1; done'],
+					name: containerName,
+					HostConfig: { AutoRemove: true },
+				}),
+			)
+			.then((container) => container.start())
+			.then(() => {
+				return dockerStorage.garbageCollect(1);
+			})
+			.then(() => promiseToBool(docker.getImage(IMAGES[0]).inspect()))
+			.then((imageInspect) => expect(imageInspect).to.be.true)
+			.finally(() => docker.getContainer(containerName).stop());
+	});
+
+	it('should get daemon host disk usage', function () {
+		this.timeout(600000);
+		return dockerStorage.getDaemonFreeSpace().then(function (du) {
+			expect(du).to.be.an('object');
+			expect(du).to.have.property('free').that.is.a('number');
+			expect(du).to.have.property('used').that.is.a('number');
+			expect(du).to.have.property('total').that.is.a('number');
+		});
+	});
+
+	it('should get the correct architecture for a remote host', function () {
+		return (
+			(
+				dockerStorage
+					// @ts-expect-error getDaemonArchitecture is private
+					.getDaemonArchitecture() as Promise<string>
+			).then((arch) => expect(arch).to.be.a('string'))
+		);
+	});
+
+	it('should set a base image to be used', function () {
+		// @ts-expect-error baseImagePromise is private
+		return (dockerStorage.baseImagePromise as Promise<string>).then((img) =>
+			expect(img).to.be.a('string'),
+		);
+	});
+});

package/tools/graphviz.ts

@@ -0,0 +1,39 @@
+import type { ImageNode } from '../lib/docker-image-tree';
+
+const id = (tree: ImageNode) => `foo_${tree.id.slice(0, 7)}`;
+
+const label = function (tree: ImageNode) {
+	const name = tree.repoTags[0] || '\\<none\\>:\\<none\\>';
+	const mtime = new Date(tree.mtime!).toISOString();
+
+	return `label="{ ${tree.id.slice(0, 13)} | ${name} | { ${mtime} | ${
+		tree.size
+	} } }"`;
+};
+
+const color = function (tree: ImageNode) {
+	if (tree.repoTags.length === 0) {
+		return 'color="black"';
+	} else {
+		return 'color="red"';
+	}
+};
+
+const $createDot = function (tree: ImageNode): string[] {
+	// define the node
+	return (
+		[`${id(tree)} [shape=record, ${label(tree)}, ${color(tree)}]`]
+			// define all relations with child nodes
+			.concat(
+				Object.values(tree.children).map(
+					(child) => `${id(tree)} -> ${id(child)}`,
+				),
+			)
+			// recurse to children
+			.concat(...Object.values(tree.children).map((child) => $createDot(child)))
+	);
+};
+
+export function createDot(tree: ImageNode) {
+	return 'digraph {\n' + $createDot(tree).join('\n') + '\n}\n';
+}

package/tsconfig.dev.json

@@ -0,0 +1,13 @@
+{
+	"extends": "./tsconfig.json",
+	"compilerOptions": {
+		"rootDir": ".",
+		"allowJs": true,
+		"checkJs": false
+	},
+	"include": [
+		"lib/**/*",
+		"test/**/*",
+		"tools/**/*"
+	]
+}

package/tsconfig.json

@@ -0,0 +1,22 @@
+{
+	"compilerOptions": {
+		"module": "Node16",
+		"strict": true,
+		"strictPropertyInitialization": false,
+		"noUnusedParameters": true,
+		"noUnusedLocals": true,
+		"outDir": "build/",
+		"preserveConstEnums": true,
+		"removeComments": true,
+		"rootDir": "lib",
+		"sourceMap": true,
+		"target": "es2022",
+		"declaration": true,
+		"skipLibCheck": true,
+		"resolveJsonModule": true,
+		"esModuleInterop": true
+	},
+	"include": [
+		"lib/**/*"
+	]
+}

package/index.d.ts

@@ -1,23 +0,0 @@
-import type * as Dockerode from 'dockerode';
-import type EventEmitter from 'eventemitter3';
-
-interface Events {
-	numberImagesToRemove(n: number): void;
-	gcRunTime(duration: number): void;
-	imageRemoved(removalType: string): void;
-	spaceReclaimed(reclaimSpace: number): void;
-	imageRemovalError(statusCode: string): void;
-}
-declare class DockerGC {
-	setHostname(hostname: string): void;
-	setupMtimeStream(): Promise<void>;
-	setDocker(dockerOpts: Dockerode.DockerOptions): Promise<void>;
-	garbageCollect(reclaimSpace: number, attemptAll?: boolean): Promise<void>;
-	getDaemonFreeSpace(): Promise<{
-		used: number;
-		total: number;
-		free: number;
-	}>;
-	metrics: EventEmitter<Events>;
-}
-export = DockerGC;

package/lib/docker-event-stream.coffee

@@ -1,46 +0,0 @@
-Bluebird = require 'bluebird'
-es = require 'event-stream'
-JSONStream = require 'JSONStream'
-
-IMAGE_EVENTS = [ 'delete', 'import', 'pull', 'push', 'tag' ]
-
-CONTAINER_EVENTS = [
-	'attach',      'commit',  'copy',  'create',  'destroy',  'die',     'exec_create',
-	'exec_start',  'export',  'kill',  'oom',     'pause',    'rename',  'resize',
-	'restart',     'start',   'stop',  'top',     'unpause'
-]
-
-exports.parseEventStream = parseEventStream = (docker) ->
-	docker.listImages(all: true).then((images) ->
-		layer_mtimes = {}
-		# Start off by setting all current images to an mtime of 0 as we've never seen them used
-		# If we've never seen the layer used then it's likely created before we started
-		# listening and so set the last used time to 0 as we know it should be older than
-		# anything we've seen
-		for image in images
-			layer_mtimes[image.Id] = 0
-
-		return es.pipeline(
-			JSONStream.parse()
-			es.mapSync ({ status, id, from, timeNano }) ->
-				if status in IMAGE_EVENTS
-					if status == 'delete'
-						if layer_mtimes[id]? then delete layer_mtimes[id]
-					else
-						layer_mtimes[id] = timeNano
-				else if status in CONTAINER_EVENTS
-					layer_mtimes[from] = timeNano
-				return layer_mtimes
-		)
-	)
-
-exports.dockerMtimeStream = (docker) ->
-	Bluebird.join(
-		docker.getEvents()
-		parseEventStream(docker)
-		(stream, streamParser) ->
-			es.pipeline(
-				stream
-				streamParser
-			)
-	)

package/lib/docker-image-tree.coffee

@@ -1,56 +0,0 @@
-_ = require 'lodash'
-Bluebird = require 'bluebird'
-
-saneRepoAttrs = (repoAttrs) ->
-	return [] if !repoAttrs?
-	return if '<none>:<none>' in repoAttrs or '<none>@<none>' in repoAttrs then [] else repoAttrs
-
-exports.createNode = createNode = (id) -> { id: id, size: 0, repoTags: [], repoDigests: [], mtime: null, children: {} }
-
-getMtimeFrom = (layer_mtimes, attributes) ->
-	for key in attributes
-		if layer_mtimes[key]?
-			return layer_mtimes[key]
-
-getMtime = (tree, layer_mtimes) ->
-	mtime = layer_mtimes[tree.id]
-	if mtime == undefined
-		mtime = getMtimeFrom(layer_mtimes, tree.repoTags)
-	if mtime == undefined
-		mtime = getMtimeFrom(layer_mtimes, tree.repoDigests)
-	return mtime
-
-exports.createTree = createTree = (images, containers, layer_mtimes) ->
-	now = Date.now() * 10 ** 6  # convert to nanoseconds
-	usedImageIds = new Set(
-		_(containers)
-		.map('ImageID')
-	)
-	tree = {}
-	root = '0000000000000000000000000000000000000000000000000000000000000000'
-
-	for image in images
-		node = tree[image.Id] ?= createNode(image.Id)
-		parentId = image.ParentId or root
-		parent = tree[parentId] ?= createNode(parentId)
-
-		node.repoTags = saneRepoAttrs(image.RepoTags)
-		node.repoDigests = saneRepoAttrs(image.RepoDigests)
-		node.size = image.Size
-		# If we haven't seen the image at all then assume it is brand new and default it's
-		# mtime to `now` to avoid removing it
-		node.mtime = getMtime(node, layer_mtimes) ? now
-		node.isUsedByAContainer = usedImageIds.has(image.Id)
-		parent.children[image.Id] = node
-
-	tree[root].mtime = now
-	tree[root].isUsedByAContainer = false
-	return tree[root]
-
-exports.dockerImageTree = (docker, layer_mtimes) ->
-	Bluebird.join(
-		docker.listImages(all: true)
-		docker.listContainers(all: true)
-		layer_mtimes
-		createTree
-	)

package/lib/docker.coffee

@@ -1,13 +0,0 @@
-Docker = require 'dockerode'
-Bluebird = require 'bluebird'
-_ = require 'lodash'
-
-getDockerConnectOpts = (hostObj) ->
-	if !_.isEmpty(hostObj)
-		return Bluebird.resolve(hostObj)
-	return Bluebird.resolve({ socketPath: '/var/run/docker.sock', Promise: Bluebird })
-
-exports.getDocker = (hostObj) ->
-	getDockerConnectOpts(hostObj)
-	.then (opts) ->
-		return new Docker(opts)

package/lib/index.coffee

@@ -1,168 +0,0 @@
-Bluebird = require 'bluebird'
-_ = require 'lodash'
-{ EventEmitter } = require 'eventemitter3'
-
-{ DockerProgress } = require 'docker-progress'
-Docker = require 'dockerode'
-
-{ dockerMtimeStream } = require './docker-event-stream'
-{ dockerImageTree } = require './docker-image-tree'
-dockerUtils = require './docker'
-
-getUnusedTreeLeafs = (tree, result = []) ->
-	if not tree.removed
-		children = _(tree.children)
-		.values()
-		.filter(_.negate(_.property('removed')))
-		.value()
-		if children.length == 0 and not tree.isUsedByAContainer
-			result.push(tree)
-		else
-			for child in children
-				getUnusedTreeLeafs(child, result)
-	return result
-
-getImagesToRemove = (tree, reclaimSpace, metrics) ->
-	# Removes the oldest, largest leafs first.
-	# This should avoid trying to remove images with children.
-	tree = _.clone(tree)
-	result = []
-	size = 0
-	while size < reclaimSpace
-		leafs = _.orderBy(
-			getUnusedTreeLeafs(tree)
-			[ 'mtime', 'size' ]
-			[ 'asc', 'desc' ]
-		)
-		if leafs.length == 0
-			break
-		leaf = leafs[0]
-		if leaf != tree
-			# don't remove the tree root
-			result.push(leaf)
-			size += leaf.size
-		leaf.removed = true
-
-	metrics.emit('numberImagesToRemove', result.length)
-	return result
-
-streamToString = (stream) ->
-	new Bluebird (resolve, reject) ->
-		chunks = []
-		stream
-		.on('error', reject)
-		.on 'data', (chunk) ->
-			chunks.push(chunk)
-		.on 'end', ->
-			resolve(Buffer.concat(chunks).toString())
-
-recordGcRunTime = (t0, metrics) ->
-	dt = process.hrtime(t0)
-	duration = dt[0] * 1000 + dt[1] / 1e6
-	metrics.emit('gcRunTime', duration)
-
-class DockerGC
-
-	constructor: ->
-		@metrics = new EventEmitter()
-		@host = 'unknown'
-
-	setHostname: (hostname) ->
-		@host = hostname
-
-	setDocker: (hostObj) ->
-		@currentMtimes = {}
-		@hostObj = _.defaults({ Promise: Bluebird }, hostObj)
-		@dockerProgress = new DockerProgress({ docker: new Docker(@hostObj) })
-		dockerUtils.getDocker(@hostObj)
-		.then (@docker) =>
-			# Docker info can take a while so do it here,
-			# and don't wait on the results
-			@baseImagePromise = @getDaemonArchitecture()
-			.then (arch) ->
-				return switch arch
-					when 'arm' then 'arm32v6/alpine:3.6'
-					when 'arm64' then 'arm64v8/alpine:3.6'
-					when 'amd64' then 'alpine:3.6'
-					else
-						throw new Error('Could not detect architecture of remote host')
-
-	setupMtimeStream: ->
-		dockerMtimeStream(@docker)
-		.then (stream) =>
-			stream
-			.on 'data', (layer_mtimes) =>
-				@currentMtimes = layer_mtimes
-
-	removeImage: (image) =>
-		return this.tryRemoveImageBy(image, image.repoTags, 'tag') ||
-					 this.tryRemoveImageBy(image, image.repoDigests, 'digest') ||
-					 this.tryRemoveImageBy(image, [image.id], 'id')
-
-	tryRemoveImageBy: (image, attributes, removalType) =>
-		if attributes? and attributes.length > 0
-			Bluebird.each attributes, (attribute) =>
-				console.log("[GC (#{@host}] Removing image : #{attribute} (id: #{image.id})")
-				@docker.getImage(attribute).remove(noprune: true)
-				.then =>
-					@metrics.emit('imageRemoved', removalType)
-
-	garbageCollect: (reclaimSpace, attemptAll = false) =>
-		err = null
-		startTime = process.hrtime()
-		@metrics.emit('spaceReclaimed', reclaimSpace)
-		dockerImageTree(@docker, @currentMtimes)
-		.then (tree) =>
-			getImagesToRemove(tree, reclaimSpace, @metrics)
-		.each (image) =>
-			@removeImage(image)
-			.catch (e) =>
-				@metrics.emit('imageRemovalError', e.statusCode)
-				console.log("[GC #{@host}]: Failed to remove image: ", image)
-				console.log(e)
-				if attemptAll
-					err ?= e
-				else
-					recordGcRunTime(startTime, @metrics)
-					throw e
-		.then =>
-			recordGcRunTime(startTime, @metrics)
-			if err?
-				throw err
-
-	getOutput: (image, command) ->
-		Bluebird.using @runDisposer(image, command), (container) ->
-			container.logs(stdout: true, follow: true)
-			.then (logs) ->
-				streamToString(logs)
-
-	runDisposer: (image, command) ->
-		@docker.run(image, command)
-		.disposer (container) ->
-			container.wait()
-			.then ->
-				container.remove()
-
-	getDaemonFreeSpace: ->
-		@baseImagePromise.tap (baseImage) =>
-			# Ensure the image is available (if it is this is essentially a no-op)
-			@dockerProgress.pull(baseImage, _.noop)
-		.then (baseImage) =>
-			@getOutput(baseImage, [ '/bin/df', '-B', '1', '/' ])
-		.then (spaceStr) ->
-			# First split the lines, as we're only interested in the second one
-			lines = spaceStr.trim().split(/\r?\n/)
-			if lines.length isnt 2
-				throw new Error('Coult not parse df output')
-
-			parts = lines[1].split(/\s+/)
-			total = parseInt(parts[1])
-			used = parseInt(parts[2])
-			free = parseInt(parts[3])
-			return { used, total, free }
-
-	getDaemonArchitecture: ->
-		@docker.version()
-		.get('Arch')
-
-module.exports = DockerGC

package/test/docker-event-stream.coffee

@@ -1,30 +0,0 @@
-{ expect } = require 'chai'
-fs = require 'fs'
-es = require 'event-stream'
-
-{ parseEventStream } = require '../lib/docker-event-stream'
-
-dockerUtils = require('../lib/docker.coffee')
-
-describe 'parseEventStream', ->
-	it.skip 'should work with empty stream', ->
-
-	it 'should return updated mtimes', ->
-		dockerUtils.getDocker({})
-		.then (docker) ->
-			parseEventStream(docker)
-		.then (streamParser) ->
-			new Promise (resolve, reject) ->
-				mtimes = null
-
-				fs.createReadStream(__dirname + '/fixtures/docker-events.json')
-				.pipe(streamParser)
-				.on 'error', reject
-				.pipe es.mapSync (data) ->
-					mtimes = data
-				.on 'end', -> resolve(mtimes)
-				.on 'error', reject
-		.then (data) ->
-			expect(data).to.have.property('busybox:latest').that.equals(1448576072937294800)
-			expect(data).to.have.property('sha256:6d41a4a0bf8168363e29da8a5ecbf3cd6c37e3f5a043decd5e7da6e427ba869c').that.equals(1448576073085559800)
-			expect(data).to.have.property('sha256:9a61b6b1315e6b457c31a03346ab94486a2f5397f4a82219bee01eead1c34c2e').that.equals(1448576073203895800)