docker-storage-gc 3.5.12 → 4.0.0

package/lib/index.ts

@@ -0,0 +1,272 @@
+import Bluebird, { Disposer } from 'bluebird';
+import _ from 'lodash';
+import { EventEmitter } from 'eventemitter3';
+import { DockerProgress } from 'docker-progress';
+import Docker from 'dockerode';
+import { LayerMtimes, dockerMtimeStream } from './docker-event-stream';
+import { ImageNode, dockerImageTree } from './docker-image-tree';
+import { getDocker } from './docker';
+
+interface Events {
+	numberImagesToRemove(n: number): void;
+	gcRunTime(duration: number): void;
+	imageRemoved(removalType: string): void;
+	spaceReclaimed(reclaimSpace: number): void;
+	imageRemovalError(statusCode: string): void;
+}
+type Metrics = EventEmitter<Events>;
+
+interface RemovableImageNode extends ImageNode {
+	removed?: true;
+}
+
+const getUnusedTreeLeafs = function (
+	tree: RemovableImageNode,
+	result: RemovableImageNode[] = [],
+) {
+	if (!tree.removed) {
+		const children = _(tree.children)
+			.values()
+			.filter(_.negate(_.property('removed')))
+			.value();
+		if (children.length === 0 && !tree.isUsedByAContainer) {
+			result.push(tree);
+		} else {
+			for (const child of children) {
+				getUnusedTreeLeafs(child, result);
+			}
+		}
+	}
+	return result;
+};
+
+const getImagesToRemove = function (
+	tree: RemovableImageNode,
+	reclaimSpace: number,
+	metrics: Metrics,
+) {
+	// Removes the oldest, largest leafs first.
+	// This should avoid trying to remove images with children.
+	tree = _.clone(tree);
+	const result = [];
+	let size = 0;
+	while (size < reclaimSpace) {
+		const leafs = _.orderBy(
+			getUnusedTreeLeafs(tree),
+			['mtime', 'size'],
+			['asc', 'desc'],
+		);
+		if (leafs.length === 0) {
+			break;
+		}
+		const leaf = leafs[0];
+		if (leaf !== tree) {
+			// don't remove the tree root
+			result.push(leaf);
+			size += leaf.size;
+		}
+		leaf.removed = true;
+	}
+
+	metrics.emit('numberImagesToRemove', result.length);
+	return result;
+};
+
+const streamToString = (stream: NodeJS.ReadableStream) =>
+	new Promise<string>(function (resolve, reject) {
+		const chunks: Buffer[] = [];
+		stream
+			.on('error', reject)
+			.on('data', (chunk) => chunks.push(chunk))
+			.on('end', () => resolve(Buffer.concat(chunks).toString()));
+	});
+
+const recordGcRunTime = function (
+	t0: ReturnType<NodeJS.HRTime>,
+	metrics: Metrics,
+) {
+	const dt = process.hrtime(t0);
+	const duration = dt[0] * 1000 + dt[1] / 1e6;
+	metrics.emit('gcRunTime', duration);
+};
+
+export default class DockerGC {
+	public metrics: Metrics = new EventEmitter<Events>();
+	private host = 'unknown';
+	private docker: Docker;
+	private dockerProgress: DockerProgress;
+	private currentMtimes: LayerMtimes = {};
+	private baseImagePromise: Promise<string>;
+
+	public setHostname(hostname: string): void {
+		this.host = hostname;
+	}
+
+	public setDocker(hostObj: Docker.DockerOptions): Promise<void> {
+		this.currentMtimes = {};
+		hostObj = _.defaults({ Promise: Bluebird }, hostObj);
+		this.dockerProgress = new DockerProgress({
+			docker: new Docker(hostObj),
+		});
+		return getDocker(hostObj)
+			.then((docker) => {
+				// Docker info can take a while so do it here,
+				// and don't wait on the results
+				this.docker = docker;
+				return (this.baseImagePromise = this.getDaemonArchitecture().then(
+					function (arch) {
+						switch (arch) {
+							case 'arm':
+								return 'arm32v6/alpine:3.6';
+							case 'arm64':
+								return 'arm64v8/alpine:3.6';
+							case 'amd64':
+								return 'alpine:3.6';
+							default:
+								throw new Error('Could not detect architecture of remote host');
+						}
+					},
+				));
+			})
+			.then(() => {
+				// noop
+			});
+	}
+
+	public setupMtimeStream(): Promise<void> {
+		return dockerMtimeStream(this.docker).then((stream) => {
+			stream.on('data', (layerMtimes: LayerMtimes) => {
+				this.currentMtimes = layerMtimes;
+			});
+		});
+	}
+
+	private removeImage(image: RemovableImageNode) {
+		return (
+			this.tryRemoveImageBy(image, image.repoTags, 'tag') ||
+			this.tryRemoveImageBy(image, image.repoDigests, 'digest') ||
+			this.tryRemoveImageBy(image, [image.id], 'id')
+		);
+	}
+
+	private tryRemoveImageBy(
+		image: RemovableImageNode,
+		attributes: [string],
+		removalType: 'tag' | 'digest' | 'id',
+	): Promise<void>;
+	private tryRemoveImageBy(
+		image: RemovableImageNode,
+		attributes: string[],
+		removalType: 'tag' | 'digest' | 'id',
+	): Promise<void> | undefined;
+	private tryRemoveImageBy(
+		image: RemovableImageNode,
+		attributes: string[],
+		removalType: 'tag' | 'digest' | 'id',
+	): Promise<void> | undefined {
+		if (attributes.length > 0) {
+			return Bluebird.each(attributes, (attribute) => {
+				console.log(
+					`[GC (${this.host}] Removing image : ${attribute} (id: ${image.id})`,
+				);
+				return this.docker
+					.getImage(attribute)
+					.remove({ noprune: true })
+					.then(() => {
+						this.metrics.emit('imageRemoved', removalType);
+					});
+			}).then(() => {
+				// noop
+			});
+		}
+	}
+
+	public garbageCollect(
+		reclaimSpace: number,
+		attemptAll = false,
+	): Promise<void> {
+		let err: any;
+		const startTime = process.hrtime();
+		this.metrics.emit('spaceReclaimed', reclaimSpace);
+		return dockerImageTree(this.docker, this.currentMtimes)
+			.then((tree) => {
+				return getImagesToRemove(tree, reclaimSpace, this.metrics);
+			})
+			.each((image) => {
+				return this.removeImage(image).catch((e) => {
+					this.metrics.emit('imageRemovalError', e.statusCode);
+					console.log(`[GC ${this.host}]: Failed to remove image: `, image);
+					console.log(e);
+					if (attemptAll) {
+						err ??= e;
+						return;
+					} else {
+						recordGcRunTime(startTime, this.metrics);
+						throw e;
+					}
+				});
+			})
+			.then(() => {
+				recordGcRunTime(startTime, this.metrics);
+				if (err != null) {
+					throw err;
+				}
+			});
+	}
+
+	private getOutput(image: string, command: string[]): Promise<string> {
+		return Bluebird.using(this.runDisposer(image, command), (container) =>
+			container
+				.logs({ stdout: true, follow: true })
+				.then((logs) => streamToString(logs)),
+		);
+	}
+
+	private runDisposer(
+		image: string,
+		command: string[],
+	): Disposer<Docker.Container> {
+		const containerPromise: Promise<[unknown, Docker.Container]> =
+			this.docker.run(
+				image,
+				command,
+				// @ts-expect-error -- The typings expect an array of streams but in reality they're optional
+				undefined,
+			);
+		return Bluebird.resolve(
+			containerPromise.then(([, container]) => container),
+		).disposer((container) => container.wait().then(() => container.remove()));
+	}
+
+	public getDaemonFreeSpace(): Promise<{
+		used: number;
+		total: number;
+		free: number;
+	}> {
+		return Bluebird.resolve(this.baseImagePromise)
+			.tap((baseImage) => {
+				// Ensure the image is available (if it is this is essentially a no-op)
+				return this.dockerProgress.pull(baseImage, _.noop);
+			})
+			.then((baseImage) => {
+				return this.getOutput(baseImage, ['/bin/df', '-B', '1', '/']);
+			})
+			.then(function (spaceStr) {
+				// First split the lines, as we're only interested in the second one
+				const lines = spaceStr.trim().split(/\r?\n/);
+				if (lines.length !== 2) {
+					throw new Error('Coult not parse df output');
+				}
+
+				const parts = lines[1].split(/\s+/);
+				const total = parseInt(parts[1], 10);
+				const used = parseInt(parts[2], 10);
+				const free = parseInt(parts[3], 10);
+				return { used, total, free };
+			});
+	}
+
+	private getDaemonArchitecture() {
+		return this.docker.version().then(({ Arch }) => Arch);
+	}
+}

package/package.json

@@ -1,36 +1,44 @@
 {
   "name": "docker-storage-gc",
-  "version": "3.5.12",
+  "version": "4.0.0",
   "main": "build/index.js",
-  "types": "index.d.ts",
   "scripts": {
-    "lint": "balena-lint lib test tools",
-    "pretest": "npm run lint && docker rm -vf docker-storage-gc-tests && docker run --privileged --name docker-storage-gc-tests -v /tmp/dind:/var/run/ -d docker:24.0.5-dind && sleep 5 && docker exec docker-storage-gc-tests chown $(id -u) /var/run/docker.sock",
-    "test": "mocha --exit --require coffeescript/register test/**/*.coffee",
-    "posttest": "docker rm -vf docker-storage-gc-tests",
-    "prepare": "coffee -o build -c lib"
+    "lint": "balena-lint -t tsconfig.dev.json -e ts -e js lib test tools",
+    "lint-fix": "balena-lint --fix  -t tsconfig.dev.json -e ts -e js lib test tools",
+    "pretest": "npm run prepare && docker rm -vf docker-storage-gc-tests && docker run --privileged --name docker-storage-gc-tests -v /tmp/dind:/var/run/ -d docker:24.0.5-dind && sleep 5 && docker exec docker-storage-gc-tests chown $(id -u) /var/run/docker.sock",
+    "test": "mocha --exit --require ts-node/register/transpile-only test/**/*.ts",
+    "posttest": "docker rm -vf docker-storage-gc-tests && npm run lint",
+    "prepare": "npx tsc"
   },
   "author": "",
   "license": "Apache 2.0",
   "description": "Automatically cleanup unused images based on various cache replacement algorithms",
   "dependencies": {
-    "@types/dockerode": "^2.5.34",
+    "@types/bluebird": "^3.5.42",
+    "@types/dockerode": "^3.3.23",
+    "@types/event-stream": "^4.0.5",
+    "@types/JSONStream": "npm:@types/jsonstream@^0.8.31",
+    "@types/lodash": "^4.14.202",
+    "@types/node": "^16.18.65",
     "bluebird": "^3.7.2",
     "docker-progress": "^5.2.0",
-    "dockerode": "^2.5.8",
+    "dockerode": "^3.3.5",
     "event-stream": "^4.0.1",
     "eventemitter3": "^5.0.1",
     "JSONStream": "^1.3.5",
     "lodash": "^4.17.21"
   },
   "devDependencies": {
-    "@balena/lint": "^5.4.2",
+    "@balena/lint": "^7.2.4",
+    "@types/chai": "^4.3.11",
+    "@types/mocha": "^10.0.6",
     "chai": "^4.3.10",
-    "coffeescript": "^1.12.7",
     "mocha": "^10.2.0",
-    "timekeeper": "^2.3.1"
+    "timekeeper": "^2.3.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.3.2"
   },
   "versionist": {
-    "publishedAt": "2023-12-04T16:00:09.489Z"
+    "publishedAt": "2023-12-04T17:24:21.020Z"
   }
 }

package/test/docker-event-stream.ts

@@ -0,0 +1,46 @@
+import { LayerMtimes } from '../build/docker-event-stream';
+
+import { expect } from 'chai';
+import fs from 'fs';
+import es from 'event-stream';
+import { parseEventStream } from '../build/docker-event-stream';
+import { getDocker } from '../build/docker';
+
+describe('parseEventStream', function () {
+	it.skip('should work with empty stream', function () {
+		// TODO
+	});
+
+	it('should return updated mtimes', () =>
+		getDocker({})
+			.then((docker) => parseEventStream(docker))
+			.then(
+				(streamParser) =>
+					new Promise<LayerMtimes>(function (resolve, reject) {
+						let mtimes: LayerMtimes;
+
+						return fs
+							.createReadStream(__dirname + '/fixtures/docker-events.json')
+							.pipe(streamParser)
+							.on('error', reject)
+							.pipe(es.mapSync((data: LayerMtimes) => (mtimes = data)))
+							.on('end', () => resolve(mtimes))
+							.on('error', reject);
+					}),
+			)
+			.then(function (data) {
+				expect(data)
+					.to.have.property('busybox:latest')
+					.that.equals(1448576072937294800);
+				expect(data)
+					.to.have.property(
+						'sha256:6d41a4a0bf8168363e29da8a5ecbf3cd6c37e3f5a043decd5e7da6e427ba869c',
+					)
+					.that.equals(1448576073085559800);
+				expect(data)
+					.to.have.property(
+						'sha256:9a61b6b1315e6b457c31a03346ab94486a2f5397f4a82219bee01eead1c34c2e',
+					)
+					.that.equals(1448576073203895800);
+			}));
+});

package/test/docker-image-tree.ts

@@ -0,0 +1,137 @@
+import type { ContainerInfo, ImageInfo } from 'dockerode';
+import { expect } from 'chai';
+import fs from 'fs';
+import tk from 'timekeeper';
+import es from 'event-stream';
+import { LayerMtimes, parseEventStream } from '../build/docker-event-stream';
+import { createTree } from '../build/docker-image-tree';
+import { getDocker } from '../build/docker';
+
+const getLayerMtimes = () =>
+	getDocker({})
+		.then((docker) => parseEventStream(docker))
+		.then(
+			(streamParser) =>
+				new Promise<LayerMtimes>(function (resolve, reject) {
+					let mtimes: LayerMtimes;
+					return fs
+						.createReadStream(__dirname + '/fixtures/docker-events.json')
+						.pipe(streamParser)
+						.on('error', reject)
+						.pipe(es.mapSync((data: LayerMtimes) => (mtimes = data)))
+						.on('end', () => resolve(mtimes))
+						.on('error', reject);
+				}),
+		);
+
+describe('createTree', function () {
+	it.skip('should work with empty input', function () {
+		// TODO
+	});
+
+	it('should return a tree of images', async function () {
+		const images = (
+			await import('./fixtures/docker-images.json', {
+				assert: { type: 'json' },
+			})
+		).default as ImageInfo[];
+		const containers = (
+			await import('./fixtures/docker-containers.json', {
+				assert: { type: 'json' },
+			})
+		).default as ContainerInfo[];
+		return getLayerMtimes().then(function (mtimes) {
+			tk.freeze(Date.UTC(2016, 0, 1));
+			const tree = createTree(images, containers, mtimes);
+			tk.reset();
+			const output = {
+				id: '0000000000000000000000000000000000000000000000000000000000000000',
+				size: 0,
+				repoTags: [],
+				repoDigests: [],
+				mtime: 1451606400000000000,
+				isUsedByAContainer: false,
+				children: {
+					'sha256:6d15899cef812e2876b9d5d43d4cd863eda7b278f7b52d00975f6a9a8e817c74':
+						{
+							id: 'sha256:6d15899cef812e2876b9d5d43d4cd863eda7b278f7b52d00975f6a9a8e817c74',
+							size: 125151141,
+							repoTags: [],
+							repoDigests: [],
+							mtime: 1451606400000000000,
+							isUsedByAContainer: false,
+							children: {
+								'sha256:e53bd4df04f86919156c4510cdc6e6c9491ec8ec226381d36aca573b46bbbbbc':
+									{
+										id: 'sha256:e53bd4df04f86919156c4510cdc6e6c9491ec8ec226381d36aca573b46bbbbbc',
+										size: 0,
+										repoTags: ['project1'],
+										repoDigests: [],
+										mtime: 1451606400000000000,
+										isUsedByAContainer: false,
+										children: {
+											'sha256:6d41a4a0bf8168363e29da8a5ecbf3cd6c37e3f5a043decd5e7da6e427ba869c':
+												{
+													id: 'sha256:6d41a4a0bf8168363e29da8a5ecbf3cd6c37e3f5a043decd5e7da6e427ba869c',
+													size: 330389,
+													repoTags: ['project2'],
+													repoDigests: [],
+													// eslint-disable-next-line @typescript-eslint/no-loss-of-precision
+													mtime: 1448576073085559863,
+													isUsedByAContainer: false,
+													children: {
+														'sha256:80dc79d29cd8618e678da508fc32f7289e6f72defb534f3f287731b1f8b355ea':
+															{
+																id: 'sha256:80dc79d29cd8618e678da508fc32f7289e6f72defb534f3f287731b1f8b355ea',
+																size: 98872,
+																repoTags: [],
+																repoDigests: [],
+																mtime: 1451606400000000000,
+																isUsedByAContainer: false,
+																children: {},
+															},
+													},
+												},
+										},
+									},
+							},
+						},
+					'sha256:902b87aaaec929e80541486828959f14fa061f529ad7f37ab300d4ef9f3a0dbf':
+						{
+							id: 'sha256:902b87aaaec929e80541486828959f14fa061f529ad7f37ab300d4ef9f3a0dbf',
+							size: 125151141,
+							repoTags: [],
+							repoDigests: [],
+							mtime: 1451606400000000000,
+							isUsedByAContainer: false,
+							children: {
+								'sha256:9a61b6b1315e6b457c31a03346ab94486a2f5397f4a82219bee01eead1c34c2e':
+									{
+										id: 'sha256:9a61b6b1315e6b457c31a03346ab94486a2f5397f4a82219bee01eead1c34c2e',
+										size: 0,
+										repoTags: ['resin/project3'],
+										repoDigests: [],
+										mtime: 1448576073203895800,
+										isUsedByAContainer: false,
+										children: {},
+									},
+							},
+						},
+					'sha256:5b0d59026729b68570d99bc4f3f7c31a2e4f2a5736435641565d93e7c25bd2c3':
+						{
+							id: 'sha256:5b0d59026729b68570d99bc4f3f7c31a2e4f2a5736435641565d93e7c25bd2c3',
+							size: 125151141,
+							repoTags: ['busybox:latest'],
+							repoDigests: [
+								'sha256:a8cf7ff6367c2afa2a90acd081b484cbded349a7076e7bdf37a05279f276bc12',
+							],
+							mtime: 1448576072937294800,
+							isUsedByAContainer: true,
+							children: {},
+						},
+				},
+			};
+			expect(tree).to.deep.equal(output);
+		});
+	});
+});