ip-asn-map 1.0.0

package/src/map.ts

@@ -0,0 +1,188 @@
+import type { AsInfo, IpInfo, Numbers } from './types.ts';
+import { parseIPv4, parseIPv6, stringifyIP4, stringifyIP6 } from './utils.ts';
+
+export class IpMap {
+
+	buffer: ArrayBuffer;
+	bytes: Uint8Array;
+	view: DataView;
+	
+	readonly ip4Count: number;
+	readonly ip6Count: number;
+	readonly asOffset: number;
+	
+	readonly ip6View: DataView;
+	readonly ip4View: DataView;
+	readonly ip6AsView: DataView;
+	readonly ip4AsView: DataView;
+	readonly asView: DataView;
+	
+	static version = 1;
+	
+	/**
+	 * Constructs an IpMap instance using the provided ArrayBuffer with the previously generated binary data.
+	 * @param buffer
+	 */
+	constructor(buffer: ArrayBuffer) {
+		
+		this.buffer = buffer;
+		const bytes = this.bytes = new Uint8Array(buffer);
+		const view = this.view = new DataView(buffer);
+		
+		new ArrayBuffer()
+		
+		if (String.fromCharCode(...bytes.slice(0, 6)) != 'ip-map') throw new Error('Not an ip-asn-map buffer');
+		const [version, num6, num4, asOffset, bufSize] = [0, 1, 2, 3, 4].map(i => view.getUint32(8 + 4 * i)) as Numbers<5>;
+		if (version != IpMap.version) throw new Error(`Expected file version ${IpMap.version}, got ${version}`);
+		if (buffer.byteLength != bufSize) throw new Error(`Expected file size ${bufSize	}, got ${buffer.byteLength}`);
+		
+		this.ip4Count = num4;
+		this.ip6Count = num6;
+		this.asOffset = asOffset;
+		
+		this.ip6View = new DataView(buffer, 32, num6 * 16);
+		this.ip4View = new DataView(buffer, this.ip6View.byteOffset + this.ip6View.byteLength, num4 * 4);
+		this.ip6AsView = new DataView(buffer, this.ip4View.byteOffset + this.ip4View.byteLength, num6 * 4);
+		this.ip4AsView = new DataView(buffer, this.ip6AsView.byteOffset + this.ip6AsView.byteLength, num4 * 4);
+		this.asView = new DataView(buffer, this.asOffset, buffer.byteLength - this.asOffset);
+		
+		if (this.ip4AsView.byteOffset + this.ip4AsView.byteLength != this.asView.byteOffset) throw new Error('Something *really* went wrong');
+		
+	}
+	
+	private getAsAtPos(pos: number): AsInfo {
+		const asNum = this.asView.getUint32(pos);
+		const ccInt = this.asView.getUint16(pos + 4);
+		const descLen = this.asView.getUint16(pos + 6);
+		const descAt = this.asOffset + pos + 8;
+		const desc = !descLen ? null : String.fromCharCode(...this.bytes.slice(descAt, descAt + descLen));
+		return {
+			number: asNum,
+			country: ccInt ? String.fromCharCode((ccInt >> 8) & 255, ccInt & 255) : null,
+			description: desc,
+		};
+	}
+	
+	/**
+	 * Looks up an IPv4 address in the little-endian uint32 representation (byte order as written)
+	 * @param needle
+	 */
+	findV4(needle: number): IpInfo<4> {
+	
+		if (needle < 0 || needle > 0xffff_ffff) {
+			throw new Error('Invalid IPv4 integer');
+		}
+		
+		const view = this.ip4View;
+		
+		let low = 0;
+		let high = this.ip4Count - 1;
+		
+		while (high - low > 1) {
+			const at = Math.round(low + (high - low) * 0.5);
+			const addr = view.getUint32(at * 4);
+			if (addr > needle) {
+				high = at;
+			}
+			else {
+				low = at;
+			}
+		}
+		
+		const first = view.getUint32(low * 4);
+		const next = view.getUint32(high * 4);
+		const last = next - 1;
+		
+		const as = this.getAsAtPos(this.ip4AsView.getUint32(4 * low));
+		
+		return {
+			family: 4,
+			ip: stringifyIP4(needle),
+			int: needle,
+			as,
+			range: {
+				family: 4,
+				first: stringifyIP4(first),
+				last: stringifyIP4(last),
+				firstInt: first,
+				lastInt: last,
+				size: next - first,
+				index: low,
+			},
+		}
+		
+	}
+	
+	/**
+	 * Looks up an IPv6 address in the little-endian uint128 representation (byte order as written)
+	 * @param needle
+	 */
+	findV6(needle: bigint): IpInfo<6> {
+	
+		if (needle < 0n || needle >= (1n << 128n)) {
+			throw new Error('Invalid IPv6 bigint');
+		}
+		
+		const view = this.ip6View;
+		
+		let low = 0;
+		let high = this.ip6Count - 1;
+		
+		const readAddr = (pos: number) => {
+			const a = view.getBigUint64(pos * 16);
+			const b = view.getBigUint64(pos * 16 + 8);
+			return (a << 64n) | b;
+		}
+		
+		while (high - low > 1) {
+			const at = Math.round(low + (high - low) * 0.5);
+			const addr = readAddr(at);
+			if (addr > needle) {
+				high = at;
+			}
+			else {
+				low = at;
+			}
+		}
+		
+		const first = readAddr(low);
+		const next = readAddr(high);
+		const last = next - 1n;
+		
+		const as = this.getAsAtPos(this.ip6AsView.getUint32(4 * low));
+		
+		return {
+			family: 6,
+			ip: stringifyIP6(needle),
+			int: needle,
+			as,
+			range: {
+				family: 6,
+				first: stringifyIP6(first),
+				last: stringifyIP6(last),
+				firstInt: first,
+				lastInt: last,
+				size: next - first,
+				index: low,
+			},
+		}
+		
+	}
+	
+	/**
+	 * Looks up an IP address in the form 1.2.3.4 or a:b::c:d, returns {@link IpInfo}
+	 * @param address
+	 */
+	lookup(address: string): IpInfo {
+		if (address.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/)) {
+			return this.findV4(parseIPv4(address));
+		}
+		else if (address.match(/^([0-9a-f]*):/)) {
+			return this.findV6(parseIPv6(address));
+		}
+		else {
+			throw new Error('Unexpected address format');
+		}
+	}
+	
+}

package/src/test.ts

@@ -0,0 +1,104 @@
+import { stat, writeFile } from 'node:fs/promises';
+import { IpMap } from './map.ts';
+import { loadIpMap, parseIp2AsnGzip } from './buffer.ts';
+import * as assert from 'node:assert';
+import type { Tuple } from './types.ts';
+import * as process from 'node:process';
+
+const gz = `data/ip2asn-combined.tsv.gz`;
+const bin = `data/ip-map.bin`;
+
+if (!await stat(gz).catch(() => null)) {
+	console.log('Fetching', gz);
+	const res = await fetch('https://iptoasn.com/data/ip2asn-combined.tsv.gz');
+	if (!res.ok) throw new Error(res.statusText);
+	await writeFile(gz, Buffer.from(await res.arrayBuffer()));
+}
+
+const { createInterface } = await import('node:readline');
+const { createReadStream } = await import('node:fs');
+const { createGunzip } = await import('node:zlib');
+const stream = createReadStream(gz);
+const gunzip = createGunzip();
+stream.pipe(gunzip);
+const rl = createInterface(gunzip);
+const tsv = await Array.fromAsync(rl).then(arr => arr.filter(s => s.trim()));
+
+let t = performance.now();
+if (!await stat(bin).catch(() => null)) {
+	const buf = await parseIp2AsnGzip(gz);
+	console.log('parse:', performance.now() - t, 'ms');
+	await writeFile(bin, buf);
+}
+
+const map = await loadIpMap(bin);
+
+const checkN = 10000;
+
+console.log('IPv4 random test');
+for (let i = 0; i < checkN; i++) {
+	const int = Math.round(Math.random() * 0xffff_ffff);
+	const res = map.findV4(int);
+	assert.equal(res.int, int);
+	assert.ok(res.range.firstInt <= int);
+	assert.ok(res.range.lastInt >= int);
+}
+
+console.log('IPv6 random test');
+for (let i = 0; i < checkN; i++) {
+	let int = BigInt(2000 + Math.round(Math.random() * 0xcff));
+	for (let c = 0; c < 7; c++) {
+		int = (int << 16n) | BigInt(Math.round(Math.random() * 0xffff));
+	}
+	const res = map.findV6(int);
+	assert.equal(res.int, int);
+	assert.ok(res.range.firstInt <= int);
+	assert.ok(res.range.lastInt >= int);
+}
+
+console.log('Random TSV lines test');
+for (let i = 0; i < checkN; i++) {
+	const line = tsv[Math.floor(Math.random() * tsv.length)]!;
+	const [start, _end, asNumStr, ccStr, desc] = line.split('\t') as Tuple<string, 5>;
+	const res = map.lookup(start);
+	assert.equal(res.as.country, ccStr == 'None' || ccStr == 'Unknown' ? null : ccStr);
+	assert.equal(String(res.as.number), asNumStr);
+	assert.equal(res.as.description, asNumStr == '0' ? null : desc);
+}
+
+if (!process.argv.includes('bench')) process.exit();
+
+const benchN = 1_000_000;
+
+t = performance.now();
+for (let i = 0; i < benchN; i++) {
+	const int = Math.round(Math.random() * 0xffff_ffff);
+	map.findV4(int);
+}
+{
+	const ms = performance.now() - t;
+	console.log('IPv4 random int', benchN, 'iters in', ms.toFixed(2), 'ms; ', Math.round(benchN / ms * 1000), 'lookups/s; ', (ms / benchN * 1000).toFixed(2), 'us per lookup');
+}
+
+t = performance.now();
+for (let i = 0; i < benchN; i++) {
+	let int = BigInt(2000 + Math.round(Math.random() * 0xcff));
+	for (let c = 0; c < 7; c++) {
+		int = (int << 16n) | BigInt(Math.round(Math.random() * 0xffff));
+	}
+	map.findV6(int);
+}
+{
+	const ms = performance.now() - t;
+	console.log('IPv6 random int', benchN, 'iters in', ms.toFixed(2), 'ms; ', Math.round(benchN / ms * 1000), 'lookups/s; ', (ms / benchN * 1000).toFixed(2), 'us per lookup');
+}
+
+t = performance.now();
+for (let i = 0; i < benchN; i++) {
+	const line = tsv[Math.floor(Math.random() * tsv.length)]!;
+	map.lookup(line.split('\t')[0]!);
+}
+{
+	const ms = performance.now() - t;
+	console.log('Random TSV row', benchN, 'iters in', ms.toFixed(2), 'ms; ', Math.round(benchN / ms * 1000), 'lookups/s; ', (ms / benchN * 1000).toFixed(2), 'us per parse + lookup');
+}

package/src/types.ts

@@ -0,0 +1,88 @@
+export interface TupleOf<T> {
+	0: [];
+	1: [T];
+	2: [T, T];
+	3: [T, T, T];
+	4: [T, T, T, T];
+	5: [T, T, T, T, T];
+	6: [T, T, T, T, T, T];
+	7: [T, T, T, T, T, T, T];
+	8: [T, T, T, T, T, T, T, T];
+	9: [T, T, T, T, T, T, T, T, T];
+	10: [T, T, T, T, T, T, T, T, T, T];
+	11: [T, T, T, T, T, T, T, T, T, T, T];
+	12: [T, T, T, T, T, T, T, T, T, T, T, T];
+}
+
+export type AnyTupleSize = keyof TupleOf<any>;
+
+export type Tuple<T, N extends AnyTupleSize> = TupleOf<T>[N];
+
+export type Numbers<N extends AnyTupleSize> = Tuple<number, N>;
+
+/**
+ * Information about the Autonomous System for the IP range.
+ */
+export interface AsInfo {
+	/**
+	 * AS number. If 0, IP range is not assigned.
+	 */
+	number: number;
+	/**
+	 * 2-digit country code, where present
+	 */
+	country: string | null;
+	/**
+	 * AS description string, where present
+	 */
+	description: string | null;
+}
+
+export interface AsRecord extends AsInfo {
+	offset: number;
+}
+
+/**
+ * IP range info
+ */
+export interface RangeInfo<Int extends number | bigint> {
+	
+	family: 4 | 6;
+	
+	/**
+	 * First address in string form.
+	 */
+	first: string;
+	/**
+	 * Last address in string form.
+	 */
+	last: string;
+	
+	firstInt: Int;
+	lastInt: Int;
+	
+	/**
+	 * Number of addresses.
+	 */
+	size: Int;
+	
+	/**
+	 * Range index in the map, for debugging purposes.
+	 */
+	index: number;
+	
+}
+
+export type IpFamily = 4 | 6;
+export type IntRepr<F extends IpFamily> = F extends 4 ? number : bigint;
+
+/**
+ * IP address lookup result
+ */
+export interface IpInfo<F extends IpFamily = IpFamily> {
+	family: F;
+	ip: string;
+	int: IntRepr<F>;
+	as: AsInfo;
+	range: RangeInfo<IntRepr<F>>;
+}

package/src/utils.ts

@@ -0,0 +1,38 @@
+export function parseIPv6(addr: string) {
+	const [headStr, tailStr] = addr.split('::') as [string, string];
+	const headChunks = headStr.split(':').filter(s => !!s).map(c => c.padStart(4, '0'));
+	const tailChunks = tailStr?.split(':').filter(s => !!s).map(c => c.padStart(4, '0')) ?? [];
+	const full = [
+		'0x',
+		...headChunks,
+		'0'.repeat(4 * (8 - (headChunks.length + tailChunks.length))),
+		...tailChunks,
+	];
+	return BigInt(full.join(''));
+}
+
+export function parseIPv4(addr: string) {
+	return parseInt(
+		addr.split('.').map(s => parseInt(s, 10).toString(16).padStart(2, '0')).join(''),
+		16,
+	);
+}
+
+export function stringifyIP4(int: number) {
+	return [
+		(int >>> 24) & 255,
+		(int >>> 16) & 255,
+		(int >>> 8) & 255,
+		int & 255
+	].join('.');
+}
+
+
+export function stringifyIP6(int: bigint) {
+	const chunks: number[] = [];
+	for (let s = 0; s < 8; s++) {
+		chunks.push(Number(int >> BigInt(s * 16) & 0xffffn));
+	}
+	while (chunks.length && !chunks[0]) chunks.shift();
+	return chunks.toReversed().map(c => c.toString(16)).join(':') + (chunks.length < 8 ? '::' : '');
+}

package/tsconfig.json

@@ -0,0 +1,23 @@
+{
+	"compilerOptions": {
+		"module": "nodenext",
+		"moduleResolution": "nodenext",
+		"target": "esnext",
+		"strict": true,
+		"sourceMap": true,
+		"declaration": true,
+		"noEmitOnError": false,
+		"allowSyntheticDefaultImports": true,
+		"strictNullChecks": true,
+		"noUncheckedIndexedAccess": true,
+		"erasableSyntaxOnly": true,
+		"allowImportingTsExtensions": true,
+		"rewriteRelativeImportExtensions": true,
+		"rootDir": "./src",
+		"outDir": "./dist",
+		"types": ["node"]
+	},
+	"include": [
+		"./src/*.ts",
+	]
+}