@netkitty/analysis 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/LICENSE +21 -0
- package/README.md +150 -0
- package/README.zh-CN.md +146 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.js +27 -0
- package/dist/lib/streaming/Analysis.d.ts +54 -0
- package/dist/lib/streaming/Analysis.js +159 -0
- package/dist/lib/streaming/backends/BrowserFileReadBackend.d.ts +14 -0
- package/dist/lib/streaming/backends/BrowserFileReadBackend.js +36 -0
- package/dist/lib/streaming/backends/NodeFileReadBackend.d.ts +15 -0
- package/dist/lib/streaming/backends/NodeFileReadBackend.js +62 -0
- package/dist/lib/streaming/filter/FilterExpression.d.ts +35 -0
- package/dist/lib/streaming/filter/FilterExpression.js +166 -0
- package/dist/lib/streaming/indexer/ConversationKey.d.ts +18 -0
- package/dist/lib/streaming/indexer/ConversationKey.js +54 -0
- package/dist/lib/streaming/indexer/FrameIndexer.d.ts +16 -0
- package/dist/lib/streaming/indexer/FrameIndexer.js +58 -0
- package/dist/lib/streaming/indexer/PcapIndexBuilder.d.ts +20 -0
- package/dist/lib/streaming/indexer/PcapIndexBuilder.js +52 -0
- package/dist/lib/streaming/interfaces/IAnalysisReducer.d.ts +14 -0
- package/dist/lib/streaming/interfaces/IAnalysisReducer.js +2 -0
- package/dist/lib/streaming/interfaces/IIndexStore.d.ts +16 -0
- package/dist/lib/streaming/interfaces/IIndexStore.js +2 -0
- package/dist/lib/streaming/interfaces/IReadBackend.d.ts +11 -0
- package/dist/lib/streaming/interfaces/IReadBackend.js +2 -0
- package/dist/lib/streaming/interfaces/IWorkerChannel.d.ts +12 -0
- package/dist/lib/streaming/interfaces/IWorkerChannel.js +2 -0
- package/dist/lib/streaming/reducers/ConversationsReducer.d.ts +34 -0
- package/dist/lib/streaming/reducers/ConversationsReducer.js +66 -0
- package/dist/lib/streaming/reducers/EndpointsReducer.d.ts +25 -0
- package/dist/lib/streaming/reducers/EndpointsReducer.js +49 -0
- package/dist/lib/streaming/reducers/ReducerFactories.d.ts +13 -0
- package/dist/lib/streaming/reducers/ReducerFactories.js +42 -0
- package/dist/lib/streaming/reducers/TcpStreamReducer.d.ts +35 -0
- package/dist/lib/streaming/reducers/TcpStreamReducer.js +153 -0
- package/dist/lib/streaming/stores/ColumnarIndexStore.d.ts +21 -0
- package/dist/lib/streaming/stores/ColumnarIndexStore.js +134 -0
- package/dist/lib/streaming/types/AnalysisEvents.d.ts +8 -0
- package/dist/lib/streaming/types/AnalysisEvents.js +2 -0
- package/dist/lib/streaming/types/AnalysisOptions.d.ts +5 -0
- package/dist/lib/streaming/types/AnalysisOptions.js +2 -0
- package/dist/lib/streaming/types/AnalysisPhase.d.ts +2 -0
- package/dist/lib/streaming/types/AnalysisPhase.js +2 -0
- package/dist/lib/streaming/types/AnalysisSource.d.ts +2 -0
- package/dist/lib/streaming/types/AnalysisSource.js +2 -0
- package/dist/lib/streaming/types/Frame.d.ts +7 -0
- package/dist/lib/streaming/types/Frame.js +2 -0
- package/dist/lib/streaming/types/FrameIndexRecord.d.ts +14 -0
- package/dist/lib/streaming/types/FrameIndexRecord.js +2 -0
- package/dist/lib/streaming/types/FrameRow.d.ts +12 -0
- package/dist/lib/streaming/types/FrameRow.js +2 -0
- package/dist/lib/streaming/types/UpdateContext.d.ts +7 -0
- package/dist/lib/streaming/types/UpdateContext.js +2 -0
- package/dist/lib/streaming/worker/AnalysisWorkerCore.d.ts +10 -0
- package/dist/lib/streaming/worker/AnalysisWorkerCore.js +192 -0
- package/dist/lib/streaming/worker/BuiltinScans.d.ts +23 -0
- package/dist/lib/streaming/worker/BuiltinScans.js +121 -0
- package/dist/lib/streaming/worker/IWorkerEndpoint.d.ts +9 -0
- package/dist/lib/streaming/worker/IWorkerEndpoint.js +2 -0
- package/dist/lib/streaming/worker/NodeWorkerChannel.d.ts +16 -0
- package/dist/lib/streaming/worker/NodeWorkerChannel.js +60 -0
- package/dist/lib/streaming/worker/SyntheticFrame.d.ts +8 -0
- package/dist/lib/streaming/worker/SyntheticFrame.js +38 -0
- package/dist/lib/streaming/worker/WebWorkerChannel.d.ts +23 -0
- package/dist/lib/streaming/worker/WebWorkerChannel.js +59 -0
- package/dist/lib/streaming/worker/WebWorkerEndpoint.d.ts +19 -0
- package/dist/lib/streaming/worker/WebWorkerEndpoint.js +44 -0
- package/dist/lib/streaming/worker/WorkerEndpoint.d.ts +13 -0
- package/dist/lib/streaming/worker/WorkerEndpoint.js +45 -0
- package/dist/lib/streaming/worker/WorkerMessage.d.ts +24 -0
- package/dist/lib/streaming/worker/WorkerMessage.js +2 -0
- package/dist/lib/streaming/worker/analysisWorker.d.ts +1 -0
- package/dist/lib/streaming/worker/analysisWorker.js +11 -0
- package/dist/lib/streaming/worker/analysisWorkerBrowser.d.ts +1 -0
- package/dist/lib/streaming/worker/analysisWorkerBrowser.js +12 -0
- package/dist/lib/streaming/worker/spawnNodeAnalysisChannel.d.ts +7 -0
- package/dist/lib/streaming/worker/spawnNodeAnalysisChannel.js +18 -0
- package/package.json +65 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { IReadBackend } from '../interfaces/IReadBackend';
|
|
2
|
+
/**
|
|
3
|
+
* node IReadBackend over a filesystem path: positional pread for random single-frame access, a
|
|
4
|
+
* chunked async iterator for sequential index building, and fs.watch for tailing a growing capture.
|
|
5
|
+
* A single FileHandle is opened lazily and reused for size/read/stream; close() releases it.
|
|
6
|
+
*/
|
|
7
|
+
export declare class NodeFileReadBackend implements IReadBackend {
|
|
8
|
+
#private;
|
|
9
|
+
constructor(path: string, chunkSize?: number);
|
|
10
|
+
size(): Promise<number>;
|
|
11
|
+
read(offset: number, length: number): Promise<Uint8Array>;
|
|
12
|
+
createStream(): AsyncIterableIterator<Uint8Array>;
|
|
13
|
+
watch(onChange: () => void): () => void;
|
|
14
|
+
close(): Promise<void>;
|
|
15
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.NodeFileReadBackend = void 0;
|
|
4
|
+
const promises_1 = require("node:fs/promises");
|
|
5
|
+
const node_fs_1 = require("node:fs");
|
|
6
|
+
/**
|
|
7
|
+
* node IReadBackend over a filesystem path: positional pread for random single-frame access, a
|
|
8
|
+
* chunked async iterator for sequential index building, and fs.watch for tailing a growing capture.
|
|
9
|
+
* A single FileHandle is opened lazily and reused for size/read/stream; close() releases it.
|
|
10
|
+
*/
|
|
11
|
+
class NodeFileReadBackend {
|
|
12
|
+
#path;
|
|
13
|
+
#chunkSize;
|
|
14
|
+
#handle = null;
|
|
15
|
+
constructor(path, chunkSize = 1 << 16) {
|
|
16
|
+
this.#path = path;
|
|
17
|
+
this.#chunkSize = chunkSize;
|
|
18
|
+
}
|
|
19
|
+
async #fileHandle() {
|
|
20
|
+
if (!this.#handle)
|
|
21
|
+
this.#handle = await (0, promises_1.open)(this.#path, 'r');
|
|
22
|
+
return this.#handle;
|
|
23
|
+
}
|
|
24
|
+
async size() {
|
|
25
|
+
return (await (0, promises_1.stat)(this.#path)).size;
|
|
26
|
+
}
|
|
27
|
+
async read(offset, length) {
|
|
28
|
+
const handle = await this.#fileHandle();
|
|
29
|
+
const buffer = Buffer.allocUnsafe(length);
|
|
30
|
+
const result = await handle.read(buffer, 0, length, offset);
|
|
31
|
+
return result.bytesRead === length ? buffer : buffer.subarray(0, result.bytesRead);
|
|
32
|
+
}
|
|
33
|
+
async *createStream() {
|
|
34
|
+
const handle = await this.#fileHandle();
|
|
35
|
+
let position = 0;
|
|
36
|
+
while (true) {
|
|
37
|
+
const buffer = Buffer.allocUnsafe(this.#chunkSize);
|
|
38
|
+
const result = await handle.read(buffer, 0, this.#chunkSize, position);
|
|
39
|
+
if (result.bytesRead <= 0)
|
|
40
|
+
break;
|
|
41
|
+
position += result.bytesRead;
|
|
42
|
+
yield result.bytesRead === this.#chunkSize ? buffer : buffer.subarray(0, result.bytesRead);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
watch(onChange) {
|
|
46
|
+
//stat-polling (watchFile) rather than fs.watch: reliable across platforms for tailing a
|
|
47
|
+
//steadily growing capture, at the cost of a small polling latency.
|
|
48
|
+
const listener = (curr, prev) => {
|
|
49
|
+
if (curr.size !== prev.size || curr.mtimeMs !== prev.mtimeMs)
|
|
50
|
+
onChange();
|
|
51
|
+
};
|
|
52
|
+
(0, node_fs_1.watchFile)(this.#path, { interval: 100 }, listener);
|
|
53
|
+
return () => (0, node_fs_1.unwatchFile)(this.#path, listener);
|
|
54
|
+
}
|
|
55
|
+
async close() {
|
|
56
|
+
if (this.#handle) {
|
|
57
|
+
await this.#handle.close();
|
|
58
|
+
this.#handle = null;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
exports.NodeFileReadBackend = NodeFileReadBackend;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { CodecDecodeResult } from '@netkitty/codec';
|
|
2
|
+
/**
|
|
3
|
+
* A single display-filter term: a bare protocol name (frame contains that layer) or a
|
|
4
|
+
* `<selector> == <value>` field test. v1 covers the common selectors; the expression is the AND of
|
|
5
|
+
* its predicates. Richer syntax (||, !=, ranges) is a later extension.
|
|
6
|
+
*/
|
|
7
|
+
export type FilterPredicate = {
|
|
8
|
+
kind: 'protocol';
|
|
9
|
+
name: string;
|
|
10
|
+
} | {
|
|
11
|
+
kind: 'field';
|
|
12
|
+
selector: string;
|
|
13
|
+
value: string;
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* A parsed display filter: the list of predicates a frame must all satisfy (logical AND). Produced by
|
|
17
|
+
* {@link parseFilter} and consumed by {@link matchesFilter} / {@link matchesIndexed}.
|
|
18
|
+
*/
|
|
19
|
+
export type FilterExpression = FilterPredicate[];
|
|
20
|
+
/** Parse a display filter into an AND-list of predicates. Empty input matches everything. */
|
|
21
|
+
export declare function parseFilter(input: string): FilterExpression;
|
|
22
|
+
/** True when the decoded layers satisfy every predicate (AND). */
|
|
23
|
+
export declare function matchesFilter(layers: CodecDecodeResult[], expression: FilterExpression): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Try to decide a predicate from the index columns alone — the conversation key (protocol + the two
|
|
26
|
+
* canonical endpoints) and the top protocol — with NO re-decode. Returns true/false when decidable,
|
|
27
|
+
* or null when the frame must be decoded: direction-sensitive fields (src/dst/srcport/dstport, since
|
|
28
|
+
* the key is canonicalized and loses direction), or protocols the key/topProtocol don't pin down.
|
|
29
|
+
*/
|
|
30
|
+
export declare function indexableEval(predicate: FilterPredicate, conversationKey: string | null, topProtocol: string, directionForward: number): boolean | null;
|
|
31
|
+
/**
|
|
32
|
+
* AND over indexableEval: false as soon as any column predicate excludes the frame (no decode); true
|
|
33
|
+
* if every predicate is column-decided true; null if a predicate needs the decoded layers to confirm.
|
|
34
|
+
*/
|
|
35
|
+
export declare function matchesIndexed(expression: FilterExpression, conversationKey: string | null, topProtocol: string, directionForward: number): boolean | null;
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseFilter = parseFilter;
|
|
4
|
+
exports.matchesFilter = matchesFilter;
|
|
5
|
+
exports.indexableEval = indexableEval;
|
|
6
|
+
exports.matchesIndexed = matchesIndexed;
|
|
7
|
+
/** Parse a display filter into an AND-list of predicates. Empty input matches everything. */
|
|
8
|
+
function parseFilter(input) {
|
|
9
|
+
return input
|
|
10
|
+
.split('&&')
|
|
11
|
+
.map((part) => part.trim())
|
|
12
|
+
.filter((part) => part.length > 0)
|
|
13
|
+
.map(parsePredicate);
|
|
14
|
+
}
|
|
15
|
+
function parsePredicate(part) {
|
|
16
|
+
const equals = part.indexOf('==');
|
|
17
|
+
if (equals < 0)
|
|
18
|
+
return { kind: 'protocol', name: part.toLowerCase() };
|
|
19
|
+
const selector = part.slice(0, equals).trim().toLowerCase();
|
|
20
|
+
let value = part.slice(equals + 2).trim();
|
|
21
|
+
if (value.length >= 2 && ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith('\'') && value.endsWith('\'')))) {
|
|
22
|
+
value = value.slice(1, -1);
|
|
23
|
+
}
|
|
24
|
+
return { kind: 'field', selector: selector, value: value };
|
|
25
|
+
}
|
|
26
|
+
/** True when the decoded layers satisfy every predicate (AND). */
|
|
27
|
+
function matchesFilter(layers, expression) {
|
|
28
|
+
return expression.every((predicate) => matchesPredicate(layers, predicate));
|
|
29
|
+
}
|
|
30
|
+
function layerOf(layers, ...ids) {
|
|
31
|
+
return layers.find((l) => ids.includes(l.id));
|
|
32
|
+
}
|
|
33
|
+
function field(layer, name) {
|
|
34
|
+
if (!layer)
|
|
35
|
+
return null;
|
|
36
|
+
const value = layer.data[name];
|
|
37
|
+
return value === undefined || value === null ? null : String(value);
|
|
38
|
+
}
|
|
39
|
+
function matchesPredicate(layers, predicate) {
|
|
40
|
+
if (predicate.kind === 'protocol')
|
|
41
|
+
return layers.some((l) => l.id === predicate.name);
|
|
42
|
+
const value = predicate.value;
|
|
43
|
+
switch (predicate.selector) {
|
|
44
|
+
case 'ip.addr': return anyEquals(value, field(layerOf(layers, 'ipv4', 'ipv6'), 'sip'), field(layerOf(layers, 'ipv4', 'ipv6'), 'dip'));
|
|
45
|
+
case 'ip.src': return field(layerOf(layers, 'ipv4', 'ipv6'), 'sip') === value;
|
|
46
|
+
case 'ip.dst': return field(layerOf(layers, 'ipv4', 'ipv6'), 'dip') === value;
|
|
47
|
+
case 'tcp.port': return anyEquals(value, field(layerOf(layers, 'tcp'), 'srcport'), field(layerOf(layers, 'tcp'), 'dstport'));
|
|
48
|
+
case 'tcp.srcport': return field(layerOf(layers, 'tcp'), 'srcport') === value;
|
|
49
|
+
case 'tcp.dstport': return field(layerOf(layers, 'tcp'), 'dstport') === value;
|
|
50
|
+
case 'udp.port': return anyEquals(value, field(layerOf(layers, 'udp'), 'srcport'), field(layerOf(layers, 'udp'), 'dstport'));
|
|
51
|
+
case 'udp.srcport': return field(layerOf(layers, 'udp'), 'srcport') === value;
|
|
52
|
+
case 'udp.dstport': return field(layerOf(layers, 'udp'), 'dstport') === value;
|
|
53
|
+
case 'eth.addr': return anyEquals(value, field(layerOf(layers, 'eth'), 'smac'), field(layerOf(layers, 'eth'), 'dmac'));
|
|
54
|
+
case 'eth.src': return field(layerOf(layers, 'eth'), 'smac') === value;
|
|
55
|
+
case 'eth.dst': return field(layerOf(layers, 'eth'), 'dmac') === value;
|
|
56
|
+
default: return false;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function anyEquals(value, a, b) {
|
|
60
|
+
return a === value || b === value;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Try to decide a predicate from the index columns alone — the conversation key (protocol + the two
|
|
64
|
+
* canonical endpoints) and the top protocol — with NO re-decode. Returns true/false when decidable,
|
|
65
|
+
* or null when the frame must be decoded: direction-sensitive fields (src/dst/srcport/dstport, since
|
|
66
|
+
* the key is canonicalized and loses direction), or protocols the key/topProtocol don't pin down.
|
|
67
|
+
*/
|
|
68
|
+
function indexableEval(predicate, conversationKey, topProtocol, directionForward) {
|
|
69
|
+
if (predicate.kind === 'protocol') {
|
|
70
|
+
if (predicate.name === 'tcp' || predicate.name === 'udp') {
|
|
71
|
+
if (conversationKey === null)
|
|
72
|
+
return false;
|
|
73
|
+
return conversationKey.slice(0, conversationKey.indexOf('|')) === predicate.name;
|
|
74
|
+
}
|
|
75
|
+
if (predicate.name === 'arp')
|
|
76
|
+
return topProtocol === 'arp';
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
if (conversationKey === null)
|
|
80
|
+
return false;
|
|
81
|
+
const bar1 = conversationKey.indexOf('|');
|
|
82
|
+
const bar2 = conversationKey.indexOf('|', bar1 + 1);
|
|
83
|
+
const proto = conversationKey.slice(0, bar1);
|
|
84
|
+
const endpointA = conversationKey.slice(bar1 + 1, bar2);
|
|
85
|
+
const endpointB = conversationKey.slice(bar2 + 1);
|
|
86
|
+
//directionForward=1 → source is endpointA; else source is endpointB.
|
|
87
|
+
const source = directionForward ? endpointA : endpointB;
|
|
88
|
+
const destination = directionForward ? endpointB : endpointA;
|
|
89
|
+
switch (predicate.selector) {
|
|
90
|
+
case 'ip.addr':
|
|
91
|
+
if (proto !== 'tcp' && proto !== 'udp' && proto !== 'ip')
|
|
92
|
+
return false;
|
|
93
|
+
return anyEquals(predicate.value, ipOf(endpointA, proto), ipOf(endpointB, proto));
|
|
94
|
+
case 'ip.src':
|
|
95
|
+
if (proto !== 'tcp' && proto !== 'udp' && proto !== 'ip')
|
|
96
|
+
return false;
|
|
97
|
+
return ipOf(source, proto) === predicate.value;
|
|
98
|
+
case 'ip.dst':
|
|
99
|
+
if (proto !== 'tcp' && proto !== 'udp' && proto !== 'ip')
|
|
100
|
+
return false;
|
|
101
|
+
return ipOf(destination, proto) === predicate.value;
|
|
102
|
+
case 'tcp.port':
|
|
103
|
+
if (proto !== 'tcp')
|
|
104
|
+
return false;
|
|
105
|
+
return anyEquals(predicate.value, portOf(endpointA), portOf(endpointB));
|
|
106
|
+
case 'tcp.srcport':
|
|
107
|
+
if (proto !== 'tcp')
|
|
108
|
+
return false;
|
|
109
|
+
return portOf(source) === predicate.value;
|
|
110
|
+
case 'tcp.dstport':
|
|
111
|
+
if (proto !== 'tcp')
|
|
112
|
+
return false;
|
|
113
|
+
return portOf(destination) === predicate.value;
|
|
114
|
+
case 'udp.port':
|
|
115
|
+
if (proto !== 'udp')
|
|
116
|
+
return false;
|
|
117
|
+
return anyEquals(predicate.value, portOf(endpointA), portOf(endpointB));
|
|
118
|
+
case 'udp.srcport':
|
|
119
|
+
if (proto !== 'udp')
|
|
120
|
+
return false;
|
|
121
|
+
return portOf(source) === predicate.value;
|
|
122
|
+
case 'udp.dstport':
|
|
123
|
+
if (proto !== 'udp')
|
|
124
|
+
return false;
|
|
125
|
+
return portOf(destination) === predicate.value;
|
|
126
|
+
case 'eth.addr':
|
|
127
|
+
if (proto !== 'eth')
|
|
128
|
+
return false;
|
|
129
|
+
return anyEquals(predicate.value, endpointA, endpointB);
|
|
130
|
+
case 'eth.src':
|
|
131
|
+
if (proto !== 'eth')
|
|
132
|
+
return false;
|
|
133
|
+
return source === predicate.value;
|
|
134
|
+
case 'eth.dst':
|
|
135
|
+
if (proto !== 'eth')
|
|
136
|
+
return false;
|
|
137
|
+
return destination === predicate.value;
|
|
138
|
+
default:
|
|
139
|
+
return null;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* AND over indexableEval: false as soon as any column predicate excludes the frame (no decode); true
|
|
144
|
+
* if every predicate is column-decided true; null if a predicate needs the decoded layers to confirm.
|
|
145
|
+
*/
|
|
146
|
+
function matchesIndexed(expression, conversationKey, topProtocol, directionForward) {
|
|
147
|
+
let needsDecode = false;
|
|
148
|
+
for (const predicate of expression) {
|
|
149
|
+
const decided = indexableEval(predicate, conversationKey, topProtocol, directionForward);
|
|
150
|
+
if (decided === false)
|
|
151
|
+
return false;
|
|
152
|
+
if (decided === null)
|
|
153
|
+
needsDecode = true;
|
|
154
|
+
}
|
|
155
|
+
return needsDecode ? null : true;
|
|
156
|
+
}
|
|
157
|
+
function ipOf(endpoint, proto) {
|
|
158
|
+
if (proto === 'ip')
|
|
159
|
+
return endpoint;
|
|
160
|
+
const colon = endpoint.lastIndexOf(':');
|
|
161
|
+
return colon < 0 ? endpoint : endpoint.slice(0, colon);
|
|
162
|
+
}
|
|
163
|
+
function portOf(endpoint) {
|
|
164
|
+
const colon = endpoint.lastIndexOf(':');
|
|
165
|
+
return colon < 0 ? '' : endpoint.slice(colon + 1);
|
|
166
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { CodecDecodeResult } from '@netkitty/codec';
|
|
2
|
+
/** The directional n-tuple of a frame, before canonicalization. */
|
|
3
|
+
export type ConversationFlow = {
|
|
4
|
+
protocol: string;
|
|
5
|
+
source: string;
|
|
6
|
+
destination: string;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Derive the conversation n-tuple from a frame's decoded layers. Prefers IP + transport (tcp/udp),
|
|
10
|
+
* falls back to bare IP, then Ethernet MACs.
|
|
11
|
+
*/
|
|
12
|
+
export declare function flowOf(layers: CodecDecodeResult[]): ConversationFlow | null;
|
|
13
|
+
/** Canonical, direction-independent key so A→B and B→A collapse to one conversation. */
|
|
14
|
+
export declare function canonicalConversationKey(flow: ConversationFlow): string;
|
|
15
|
+
/** The innermost real protocol id (last layer with protocol === true), skipping RawData tails. */
|
|
16
|
+
export declare function topProtocolOf(layers: CodecDecodeResult[]): string;
|
|
17
|
+
/** FNV-1a 32-bit hash — a compact numeric key for the conversation column. */
|
|
18
|
+
export declare function hash32(input: string): number;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.flowOf = flowOf;
|
|
4
|
+
exports.canonicalConversationKey = canonicalConversationKey;
|
|
5
|
+
exports.topProtocolOf = topProtocolOf;
|
|
6
|
+
exports.hash32 = hash32;
|
|
7
|
+
/**
|
|
8
|
+
* Derive the conversation n-tuple from a frame's decoded layers. Prefers IP + transport (tcp/udp),
|
|
9
|
+
* falls back to bare IP, then Ethernet MACs.
|
|
10
|
+
*/
|
|
11
|
+
function flowOf(layers) {
|
|
12
|
+
const ip = layers.find((l) => l.id === 'ipv4' || l.id === 'ipv6');
|
|
13
|
+
if (ip) {
|
|
14
|
+
const source = String(ip.data.sip);
|
|
15
|
+
const destination = String(ip.data.dip);
|
|
16
|
+
const transport = layers.find((l) => l.id === 'tcp' || l.id === 'udp');
|
|
17
|
+
if (transport) {
|
|
18
|
+
return {
|
|
19
|
+
protocol: transport.id,
|
|
20
|
+
source: `${source}:${transport.data.srcport}`,
|
|
21
|
+
destination: `${destination}:${transport.data.dstport}`
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
return { protocol: 'ip', source: source, destination: destination };
|
|
25
|
+
}
|
|
26
|
+
const eth = layers.find((l) => l.id === 'eth');
|
|
27
|
+
if (eth)
|
|
28
|
+
return { protocol: 'eth', source: String(eth.data.smac), destination: String(eth.data.dmac) };
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
/** Canonical, direction-independent key so A→B and B→A collapse to one conversation. */
|
|
32
|
+
function canonicalConversationKey(flow) {
|
|
33
|
+
const forward = flow.source <= flow.destination;
|
|
34
|
+
const a = forward ? flow.source : flow.destination;
|
|
35
|
+
const b = forward ? flow.destination : flow.source;
|
|
36
|
+
return `${flow.protocol}|${a}|${b}`;
|
|
37
|
+
}
|
|
38
|
+
/** The innermost real protocol id (last layer with protocol === true), skipping RawData tails. */
|
|
39
|
+
function topProtocolOf(layers) {
|
|
40
|
+
for (let i = layers.length - 1; i >= 0; i--) {
|
|
41
|
+
if (layers[i].protocol)
|
|
42
|
+
return layers[i].id;
|
|
43
|
+
}
|
|
44
|
+
return layers.length > 0 ? layers[layers.length - 1].id : 'unknown';
|
|
45
|
+
}
|
|
46
|
+
/** FNV-1a 32-bit hash — a compact numeric key for the conversation column. */
|
|
47
|
+
function hash32(input) {
|
|
48
|
+
let hash = 0x811c9dc5;
|
|
49
|
+
for (let i = 0; i < input.length; i++) {
|
|
50
|
+
hash ^= input.charCodeAt(i);
|
|
51
|
+
hash = (hash + ((hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24))) >>> 0;
|
|
52
|
+
}
|
|
53
|
+
return hash >>> 0;
|
|
54
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { CodecDecodeResult } from '@netkitty/codec';
|
|
2
|
+
import { IIndexStore } from '../interfaces/IIndexStore';
|
|
3
|
+
/**
|
|
4
|
+
* Turns a frame's decoded layers + capture metadata into a columnar index record, maintaining the two
|
|
5
|
+
* small side dictionaries the numeric columns reference: protocol name↔id and conversation hash→key.
|
|
6
|
+
* Both are per-protocol / per-conversation sized (far smaller than the frame count), so they stay in
|
|
7
|
+
* memory while the bulk per-frame data lives in the TypedArray columns. conversationHash 0 is reserved
|
|
8
|
+
* for frames with no derivable n-tuple.
|
|
9
|
+
*/
|
|
10
|
+
export declare class FrameIndexer {
|
|
11
|
+
#private;
|
|
12
|
+
constructor(store: IIndexStore);
|
|
13
|
+
add(layers: CodecDecodeResult[], fileOffset: number, capturedLength: number, originalLength: number, timestamp: number): number;
|
|
14
|
+
protocolName(id: number): string;
|
|
15
|
+
conversationKey(hash: number): string | null;
|
|
16
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.FrameIndexer = void 0;
|
|
4
|
+
const ConversationKey_1 = require("./ConversationKey");
|
|
5
|
+
/**
|
|
6
|
+
* Turns a frame's decoded layers + capture metadata into a columnar index record, maintaining the two
|
|
7
|
+
* small side dictionaries the numeric columns reference: protocol name↔id and conversation hash→key.
|
|
8
|
+
* Both are per-protocol / per-conversation sized (far smaller than the frame count), so they stay in
|
|
9
|
+
* memory while the bulk per-frame data lives in the TypedArray columns. conversationHash 0 is reserved
|
|
10
|
+
* for frames with no derivable n-tuple.
|
|
11
|
+
*/
|
|
12
|
+
class FrameIndexer {
|
|
13
|
+
#store;
|
|
14
|
+
#protocolIds = new Map();
|
|
15
|
+
#protocolNames = [];
|
|
16
|
+
#conversationKeys = new Map();
|
|
17
|
+
constructor(store) {
|
|
18
|
+
this.#store = store;
|
|
19
|
+
}
|
|
20
|
+
add(layers, fileOffset, capturedLength, originalLength, timestamp) {
|
|
21
|
+
const flow = (0, ConversationKey_1.flowOf)(layers);
|
|
22
|
+
const key = flow ? (0, ConversationKey_1.canonicalConversationKey)(flow) : null;
|
|
23
|
+
const conversationHash = key !== null ? (0, ConversationKey_1.hash32)(key) : 0;
|
|
24
|
+
if (key !== null && !this.#conversationKeys.has(conversationHash))
|
|
25
|
+
this.#conversationKeys.set(conversationHash, key);
|
|
26
|
+
//Forward = the original source is the canonical endpointA (source <= destination), matching
|
|
27
|
+
//canonicalConversationKey. Lets src/dst filters recover direction from the key without decoding.
|
|
28
|
+
const directionForward = flow ? (flow.source <= flow.destination ? 1 : 0) : 1;
|
|
29
|
+
const record = {
|
|
30
|
+
index: -1,
|
|
31
|
+
fileOffset: fileOffset,
|
|
32
|
+
capturedLength: capturedLength,
|
|
33
|
+
originalLength: originalLength,
|
|
34
|
+
timestamp: timestamp,
|
|
35
|
+
protocolId: this.#protocolId((0, ConversationKey_1.topProtocolOf)(layers)),
|
|
36
|
+
conversationHash: conversationHash,
|
|
37
|
+
directionForward: directionForward
|
|
38
|
+
};
|
|
39
|
+
return this.#store.append(record);
|
|
40
|
+
}
|
|
41
|
+
protocolName(id) {
|
|
42
|
+
return id >= 0 && id < this.#protocolNames.length ? this.#protocolNames[id] : 'unknown';
|
|
43
|
+
}
|
|
44
|
+
conversationKey(hash) {
|
|
45
|
+
const key = this.#conversationKeys.get(hash);
|
|
46
|
+
return key !== undefined ? key : null;
|
|
47
|
+
}
|
|
48
|
+
#protocolId(name) {
|
|
49
|
+
const existing = this.#protocolIds.get(name);
|
|
50
|
+
if (existing !== undefined)
|
|
51
|
+
return existing;
|
|
52
|
+
const id = this.#protocolNames.length;
|
|
53
|
+
this.#protocolIds.set(name, id);
|
|
54
|
+
this.#protocolNames.push(name);
|
|
55
|
+
return id;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
exports.FrameIndexer = FrameIndexer;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { Codec } from '@netkitty/codec';
|
|
2
|
+
import { IPcapPacketInfo } from '@netkitty/pcap-core';
|
|
3
|
+
import { IReadBackend } from '../interfaces/IReadBackend';
|
|
4
|
+
import { FrameIndexer } from './FrameIndexer';
|
|
5
|
+
/** Callbacks fired while building the index, for progress reporting and per-frame notification. */
|
|
6
|
+
export type IndexBuildHooks = {
|
|
7
|
+
onFrame?: (index: number, info: IPcapPacketInfo) => void;
|
|
8
|
+
onProgress?: (bytesRead: number, totalBytes: number) => void;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Drives the index build: pulls bytes from an IReadBackend stream, feeds PcapParserCore, and for each
|
|
12
|
+
* emitted frame runs codec.decode to extract the n-tuple and append a columnar record via FrameIndexer.
|
|
13
|
+
* Frames produced by one chunk are drained (decoded + indexed) before the next chunk is read, so memory
|
|
14
|
+
* stays bounded to one chunk's worth of frames and the async decode naturally back-pressures the reader.
|
|
15
|
+
*/
|
|
16
|
+
export declare class PcapIndexBuilder {
|
|
17
|
+
#private;
|
|
18
|
+
constructor(indexer: FrameIndexer, codec: Codec);
|
|
19
|
+
build(backend: IReadBackend, hooks?: IndexBuildHooks): Promise<number>;
|
|
20
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.PcapIndexBuilder = void 0;
|
|
4
|
+
const pcap_core_1 = require("@netkitty/pcap-core");
|
|
5
|
+
/**
|
|
6
|
+
* Drives the index build: pulls bytes from an IReadBackend stream, feeds PcapParserCore, and for each
|
|
7
|
+
* emitted frame runs codec.decode to extract the n-tuple and append a columnar record via FrameIndexer.
|
|
8
|
+
* Frames produced by one chunk are drained (decoded + indexed) before the next chunk is read, so memory
|
|
9
|
+
* stays bounded to one chunk's worth of frames and the async decode naturally back-pressures the reader.
|
|
10
|
+
*/
|
|
11
|
+
class PcapIndexBuilder {
|
|
12
|
+
#indexer;
|
|
13
|
+
#codec;
|
|
14
|
+
constructor(indexer, codec) {
|
|
15
|
+
this.#indexer = indexer;
|
|
16
|
+
this.#codec = codec;
|
|
17
|
+
}
|
|
18
|
+
async build(backend, hooks = {}) {
|
|
19
|
+
const total = await backend.size();
|
|
20
|
+
let bytesRead = 0;
|
|
21
|
+
let count = 0;
|
|
22
|
+
const pending = [];
|
|
23
|
+
let lastData = Buffer.alloc(0);
|
|
24
|
+
const parser = new pcap_core_1.PcapParserCore({
|
|
25
|
+
onPacketData: (data) => { lastData = data; },
|
|
26
|
+
onPacket: (info) => { pending.push({ info: info, data: lastData }); }
|
|
27
|
+
});
|
|
28
|
+
const drain = async () => {
|
|
29
|
+
for (const frame of pending) {
|
|
30
|
+
const layers = await this.#codec.decode(frame.data);
|
|
31
|
+
const timestamp = frame.info.seconds + frame.info.microseconds / 1_000_000;
|
|
32
|
+
const index = this.#indexer.add(layers, frame.info.packetOffset, frame.info.packetLength, frame.info.packetLength, timestamp);
|
|
33
|
+
if (hooks.onFrame)
|
|
34
|
+
hooks.onFrame(index, frame.info);
|
|
35
|
+
count++;
|
|
36
|
+
}
|
|
37
|
+
pending.length = 0;
|
|
38
|
+
};
|
|
39
|
+
for await (const chunk of backend.createStream()) {
|
|
40
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
41
|
+
parser.write(buffer);
|
|
42
|
+
await drain();
|
|
43
|
+
bytesRead += buffer.length;
|
|
44
|
+
if (hooks.onProgress)
|
|
45
|
+
hooks.onProgress(bytesRead, total);
|
|
46
|
+
}
|
|
47
|
+
parser.end();
|
|
48
|
+
await drain();
|
|
49
|
+
return count;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
exports.PcapIndexBuilder = PcapIndexBuilder;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { Frame } from '../types/Frame';
|
|
2
|
+
import { UpdateContext } from '../types/UpdateContext';
|
|
3
|
+
/**
|
|
4
|
+
* A pluggable rolling analysis over the frame stream. Fed every frame (replay backlog first, then
|
|
5
|
+
* live) via update(); result() returns the current snapshot at any time — there is no finalize. A
|
|
6
|
+
* reducer may additionally extend an event emitter to push progress/update notifications of its own.
|
|
7
|
+
*/
|
|
8
|
+
export interface IAnalysisReducer<TResult> {
|
|
9
|
+
readonly needs?: string[];
|
|
10
|
+
readonly indexOnly?: boolean;
|
|
11
|
+
update(frame: Frame, context: UpdateContext): void;
|
|
12
|
+
result(): TResult;
|
|
13
|
+
reset(): void;
|
|
14
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { FrameIndexRecord } from '../types/FrameIndexRecord';
|
|
2
|
+
/**
|
|
3
|
+
* Append-only frame index with random access, range slicing, predicate scan, and FIFO eviction. The
|
|
4
|
+
* v1 implementation is columnar TypedArray (heap-external, shared by node and browser); a SQLite
|
|
5
|
+
* backend is a v2 seam for when the index outgrows memory or needs persistence.
|
|
6
|
+
*/
|
|
7
|
+
export interface IIndexStore {
|
|
8
|
+
append(record: FrameIndexRecord): number;
|
|
9
|
+
get(index: number): FrameIndexRecord | null;
|
|
10
|
+
range(from: number, to: number): FrameIndexRecord[];
|
|
11
|
+
scan(predicate: (record: FrameIndexRecord) => boolean): number[];
|
|
12
|
+
count(): number;
|
|
13
|
+
firstIndex(): number;
|
|
14
|
+
evictOldest(count: number): void;
|
|
15
|
+
clear(): void;
|
|
16
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Byte-I/O seam over a capture source. The only place node (FileHandle pread / fs.watch) and browser
|
|
3
|
+
* (File.slice / OPFS) diverge; everything above consumes bytes environment-agnostically.
|
|
4
|
+
*/
|
|
5
|
+
export interface IReadBackend {
|
|
6
|
+
size(): Promise<number>;
|
|
7
|
+
read(offset: number, length: number): Promise<Uint8Array>;
|
|
8
|
+
createStream(): AsyncIterable<Uint8Array>;
|
|
9
|
+
watch?(onChange: () => void): () => void;
|
|
10
|
+
close?(): void | Promise<void>;
|
|
11
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thin transport seam to the analysis worker: correlated request/response plus fire-and-forget
|
|
3
|
+
* notify, over worker_threads (node) or Web Worker (browser) postMessage. ArrayBuffers in the
|
|
4
|
+
* transfer list move without copying. Only the invoke/notify semantics are borrowed from capture's
|
|
5
|
+
* pipe layer — the transport itself is plain postMessage, not socket-ipc.
|
|
6
|
+
*/
|
|
7
|
+
export interface IWorkerChannel {
|
|
8
|
+
request<T>(method: string, payload?: unknown, transfer?: ArrayBuffer[]): Promise<T>;
|
|
9
|
+
notify(method: string, payload?: unknown, transfer?: ArrayBuffer[]): void;
|
|
10
|
+
on(method: string, handler: (payload: unknown) => void): void;
|
|
11
|
+
terminate(): void;
|
|
12
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { Frame } from '../types/Frame';
|
|
2
|
+
import { UpdateContext } from '../types/UpdateContext';
|
|
3
|
+
import { IAnalysisReducer } from '../interfaces/IAnalysisReducer';
|
|
4
|
+
/**
|
|
5
|
+
* A bidirectional conversation summary. Instead of holding the full member-frame list, it keeps only
|
|
6
|
+
* firstIndex/lastIndex — the member frames are recovered from the index layer (filter/range) on
|
|
7
|
+
* demand, so per-conversation state stays bounded no matter how many frames it spans.
|
|
8
|
+
*/
|
|
9
|
+
export type ConversationSummary = {
|
|
10
|
+
protocol: string;
|
|
11
|
+
endpointA: string;
|
|
12
|
+
endpointB: string;
|
|
13
|
+
packets: number;
|
|
14
|
+
bytes: number;
|
|
15
|
+
packetsAToB: number;
|
|
16
|
+
packetsBToA: number;
|
|
17
|
+
firstTimestamp: number;
|
|
18
|
+
lastTimestamp: number;
|
|
19
|
+
firstIndex: number;
|
|
20
|
+
lastIndex: number;
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* Rolling conversation table: groups frames direction-independently by n-tuple, tallying packets,
|
|
24
|
+
* bytes, per-direction counts and time/index spans. result() is a snapshot at any point; reset()
|
|
25
|
+
* clears it.
|
|
26
|
+
*/
|
|
27
|
+
export declare class ConversationsReducer implements IAnalysisReducer<ConversationSummary[]> {
|
|
28
|
+
#private;
|
|
29
|
+
readonly needs: string[];
|
|
30
|
+
readonly indexOnly: boolean;
|
|
31
|
+
update(frame: Frame, context: UpdateContext): void;
|
|
32
|
+
result(): ConversationSummary[];
|
|
33
|
+
reset(): void;
|
|
34
|
+
}
|