@netkitty/analysis 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/LICENSE +21 -0
- package/README.md +150 -0
- package/README.zh-CN.md +146 -0
- package/dist/index.d.ts +30 -0
- package/dist/index.js +27 -0
- package/dist/lib/streaming/Analysis.d.ts +54 -0
- package/dist/lib/streaming/Analysis.js +159 -0
- package/dist/lib/streaming/backends/BrowserFileReadBackend.d.ts +14 -0
- package/dist/lib/streaming/backends/BrowserFileReadBackend.js +36 -0
- package/dist/lib/streaming/backends/NodeFileReadBackend.d.ts +15 -0
- package/dist/lib/streaming/backends/NodeFileReadBackend.js +62 -0
- package/dist/lib/streaming/filter/FilterExpression.d.ts +35 -0
- package/dist/lib/streaming/filter/FilterExpression.js +166 -0
- package/dist/lib/streaming/indexer/ConversationKey.d.ts +18 -0
- package/dist/lib/streaming/indexer/ConversationKey.js +54 -0
- package/dist/lib/streaming/indexer/FrameIndexer.d.ts +16 -0
- package/dist/lib/streaming/indexer/FrameIndexer.js +58 -0
- package/dist/lib/streaming/indexer/PcapIndexBuilder.d.ts +20 -0
- package/dist/lib/streaming/indexer/PcapIndexBuilder.js +52 -0
- package/dist/lib/streaming/interfaces/IAnalysisReducer.d.ts +14 -0
- package/dist/lib/streaming/interfaces/IAnalysisReducer.js +2 -0
- package/dist/lib/streaming/interfaces/IIndexStore.d.ts +16 -0
- package/dist/lib/streaming/interfaces/IIndexStore.js +2 -0
- package/dist/lib/streaming/interfaces/IReadBackend.d.ts +11 -0
- package/dist/lib/streaming/interfaces/IReadBackend.js +2 -0
- package/dist/lib/streaming/interfaces/IWorkerChannel.d.ts +12 -0
- package/dist/lib/streaming/interfaces/IWorkerChannel.js +2 -0
- package/dist/lib/streaming/reducers/ConversationsReducer.d.ts +34 -0
- package/dist/lib/streaming/reducers/ConversationsReducer.js +66 -0
- package/dist/lib/streaming/reducers/EndpointsReducer.d.ts +25 -0
- package/dist/lib/streaming/reducers/EndpointsReducer.js +49 -0
- package/dist/lib/streaming/reducers/ReducerFactories.d.ts +13 -0
- package/dist/lib/streaming/reducers/ReducerFactories.js +42 -0
- package/dist/lib/streaming/reducers/TcpStreamReducer.d.ts +35 -0
- package/dist/lib/streaming/reducers/TcpStreamReducer.js +153 -0
- package/dist/lib/streaming/stores/ColumnarIndexStore.d.ts +21 -0
- package/dist/lib/streaming/stores/ColumnarIndexStore.js +134 -0
- package/dist/lib/streaming/types/AnalysisEvents.d.ts +8 -0
- package/dist/lib/streaming/types/AnalysisEvents.js +2 -0
- package/dist/lib/streaming/types/AnalysisOptions.d.ts +5 -0
- package/dist/lib/streaming/types/AnalysisOptions.js +2 -0
- package/dist/lib/streaming/types/AnalysisPhase.d.ts +2 -0
- package/dist/lib/streaming/types/AnalysisPhase.js +2 -0
- package/dist/lib/streaming/types/AnalysisSource.d.ts +2 -0
- package/dist/lib/streaming/types/AnalysisSource.js +2 -0
- package/dist/lib/streaming/types/Frame.d.ts +7 -0
- package/dist/lib/streaming/types/Frame.js +2 -0
- package/dist/lib/streaming/types/FrameIndexRecord.d.ts +14 -0
- package/dist/lib/streaming/types/FrameIndexRecord.js +2 -0
- package/dist/lib/streaming/types/FrameRow.d.ts +12 -0
- package/dist/lib/streaming/types/FrameRow.js +2 -0
- package/dist/lib/streaming/types/UpdateContext.d.ts +7 -0
- package/dist/lib/streaming/types/UpdateContext.js +2 -0
- package/dist/lib/streaming/worker/AnalysisWorkerCore.d.ts +10 -0
- package/dist/lib/streaming/worker/AnalysisWorkerCore.js +192 -0
- package/dist/lib/streaming/worker/BuiltinScans.d.ts +23 -0
- package/dist/lib/streaming/worker/BuiltinScans.js +121 -0
- package/dist/lib/streaming/worker/IWorkerEndpoint.d.ts +9 -0
- package/dist/lib/streaming/worker/IWorkerEndpoint.js +2 -0
- package/dist/lib/streaming/worker/NodeWorkerChannel.d.ts +16 -0
- package/dist/lib/streaming/worker/NodeWorkerChannel.js +60 -0
- package/dist/lib/streaming/worker/SyntheticFrame.d.ts +8 -0
- package/dist/lib/streaming/worker/SyntheticFrame.js +38 -0
- package/dist/lib/streaming/worker/WebWorkerChannel.d.ts +23 -0
- package/dist/lib/streaming/worker/WebWorkerChannel.js +59 -0
- package/dist/lib/streaming/worker/WebWorkerEndpoint.d.ts +19 -0
- package/dist/lib/streaming/worker/WebWorkerEndpoint.js +44 -0
- package/dist/lib/streaming/worker/WorkerEndpoint.d.ts +13 -0
- package/dist/lib/streaming/worker/WorkerEndpoint.js +45 -0
- package/dist/lib/streaming/worker/WorkerMessage.d.ts +24 -0
- package/dist/lib/streaming/worker/WorkerMessage.js +2 -0
- package/dist/lib/streaming/worker/analysisWorker.d.ts +1 -0
- package/dist/lib/streaming/worker/analysisWorker.js +11 -0
- package/dist/lib/streaming/worker/analysisWorkerBrowser.d.ts +1 -0
- package/dist/lib/streaming/worker/analysisWorkerBrowser.js +12 -0
- package/dist/lib/streaming/worker/spawnNodeAnalysisChannel.d.ts +7 -0
- package/dist/lib/streaming/worker/spawnNodeAnalysisChannel.js +18 -0
- package/package.json +65 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ConversationsReducer = void 0;
|
|
4
|
+
const ConversationKey_1 = require("../indexer/ConversationKey");
|
|
5
|
+
/**
|
|
6
|
+
* Rolling conversation table: groups frames direction-independently by n-tuple, tallying packets,
|
|
7
|
+
* bytes, per-direction counts and time/index spans. result() is a snapshot at any point; reset()
|
|
8
|
+
* clears it.
|
|
9
|
+
*/
|
|
10
|
+
class ConversationsReducer {
|
|
11
|
+
constructor() {
|
|
12
|
+
this.needs = ['eth', 'ipv4', 'ipv6', 'tcp', 'udp', 'arp'];
|
|
13
|
+
//Only the five-tuple — replay can feed frames synthesized from the index columns (no re-decode).
|
|
14
|
+
this.indexOnly = true;
|
|
15
|
+
this.#conversations = new Map();
|
|
16
|
+
}
|
|
17
|
+
#conversations;
|
|
18
|
+
update(frame, context) {
|
|
19
|
+
void context;
|
|
20
|
+
const flow = (0, ConversationKey_1.flowOf)(frame.layers);
|
|
21
|
+
if (!flow)
|
|
22
|
+
return;
|
|
23
|
+
const forward = flow.source <= flow.destination;
|
|
24
|
+
const endpointA = forward ? flow.source : flow.destination;
|
|
25
|
+
const endpointB = forward ? flow.destination : flow.source;
|
|
26
|
+
const key = `${flow.protocol}|${endpointA}|${endpointB}`;
|
|
27
|
+
let conversation = this.#conversations.get(key);
|
|
28
|
+
if (!conversation) {
|
|
29
|
+
conversation = {
|
|
30
|
+
protocol: flow.protocol,
|
|
31
|
+
endpointA: endpointA,
|
|
32
|
+
endpointB: endpointB,
|
|
33
|
+
packets: 0,
|
|
34
|
+
bytes: 0,
|
|
35
|
+
packetsAToB: 0,
|
|
36
|
+
packetsBToA: 0,
|
|
37
|
+
firstTimestamp: frame.timestamp,
|
|
38
|
+
lastTimestamp: frame.timestamp,
|
|
39
|
+
firstIndex: frame.index,
|
|
40
|
+
lastIndex: frame.index
|
|
41
|
+
};
|
|
42
|
+
this.#conversations.set(key, conversation);
|
|
43
|
+
}
|
|
44
|
+
conversation.packets++;
|
|
45
|
+
conversation.bytes += frame.length;
|
|
46
|
+
if (flow.source === endpointA)
|
|
47
|
+
conversation.packetsAToB++;
|
|
48
|
+
else
|
|
49
|
+
conversation.packetsBToA++;
|
|
50
|
+
if (frame.timestamp < conversation.firstTimestamp)
|
|
51
|
+
conversation.firstTimestamp = frame.timestamp;
|
|
52
|
+
if (frame.timestamp > conversation.lastTimestamp)
|
|
53
|
+
conversation.lastTimestamp = frame.timestamp;
|
|
54
|
+
if (frame.index < conversation.firstIndex)
|
|
55
|
+
conversation.firstIndex = frame.index;
|
|
56
|
+
if (frame.index > conversation.lastIndex)
|
|
57
|
+
conversation.lastIndex = frame.index;
|
|
58
|
+
}
|
|
59
|
+
result() {
|
|
60
|
+
return [...this.#conversations.values()];
|
|
61
|
+
}
|
|
62
|
+
reset() {
|
|
63
|
+
this.#conversations.clear();
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
exports.ConversationsReducer = ConversationsReducer;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { Frame } from '../types/Frame';
|
|
2
|
+
import { UpdateContext } from '../types/UpdateContext';
|
|
3
|
+
import { IAnalysisReducer } from '../interfaces/IAnalysisReducer';
|
|
4
|
+
/** Per-endpoint traffic totals, direction-aware (tx = seen as source, rx = seen as destination). */
|
|
5
|
+
export type EndpointSummary = {
|
|
6
|
+
address: string;
|
|
7
|
+
packets: number;
|
|
8
|
+
bytes: number;
|
|
9
|
+
txPackets: number;
|
|
10
|
+
txBytes: number;
|
|
11
|
+
rxPackets: number;
|
|
12
|
+
rxBytes: number;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Rolling per-endpoint totals: every frame credits its source (tx) and destination (rx). result() is
|
|
16
|
+
* a snapshot; reset() clears it.
|
|
17
|
+
*/
|
|
18
|
+
export declare class EndpointsReducer implements IAnalysisReducer<EndpointSummary[]> {
|
|
19
|
+
#private;
|
|
20
|
+
readonly needs: string[];
|
|
21
|
+
readonly indexOnly: boolean;
|
|
22
|
+
update(frame: Frame, context: UpdateContext): void;
|
|
23
|
+
result(): EndpointSummary[];
|
|
24
|
+
reset(): void;
|
|
25
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.EndpointsReducer = void 0;
|
|
4
|
+
const ConversationKey_1 = require("../indexer/ConversationKey");
|
|
5
|
+
/**
|
|
6
|
+
* Rolling per-endpoint totals: every frame credits its source (tx) and destination (rx). result() is
|
|
7
|
+
* a snapshot; reset() clears it.
|
|
8
|
+
*/
|
|
9
|
+
class EndpointsReducer {
|
|
10
|
+
constructor() {
|
|
11
|
+
this.needs = ['eth', 'ipv4', 'ipv6', 'tcp', 'udp', 'arp'];
|
|
12
|
+
//Only the five-tuple — replay can feed frames synthesized from the index columns (no re-decode).
|
|
13
|
+
this.indexOnly = true;
|
|
14
|
+
this.#endpoints = new Map();
|
|
15
|
+
}
|
|
16
|
+
#endpoints;
|
|
17
|
+
update(frame, context) {
|
|
18
|
+
void context;
|
|
19
|
+
const flow = (0, ConversationKey_1.flowOf)(frame.layers);
|
|
20
|
+
if (!flow)
|
|
21
|
+
return;
|
|
22
|
+
this.#account(flow.source, frame.length, true);
|
|
23
|
+
this.#account(flow.destination, frame.length, false);
|
|
24
|
+
}
|
|
25
|
+
result() {
|
|
26
|
+
return [...this.#endpoints.values()];
|
|
27
|
+
}
|
|
28
|
+
reset() {
|
|
29
|
+
this.#endpoints.clear();
|
|
30
|
+
}
|
|
31
|
+
#account(address, length, isSource) {
|
|
32
|
+
let endpoint = this.#endpoints.get(address);
|
|
33
|
+
if (!endpoint) {
|
|
34
|
+
endpoint = { address: address, packets: 0, bytes: 0, txPackets: 0, txBytes: 0, rxPackets: 0, rxBytes: 0 };
|
|
35
|
+
this.#endpoints.set(address, endpoint);
|
|
36
|
+
}
|
|
37
|
+
endpoint.packets++;
|
|
38
|
+
endpoint.bytes += length;
|
|
39
|
+
if (isSource) {
|
|
40
|
+
endpoint.txPackets++;
|
|
41
|
+
endpoint.txBytes += length;
|
|
42
|
+
}
|
|
43
|
+
else {
|
|
44
|
+
endpoint.rxPackets++;
|
|
45
|
+
endpoint.rxBytes += length;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
exports.EndpointsReducer = EndpointsReducer;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { Frame } from '../types/Frame';
|
|
2
|
+
import { UpdateContext } from '../types/UpdateContext';
|
|
3
|
+
import { IAnalysisReducer } from '../interfaces/IAnalysisReducer';
|
|
4
|
+
/**
|
|
5
|
+
* Fold every frame into a single accumulator — the streaming analogue of Array.reduce. `fold` must
|
|
6
|
+
* return the next accumulator (treat it as immutable); reset() restores the seed.
|
|
7
|
+
*/
|
|
8
|
+
export declare function reduceReducer<TResult>(seed: TResult, fold: (accumulator: TResult, frame: Frame, context: UpdateContext) => TResult): IAnalysisReducer<TResult>;
|
|
9
|
+
/**
|
|
10
|
+
* Group frames by a derived key, folding each group's frames into a per-group value (seeded per
|
|
11
|
+
* group). result() returns a snapshot copy of the group map; reset() clears it.
|
|
12
|
+
*/
|
|
13
|
+
export declare function groupByReducer<TKey, TValue>(keyOf: (frame: Frame) => TKey, seed: TValue, fold: (accumulator: TValue, frame: Frame) => TValue): IAnalysisReducer<Map<TKey, TValue>>;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.reduceReducer = reduceReducer;
|
|
4
|
+
exports.groupByReducer = groupByReducer;
|
|
5
|
+
/**
|
|
6
|
+
* Fold every frame into a single accumulator — the streaming analogue of Array.reduce. `fold` must
|
|
7
|
+
* return the next accumulator (treat it as immutable); reset() restores the seed.
|
|
8
|
+
*/
|
|
9
|
+
function reduceReducer(seed, fold) {
|
|
10
|
+
let accumulator = seed;
|
|
11
|
+
return {
|
|
12
|
+
update(frame, context) {
|
|
13
|
+
accumulator = fold(accumulator, frame, context);
|
|
14
|
+
},
|
|
15
|
+
result() {
|
|
16
|
+
return accumulator;
|
|
17
|
+
},
|
|
18
|
+
reset() {
|
|
19
|
+
accumulator = seed;
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Group frames by a derived key, folding each group's frames into a per-group value (seeded per
|
|
25
|
+
* group). result() returns a snapshot copy of the group map; reset() clears it.
|
|
26
|
+
*/
|
|
27
|
+
function groupByReducer(keyOf, seed, fold) {
|
|
28
|
+
const groups = new Map();
|
|
29
|
+
return {
|
|
30
|
+
update(frame) {
|
|
31
|
+
const key = keyOf(frame);
|
|
32
|
+
const current = groups.has(key) ? groups.get(key) : seed;
|
|
33
|
+
groups.set(key, fold(current, frame));
|
|
34
|
+
},
|
|
35
|
+
result() {
|
|
36
|
+
return new Map(groups);
|
|
37
|
+
},
|
|
38
|
+
reset() {
|
|
39
|
+
groups.clear();
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Frame } from '../types/Frame';
|
|
2
|
+
import { UpdateContext } from '../types/UpdateContext';
|
|
3
|
+
import { IAnalysisReducer } from '../interfaces/IAnalysisReducer';
|
|
4
|
+
/** One RTT sample: a data/SYN/FIN segment matched to the first ACK that covers it. */
|
|
5
|
+
export type RttSample = {
|
|
6
|
+
segmentIndex: number;
|
|
7
|
+
ackIndex: number;
|
|
8
|
+
rtt: number;
|
|
9
|
+
};
|
|
10
|
+
/** Diagnostics for one bidirectional TCP conversation. */
|
|
11
|
+
export type TcpStreamDiagnostic = {
|
|
12
|
+
key: string;
|
|
13
|
+
endpointA: string;
|
|
14
|
+
endpointB: string;
|
|
15
|
+
packets: number;
|
|
16
|
+
retransmissions: number[];
|
|
17
|
+
duplicateAcks: number[];
|
|
18
|
+
rttSamples: RttSample[];
|
|
19
|
+
rttMin: number | null;
|
|
20
|
+
rttMax: number | null;
|
|
21
|
+
rttMean: number | null;
|
|
22
|
+
};
|
|
23
|
+
/**
|
|
24
|
+
* Streaming TCP stream diagnostics, derived per-frame: retransmissions
|
|
25
|
+
* (sequence space already sent, RFC 1982-aware), duplicate ACKs, and RTT (a segment matched to the
|
|
26
|
+
* first ACK covering it). result() is a rolling snapshot; RTT min/max/mean are recomputed at result().
|
|
27
|
+
* State grows per unique TCP stream — watch governance (eviction) layers on in a later step.
|
|
28
|
+
*/
|
|
29
|
+
export declare class TcpStreamReducer implements IAnalysisReducer<TcpStreamDiagnostic[]> {
|
|
30
|
+
#private;
|
|
31
|
+
readonly needs: string[];
|
|
32
|
+
update(frame: Frame, context: UpdateContext): void;
|
|
33
|
+
result(): TcpStreamDiagnostic[];
|
|
34
|
+
reset(): void;
|
|
35
|
+
}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.TcpStreamReducer = void 0;
|
|
4
|
+
function find(layers, id) {
|
|
5
|
+
return layers.find((l) => l.id === id);
|
|
6
|
+
}
|
|
7
|
+
//RFC 1982 serial comparison over the 32-bit sequence space (wraps at 2^32): a at/before b.
|
|
8
|
+
function seqLte(a, b) {
|
|
9
|
+
return ((b - a) >>> 0) < 0x80000000;
|
|
10
|
+
}
|
|
11
|
+
/** Project a frame to the TCP fields the reducer needs, or null if it is not TCP-over-IP. */
|
|
12
|
+
function tcpViewOf(frame) {
|
|
13
|
+
const tcp = find(frame.layers, 'tcp');
|
|
14
|
+
if (!tcp)
|
|
15
|
+
return null;
|
|
16
|
+
const ip = find(frame.layers, 'ipv4') || find(frame.layers, 'ipv6');
|
|
17
|
+
if (!ip)
|
|
18
|
+
return null;
|
|
19
|
+
const tcpData = tcp.data;
|
|
20
|
+
const ipData = ip.data;
|
|
21
|
+
const tcpHeaderBytes = Number(tcpData.hdrLen) || 20;
|
|
22
|
+
let payload;
|
|
23
|
+
if (ip.id === 'ipv4')
|
|
24
|
+
payload = Number(ipData.length) - (Number(ipData.hdrLen) || 20) - tcpHeaderBytes;
|
|
25
|
+
//IPv6 uses plen (bytes after the 40-byte header), not length; over-counts with extension headers.
|
|
26
|
+
else
|
|
27
|
+
payload = Number(ipData.plen) - tcpHeaderBytes;
|
|
28
|
+
if (!(payload >= 0))
|
|
29
|
+
payload = 0;
|
|
30
|
+
const flags = tcpData.flags || {};
|
|
31
|
+
return {
|
|
32
|
+
source: `${ipData.sip}:${tcpData.srcport}`,
|
|
33
|
+
destination: `${ipData.dip}:${tcpData.dstport}`,
|
|
34
|
+
seq: Number(tcpData.seq) || 0,
|
|
35
|
+
ack: Number(tcpData.ack) || 0,
|
|
36
|
+
syn: Boolean(flags.syn),
|
|
37
|
+
fin: Boolean(flags.fin),
|
|
38
|
+
ackFlag: Boolean(flags.ack),
|
|
39
|
+
segmentLength: payload,
|
|
40
|
+
timestamp: frame.timestamp,
|
|
41
|
+
index: frame.index
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
function summarizeRtt(stream) {
|
|
45
|
+
if (stream.rttSamples.length === 0) {
|
|
46
|
+
stream.rttMin = null;
|
|
47
|
+
stream.rttMax = null;
|
|
48
|
+
stream.rttMean = null;
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
let min = Infinity;
|
|
52
|
+
let max = -Infinity;
|
|
53
|
+
let sum = 0;
|
|
54
|
+
for (const sample of stream.rttSamples) {
|
|
55
|
+
if (sample.rtt < min)
|
|
56
|
+
min = sample.rtt;
|
|
57
|
+
if (sample.rtt > max)
|
|
58
|
+
max = sample.rtt;
|
|
59
|
+
sum += sample.rtt;
|
|
60
|
+
}
|
|
61
|
+
stream.rttMin = min;
|
|
62
|
+
stream.rttMax = max;
|
|
63
|
+
stream.rttMean = sum / stream.rttSamples.length;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Streaming TCP stream diagnostics, derived per-frame: retransmissions
|
|
67
|
+
* (sequence space already sent, RFC 1982-aware), duplicate ACKs, and RTT (a segment matched to the
|
|
68
|
+
* first ACK covering it). result() is a rolling snapshot; RTT min/max/mean are recomputed at result().
|
|
69
|
+
* State grows per unique TCP stream — watch governance (eviction) layers on in a later step.
|
|
70
|
+
*/
|
|
71
|
+
class TcpStreamReducer {
|
|
72
|
+
constructor() {
|
|
73
|
+
this.needs = ['ipv4', 'ipv6', 'tcp'];
|
|
74
|
+
this.#streams = new Map();
|
|
75
|
+
this.#halves = new Map();
|
|
76
|
+
}
|
|
77
|
+
#streams;
|
|
78
|
+
#halves;
|
|
79
|
+
update(frame, context) {
|
|
80
|
+
void context;
|
|
81
|
+
const view = tcpViewOf(frame);
|
|
82
|
+
if (!view)
|
|
83
|
+
return;
|
|
84
|
+
const forward = view.source <= view.destination;
|
|
85
|
+
const endpointA = forward ? view.source : view.destination;
|
|
86
|
+
const endpointB = forward ? view.destination : view.source;
|
|
87
|
+
const key = `tcp|${endpointA}|${endpointB}`;
|
|
88
|
+
let stream = this.#streams.get(key);
|
|
89
|
+
if (!stream) {
|
|
90
|
+
stream = { key: key, endpointA: endpointA, endpointB: endpointB, packets: 0, retransmissions: [], duplicateAcks: [], rttSamples: [], rttMin: null, rttMax: null, rttMean: null };
|
|
91
|
+
this.#streams.set(key, stream);
|
|
92
|
+
}
|
|
93
|
+
stream.packets++;
|
|
94
|
+
const forwardHalf = this.#half(`${key}|${view.source}`);
|
|
95
|
+
const reverseHalf = this.#half(`${key}|${view.destination}`);
|
|
96
|
+
const consumed = view.segmentLength + (view.syn ? 1 : 0) + (view.fin ? 1 : 0);
|
|
97
|
+
const endSeq = view.seq + consumed;
|
|
98
|
+
if (consumed > 0) {
|
|
99
|
+
const alreadySent = forwardHalf.hasMaxEndSeq && seqLte(endSeq, forwardHalf.maxEndSeq);
|
|
100
|
+
if (alreadySent || (view.syn && forwardHalf.seenSyn)) {
|
|
101
|
+
stream.retransmissions.push(view.index);
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
forwardHalf.pending.push({ endSeq: endSeq, timestamp: view.timestamp, index: view.index });
|
|
105
|
+
}
|
|
106
|
+
if (!forwardHalf.hasMaxEndSeq || seqLte(forwardHalf.maxEndSeq, endSeq)) {
|
|
107
|
+
forwardHalf.maxEndSeq = endSeq;
|
|
108
|
+
forwardHalf.hasMaxEndSeq = true;
|
|
109
|
+
}
|
|
110
|
+
if (view.syn)
|
|
111
|
+
forwardHalf.seenSyn = true;
|
|
112
|
+
}
|
|
113
|
+
if (view.ackFlag && consumed === 0) {
|
|
114
|
+
if (forwardHalf.lastPureAck !== null && forwardHalf.lastPureAck === view.ack) {
|
|
115
|
+
stream.duplicateAcks.push(view.index);
|
|
116
|
+
}
|
|
117
|
+
forwardHalf.lastPureAck = view.ack;
|
|
118
|
+
}
|
|
119
|
+
if (view.ackFlag && reverseHalf.pending.length > 0) {
|
|
120
|
+
const remaining = [];
|
|
121
|
+
for (const segment of reverseHalf.pending) {
|
|
122
|
+
if (seqLte(segment.endSeq, view.ack)) {
|
|
123
|
+
if (view.timestamp >= segment.timestamp) {
|
|
124
|
+
const sample = { segmentIndex: segment.index, ackIndex: view.index, rtt: view.timestamp - segment.timestamp };
|
|
125
|
+
stream.rttSamples.push(sample);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
else {
|
|
129
|
+
remaining.push(segment);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
reverseHalf.pending = remaining;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
result() {
|
|
136
|
+
for (const stream of this.#streams.values())
|
|
137
|
+
summarizeRtt(stream);
|
|
138
|
+
return [...this.#streams.values()];
|
|
139
|
+
}
|
|
140
|
+
reset() {
|
|
141
|
+
this.#streams.clear();
|
|
142
|
+
this.#halves.clear();
|
|
143
|
+
}
|
|
144
|
+
#half(key) {
|
|
145
|
+
let half = this.#halves.get(key);
|
|
146
|
+
if (!half) {
|
|
147
|
+
half = { maxEndSeq: 0, hasMaxEndSeq: false, seenSyn: false, lastPureAck: null, pending: [] };
|
|
148
|
+
this.#halves.set(key, half);
|
|
149
|
+
}
|
|
150
|
+
return half;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
exports.TcpStreamReducer = TcpStreamReducer;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { FrameIndexRecord } from '../types/FrameIndexRecord';
|
|
2
|
+
import { IIndexStore } from '../interfaces/IIndexStore';
|
|
3
|
+
/**
|
|
4
|
+
* Columnar, heap-external frame index: each fixed-width field is its own growable TypedArray, so a
|
|
5
|
+
* few million frames cost tens of MB rather than a JS object per frame. Frame numbers are global and
|
|
6
|
+
* monotonic (never reused); physical slot = globalIndex - firstIndex. evictOldest shifts the columns
|
|
7
|
+
* down (FIFO) for watch governance. This is the v1 IIndexStore; a SQLite backend is a v2 seam.
|
|
8
|
+
*/
|
|
9
|
+
export declare class ColumnarIndexStore implements IIndexStore {
|
|
10
|
+
#private;
|
|
11
|
+
constructor(initialCapacity?: number);
|
|
12
|
+
append(record: FrameIndexRecord): number;
|
|
13
|
+
get(index: number): FrameIndexRecord | null;
|
|
14
|
+
/** Records whose global frame number is in the half-open range [from, to), clamped to what exists. */
|
|
15
|
+
range(from: number, to: number): FrameIndexRecord[];
|
|
16
|
+
scan(predicate: (record: FrameIndexRecord) => boolean): number[];
|
|
17
|
+
count(): number;
|
|
18
|
+
firstIndex(): number;
|
|
19
|
+
evictOldest(count: number): void;
|
|
20
|
+
clear(): void;
|
|
21
|
+
}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.ColumnarIndexStore = void 0;
|
|
4
|
+
/**
|
|
5
|
+
* Columnar, heap-external frame index: each fixed-width field is its own growable TypedArray, so a
|
|
6
|
+
* few million frames cost tens of MB rather than a JS object per frame. Frame numbers are global and
|
|
7
|
+
* monotonic (never reused); physical slot = globalIndex - firstIndex. evictOldest shifts the columns
|
|
8
|
+
* down (FIFO) for watch governance. This is the v1 IIndexStore; a SQLite backend is a v2 seam.
|
|
9
|
+
*/
|
|
10
|
+
class ColumnarIndexStore {
|
|
11
|
+
#capacity;
|
|
12
|
+
//Number of frames physically retained.
|
|
13
|
+
#length = 0;
|
|
14
|
+
//Global frame number of physical slot 0 (advances on eviction).
|
|
15
|
+
#base = 0;
|
|
16
|
+
#fileOffset;
|
|
17
|
+
#capturedLength;
|
|
18
|
+
#originalLength;
|
|
19
|
+
#timestamp;
|
|
20
|
+
#protocolId;
|
|
21
|
+
#conversationHash;
|
|
22
|
+
#directionForward;
|
|
23
|
+
constructor(initialCapacity = 1024) {
|
|
24
|
+
this.#capacity = Math.max(1, initialCapacity);
|
|
25
|
+
this.#fileOffset = new Float64Array(this.#capacity);
|
|
26
|
+
this.#capturedLength = new Uint32Array(this.#capacity);
|
|
27
|
+
this.#originalLength = new Uint32Array(this.#capacity);
|
|
28
|
+
this.#timestamp = new Float64Array(this.#capacity);
|
|
29
|
+
this.#protocolId = new Uint16Array(this.#capacity);
|
|
30
|
+
this.#conversationHash = new Uint32Array(this.#capacity);
|
|
31
|
+
this.#directionForward = new Uint8Array(this.#capacity);
|
|
32
|
+
}
|
|
33
|
+
append(record) {
|
|
34
|
+
if (this.#length >= this.#capacity)
|
|
35
|
+
this.#grow();
|
|
36
|
+
const slot = this.#length;
|
|
37
|
+
this.#fileOffset[slot] = record.fileOffset;
|
|
38
|
+
this.#capturedLength[slot] = record.capturedLength;
|
|
39
|
+
this.#originalLength[slot] = record.originalLength;
|
|
40
|
+
this.#timestamp[slot] = record.timestamp;
|
|
41
|
+
this.#protocolId[slot] = record.protocolId;
|
|
42
|
+
this.#conversationHash[slot] = record.conversationHash;
|
|
43
|
+
this.#directionForward[slot] = record.directionForward;
|
|
44
|
+
this.#length++;
|
|
45
|
+
return this.#base + slot;
|
|
46
|
+
}
|
|
47
|
+
get(index) {
|
|
48
|
+
const slot = index - this.#base;
|
|
49
|
+
if (slot < 0 || slot >= this.#length)
|
|
50
|
+
return null;
|
|
51
|
+
return this.#recordAt(slot);
|
|
52
|
+
}
|
|
53
|
+
/** Records whose global frame number is in the half-open range [from, to), clamped to what exists. */
|
|
54
|
+
range(from, to) {
|
|
55
|
+
const start = Math.max(from, this.#base);
|
|
56
|
+
const end = Math.min(to, this.#base + this.#length);
|
|
57
|
+
const out = [];
|
|
58
|
+
for (let index = start; index < end; index++)
|
|
59
|
+
out.push(this.#recordAt(index - this.#base));
|
|
60
|
+
return out;
|
|
61
|
+
}
|
|
62
|
+
scan(predicate) {
|
|
63
|
+
const out = [];
|
|
64
|
+
for (let slot = 0; slot < this.#length; slot++) {
|
|
65
|
+
const record = this.#recordAt(slot);
|
|
66
|
+
if (predicate(record))
|
|
67
|
+
out.push(record.index);
|
|
68
|
+
}
|
|
69
|
+
return out;
|
|
70
|
+
}
|
|
71
|
+
count() {
|
|
72
|
+
return this.#length;
|
|
73
|
+
}
|
|
74
|
+
firstIndex() {
|
|
75
|
+
return this.#base;
|
|
76
|
+
}
|
|
77
|
+
evictOldest(count) {
|
|
78
|
+
const drop = Math.min(count, this.#length);
|
|
79
|
+
if (drop <= 0)
|
|
80
|
+
return;
|
|
81
|
+
const remaining = this.#length - drop;
|
|
82
|
+
this.#fileOffset.copyWithin(0, drop, this.#length);
|
|
83
|
+
this.#capturedLength.copyWithin(0, drop, this.#length);
|
|
84
|
+
this.#originalLength.copyWithin(0, drop, this.#length);
|
|
85
|
+
this.#timestamp.copyWithin(0, drop, this.#length);
|
|
86
|
+
this.#protocolId.copyWithin(0, drop, this.#length);
|
|
87
|
+
this.#conversationHash.copyWithin(0, drop, this.#length);
|
|
88
|
+
this.#directionForward.copyWithin(0, drop, this.#length);
|
|
89
|
+
this.#length = remaining;
|
|
90
|
+
this.#base += drop;
|
|
91
|
+
}
|
|
92
|
+
clear() {
|
|
93
|
+
this.#length = 0;
|
|
94
|
+
this.#base = 0;
|
|
95
|
+
}
|
|
96
|
+
#recordAt(slot) {
|
|
97
|
+
return {
|
|
98
|
+
index: this.#base + slot,
|
|
99
|
+
fileOffset: this.#fileOffset[slot],
|
|
100
|
+
capturedLength: this.#capturedLength[slot],
|
|
101
|
+
originalLength: this.#originalLength[slot],
|
|
102
|
+
timestamp: this.#timestamp[slot],
|
|
103
|
+
protocolId: this.#protocolId[slot],
|
|
104
|
+
conversationHash: this.#conversationHash[slot],
|
|
105
|
+
directionForward: this.#directionForward[slot]
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
#grow() {
|
|
109
|
+
const next = this.#capacity * 2;
|
|
110
|
+
const fileOffset = new Float64Array(next);
|
|
111
|
+
const capturedLength = new Uint32Array(next);
|
|
112
|
+
const originalLength = new Uint32Array(next);
|
|
113
|
+
const timestamp = new Float64Array(next);
|
|
114
|
+
const protocolId = new Uint16Array(next);
|
|
115
|
+
const conversationHash = new Uint32Array(next);
|
|
116
|
+
const directionForward = new Uint8Array(next);
|
|
117
|
+
fileOffset.set(this.#fileOffset);
|
|
118
|
+
capturedLength.set(this.#capturedLength);
|
|
119
|
+
originalLength.set(this.#originalLength);
|
|
120
|
+
timestamp.set(this.#timestamp);
|
|
121
|
+
protocolId.set(this.#protocolId);
|
|
122
|
+
conversationHash.set(this.#conversationHash);
|
|
123
|
+
directionForward.set(this.#directionForward);
|
|
124
|
+
this.#fileOffset = fileOffset;
|
|
125
|
+
this.#capturedLength = capturedLength;
|
|
126
|
+
this.#originalLength = originalLength;
|
|
127
|
+
this.#timestamp = timestamp;
|
|
128
|
+
this.#protocolId = protocolId;
|
|
129
|
+
this.#conversationHash = conversationHash;
|
|
130
|
+
this.#directionForward = directionForward;
|
|
131
|
+
this.#capacity = next;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
exports.ColumnarIndexStore = ColumnarIndexStore;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { FrameRow } from './FrameRow';
|
|
2
|
+
/** Event map for the Analysis facade: newly indexed frame, indexing progress, completion, error. */
|
|
3
|
+
export type AnalysisEvents = {
|
|
4
|
+
frame: (row: FrameRow) => void;
|
|
5
|
+
progress: (done: number, total: number) => void;
|
|
6
|
+
complete: () => void;
|
|
7
|
+
error: (error: Error) => void;
|
|
8
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { CodecDecodeResult } from '@netkitty/codec';
|
|
2
|
+
import { FrameRow } from './FrameRow';
|
|
3
|
+
/** A fully materialized frame: the lightweight row plus its decoded layers (fetched on demand via LRU). */
|
|
4
|
+
export type Frame = FrameRow & {
|
|
5
|
+
capturedLength: number;
|
|
6
|
+
layers: CodecDecodeResult[];
|
|
7
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Logical view of one frame's fixed-width index record. Physically stored columnar across TypedArrays
|
|
3
|
+
* by the index store; this shape is what append/get/range/scan hand back.
|
|
4
|
+
*/
|
|
5
|
+
export type FrameIndexRecord = {
|
|
6
|
+
index: number;
|
|
7
|
+
fileOffset: number;
|
|
8
|
+
capturedLength: number;
|
|
9
|
+
originalLength: number;
|
|
10
|
+
timestamp: number;
|
|
11
|
+
protocolId: number;
|
|
12
|
+
conversationHash: number;
|
|
13
|
+
directionForward: number;
|
|
14
|
+
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight per-frame row for list display and batch projection — no decoded layers. Derived from
|
|
3
|
+
* the columnar index, cheap to stream across the worker boundary in bulk.
|
|
4
|
+
*/
|
|
5
|
+
export type FrameRow = {
|
|
6
|
+
index: number;
|
|
7
|
+
timestamp: number;
|
|
8
|
+
length: number;
|
|
9
|
+
topProtocol: string;
|
|
10
|
+
conversationKey: string | null;
|
|
11
|
+
info: string;
|
|
12
|
+
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { IReadBackend } from '../interfaces/IReadBackend';
|
|
2
|
+
import { IWorkerEndpoint } from './IWorkerEndpoint';
|
|
3
|
+
/**
|
|
4
|
+
* Environment-agnostic analysis-worker logic: owns the read backend, columnar index, indexer
|
|
5
|
+
* dictionaries and codec, and wires all request handlers plus watch tailing onto the given endpoint.
|
|
6
|
+
* `makeBackend` adapts the source (node: a path string → NodeFileReadBackend; browser: a Blob →
|
|
7
|
+
* BrowserFileReadBackend). node and browser worker entrypoints each call this with their endpoint and
|
|
8
|
+
* backend factory, so the handler logic lives in one place.
|
|
9
|
+
*/
|
|
10
|
+
export declare function installAnalysisHandlers(endpoint: IWorkerEndpoint, makeBackend: (source: unknown) => IReadBackend): void;
|