@netkitty/analysis 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/CHANGELOG.md +12 -0
  2. package/LICENSE +21 -0
  3. package/README.md +150 -0
  4. package/README.zh-CN.md +146 -0
  5. package/dist/index.d.ts +30 -0
  6. package/dist/index.js +27 -0
  7. package/dist/lib/streaming/Analysis.d.ts +54 -0
  8. package/dist/lib/streaming/Analysis.js +159 -0
  9. package/dist/lib/streaming/backends/BrowserFileReadBackend.d.ts +14 -0
  10. package/dist/lib/streaming/backends/BrowserFileReadBackend.js +36 -0
  11. package/dist/lib/streaming/backends/NodeFileReadBackend.d.ts +15 -0
  12. package/dist/lib/streaming/backends/NodeFileReadBackend.js +62 -0
  13. package/dist/lib/streaming/filter/FilterExpression.d.ts +35 -0
  14. package/dist/lib/streaming/filter/FilterExpression.js +166 -0
  15. package/dist/lib/streaming/indexer/ConversationKey.d.ts +18 -0
  16. package/dist/lib/streaming/indexer/ConversationKey.js +54 -0
  17. package/dist/lib/streaming/indexer/FrameIndexer.d.ts +16 -0
  18. package/dist/lib/streaming/indexer/FrameIndexer.js +58 -0
  19. package/dist/lib/streaming/indexer/PcapIndexBuilder.d.ts +20 -0
  20. package/dist/lib/streaming/indexer/PcapIndexBuilder.js +52 -0
  21. package/dist/lib/streaming/interfaces/IAnalysisReducer.d.ts +14 -0
  22. package/dist/lib/streaming/interfaces/IAnalysisReducer.js +2 -0
  23. package/dist/lib/streaming/interfaces/IIndexStore.d.ts +16 -0
  24. package/dist/lib/streaming/interfaces/IIndexStore.js +2 -0
  25. package/dist/lib/streaming/interfaces/IReadBackend.d.ts +11 -0
  26. package/dist/lib/streaming/interfaces/IReadBackend.js +2 -0
  27. package/dist/lib/streaming/interfaces/IWorkerChannel.d.ts +12 -0
  28. package/dist/lib/streaming/interfaces/IWorkerChannel.js +2 -0
  29. package/dist/lib/streaming/reducers/ConversationsReducer.d.ts +34 -0
  30. package/dist/lib/streaming/reducers/ConversationsReducer.js +66 -0
  31. package/dist/lib/streaming/reducers/EndpointsReducer.d.ts +25 -0
  32. package/dist/lib/streaming/reducers/EndpointsReducer.js +49 -0
  33. package/dist/lib/streaming/reducers/ReducerFactories.d.ts +13 -0
  34. package/dist/lib/streaming/reducers/ReducerFactories.js +42 -0
  35. package/dist/lib/streaming/reducers/TcpStreamReducer.d.ts +35 -0
  36. package/dist/lib/streaming/reducers/TcpStreamReducer.js +153 -0
  37. package/dist/lib/streaming/stores/ColumnarIndexStore.d.ts +21 -0
  38. package/dist/lib/streaming/stores/ColumnarIndexStore.js +134 -0
  39. package/dist/lib/streaming/types/AnalysisEvents.d.ts +8 -0
  40. package/dist/lib/streaming/types/AnalysisEvents.js +2 -0
  41. package/dist/lib/streaming/types/AnalysisOptions.d.ts +5 -0
  42. package/dist/lib/streaming/types/AnalysisOptions.js +2 -0
  43. package/dist/lib/streaming/types/AnalysisPhase.d.ts +2 -0
  44. package/dist/lib/streaming/types/AnalysisPhase.js +2 -0
  45. package/dist/lib/streaming/types/AnalysisSource.d.ts +2 -0
  46. package/dist/lib/streaming/types/AnalysisSource.js +2 -0
  47. package/dist/lib/streaming/types/Frame.d.ts +7 -0
  48. package/dist/lib/streaming/types/Frame.js +2 -0
  49. package/dist/lib/streaming/types/FrameIndexRecord.d.ts +14 -0
  50. package/dist/lib/streaming/types/FrameIndexRecord.js +2 -0
  51. package/dist/lib/streaming/types/FrameRow.d.ts +12 -0
  52. package/dist/lib/streaming/types/FrameRow.js +2 -0
  53. package/dist/lib/streaming/types/UpdateContext.d.ts +7 -0
  54. package/dist/lib/streaming/types/UpdateContext.js +2 -0
  55. package/dist/lib/streaming/worker/AnalysisWorkerCore.d.ts +10 -0
  56. package/dist/lib/streaming/worker/AnalysisWorkerCore.js +192 -0
  57. package/dist/lib/streaming/worker/BuiltinScans.d.ts +23 -0
  58. package/dist/lib/streaming/worker/BuiltinScans.js +121 -0
  59. package/dist/lib/streaming/worker/IWorkerEndpoint.d.ts +9 -0
  60. package/dist/lib/streaming/worker/IWorkerEndpoint.js +2 -0
  61. package/dist/lib/streaming/worker/NodeWorkerChannel.d.ts +16 -0
  62. package/dist/lib/streaming/worker/NodeWorkerChannel.js +60 -0
  63. package/dist/lib/streaming/worker/SyntheticFrame.d.ts +8 -0
  64. package/dist/lib/streaming/worker/SyntheticFrame.js +38 -0
  65. package/dist/lib/streaming/worker/WebWorkerChannel.d.ts +23 -0
  66. package/dist/lib/streaming/worker/WebWorkerChannel.js +59 -0
  67. package/dist/lib/streaming/worker/WebWorkerEndpoint.d.ts +19 -0
  68. package/dist/lib/streaming/worker/WebWorkerEndpoint.js +44 -0
  69. package/dist/lib/streaming/worker/WorkerEndpoint.d.ts +13 -0
  70. package/dist/lib/streaming/worker/WorkerEndpoint.js +45 -0
  71. package/dist/lib/streaming/worker/WorkerMessage.d.ts +24 -0
  72. package/dist/lib/streaming/worker/WorkerMessage.js +2 -0
  73. package/dist/lib/streaming/worker/analysisWorker.d.ts +1 -0
  74. package/dist/lib/streaming/worker/analysisWorker.js +11 -0
  75. package/dist/lib/streaming/worker/analysisWorkerBrowser.d.ts +1 -0
  76. package/dist/lib/streaming/worker/analysisWorkerBrowser.js +12 -0
  77. package/dist/lib/streaming/worker/spawnNodeAnalysisChannel.d.ts +7 -0
  78. package/dist/lib/streaming/worker/spawnNodeAnalysisChannel.js +18 -0
  79. package/package.json +65 -0
package/CHANGELOG.md ADDED
@@ -0,0 +1,12 @@
1
+ # Changelog
2
+
3
+ All notable changes to this package are documented here, following
4
+ [Semantic Versioning](https://semver.org/). From the next release onward, entries
5
+ are generated automatically from Conventional Commits.
6
+
7
+ ## 1.0.0 - 2026-07-22
8
+
9
+ First stable release.
10
+
11
+ - Streaming, Wireshark-style cross-packet analysis over a capture file
12
+ (conversations, endpoints, TCP streams) running off a worker.
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Alex Ma
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,150 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/myq1991/netkitty/main/assets/NetKittyLogo.webp" alt="NetKitty" width="180">
3
+ </p>
4
+
5
+ # @netkitty/analysis
6
+
7
+ Streaming, cross-packet analysis for capture files — a programmable, Wireshark-style front door over
8
+ pcap/pcapng. The same `Analysis` facade runs in **node and the browser**: heavy work (read → parse →
9
+ partial-decode → columnar index → decode) lives in a single worker, so `open()` never blocks the
10
+ caller and `close()` releases everything by terminating the worker.
11
+
12
+ > 中文文档见 [README.zh-CN.md](./README.zh-CN.md)。
13
+
14
+ Built on [`@netkitty/codec`](../codec) (protocol decoding) and [`@netkitty/pcap-core`](../pcap-core)
15
+ (pcap/pcapng parsing) — both pure-buffer and browser-safe.
16
+
17
+ ## Install
18
+
19
+ ```bash
20
+ npm i @netkitty/analysis
21
+ # or use the aggregate package: import ... from 'netkitty/analysis'
22
+ ```
23
+
24
+ ## Quick start (node)
25
+
26
+ ```ts
27
+ import {Analysis, ConversationsReducer} from '@netkitty/analysis'
28
+
29
+ const analysis = new Analysis()
30
+ await analysis.open('/path/to/capture.pcap') // bounded file: index, then emit 'complete'
31
+
32
+ analysis.frameCount() // total frames indexed
33
+ const rows = await analysis.getFrames(0, 100) // lightweight rows (no decoded layers)
34
+ const frame = await analysis.getFrame(42) // one frame WITH decoded layers
35
+ const tcp = await analysis.filter('tcp') // matching frame indices
36
+
37
+ const conversations = new ConversationsReducer()
38
+ await analysis.attachReducer(conversations) // replays every indexed frame, then follows live
39
+ conversations.result() // rolling snapshot (no finalize)
40
+
41
+ await analysis.close() // terminate the worker, release everything
42
+ ```
43
+
44
+ ## Reducers
45
+
46
+ A reducer is a pluggable rolling analysis fed every frame; `result()` is a snapshot at any time (there
47
+ is no finalize). `attachReducer` **replays** the whole indexed backlog first, then follows the live
48
+ stream — so stats are complete the moment it resolves.
49
+
50
+ - **Built-in** (exported): `ConversationsReducer`, `EndpointsReducer`, `TcpStreamReducer`
51
+ (retransmissions / duplicate ACKs / RTT). For Conversations/Endpoints prefer the shortcut methods
52
+ `analysis.conversations()` / `analysis.endpoints()` — they compute the same result **entirely inside
53
+ the worker** by scanning the index columns (no re-decode, no per-frame cross-thread transfer, zero
54
+ main-thread work), so they stay fast even at hundreds of millions of frames.
55
+ - **Factories**: `reduceReducer(seed, fold)`, `groupByReducer(keyOf, seed, fold)`.
56
+ - **Custom**: implement `IAnalysisReducer<T>` — see [Writing a custom reducer](#writing-a-custom-reducer).
57
+
58
+ ```ts
59
+ import {reduceReducer, groupByReducer} from '@netkitty/analysis'
60
+
61
+ const totalBytes = reduceReducer(0, (sum, frame) => sum + frame.length)
62
+ const perProtocol = groupByReducer(f => f.topProtocol, 0, count => count + 1)
63
+ ```
64
+
65
+ ### Writing a custom reducer
66
+
67
+ A reducer is a plain object (or class) implementing `IAnalysisReducer<T>`:
68
+
69
+ ```ts
70
+ interface IAnalysisReducer<T> {
71
+ update(frame: Frame, context: UpdateContext): void // called once per frame (replay, then live)
72
+ result(): T // rolling snapshot — callable any time, no finalize
73
+ reset(): void // clear all accumulated state
74
+ readonly needs?: string[] // optional: layer ids you read (projection)
75
+ readonly indexOnly?: boolean // optional: correctness/perf contract, see below
76
+ }
77
+ ```
78
+
79
+ - **`update(frame, context)`** runs for every frame. `frame` carries `index`, `timestamp`, `length`,
80
+ `topProtocol`, `conversationKey`, and `layers` (the decoded protocol tree). `context` carries `index`,
81
+ `total`, and `phase` — `'replay'` while catching up on the already-indexed backlog, `'live'` while
82
+ tailing under `watch()`.
83
+ - **`result()`** is a rolling snapshot; there is **no finalize**, so call it whenever (mid-replay,
84
+ mid-tail). **`reset()`** clears state.
85
+ - Keep state bounded **per conversation/endpoint**, not per frame — the built-ins hold a first/last
86
+ index span instead of a member-frame list, so memory stays flat across hundreds of millions of frames.
87
+ - **`needs`** (optional): the layer ids you actually read, e.g. `['ipv4', 'tcp']`. The worker then
88
+ transfers only those layers across the thread boundary instead of the whole tree — pure bandwidth win,
89
+ no behavioural change.
90
+ - **`indexOnly`** (optional — a **correctness contract**, not just a hint): set `true` **only if**
91
+ `update` reads nothing beyond frame metadata (`index` / `timestamp` / `length` / `topProtocol` /
92
+ `conversationKey`) and the five-tuple carried in `layers` — the endpoint addresses and ports
93
+ (`eth` `smac`/`dmac`, `ipv4`/`ipv6` `sip`/`dip`, `tcp`/`udp` `srcport`/`dstport`). Replay then feeds
94
+ frames **synthesized straight from the index columns, skipping the per-frame re-decode** — measured
95
+ ~13× faster on large captures. ⚠️ If you set it while reading any deeper field (`ipv4.ttl`,
96
+ `tcp.flags`, TLS SNI, payload …), that field is simply **absent** on replay and your result is
97
+ silently wrong. When unsure, leave it unset — full decode is the safe default. `ConversationsReducer`
98
+ and `EndpointsReducer` set it (five-tuple only); `TcpStreamReducer` does not (it reads seq/ack/flags).
99
+
100
+ ```ts
101
+ import {IAnalysisReducer, Frame, UpdateContext} from '@netkitty/analysis'
102
+
103
+ //Counts frames per top-level protocol. Reads only metadata → indexOnly is safe.
104
+ class ProtocolCounter implements IAnalysisReducer<Record<string, number>> {
105
+ readonly indexOnly = true
106
+ #counts: Record<string, number> = {}
107
+ update(frame: Frame, _context: UpdateContext): void {
108
+ this.#counts[frame.topProtocol] = (this.#counts[frame.topProtocol] ?? 0) + 1
109
+ }
110
+ result(): Record<string, number> { return {...this.#counts} }
111
+ reset(): void { this.#counts = {} }
112
+ }
113
+ ```
114
+
115
+ A reducer runs **on the main thread** — its `update` is a function, so it can't be copied into the
116
+ worker to run there. That is fine: `update` is just cheap arithmetic; the expensive part (read + decode)
117
+ already happened in the worker, and `indexOnly`/`needs` decide how much of that data is sent back. A
118
+ reducer may also extend an event emitter and push its own `progress`/`update` events from `update()`.
119
+
120
+ ## Live capture (watch)
121
+
122
+ `watch()` follows a capture file that is still being written, feeding each new frame to reducers with
123
+ phase `'live'`. The index is **unbounded by default** (the memory is yours to manage). `maxFrames` is an
124
+ optional guard: once the index passes the cap, the oldest entries are dropped first-in-first-out
125
+ (a Wireshark-style ring buffer), keeping a long-running tail bounded — only the in-memory index is
126
+ dropped, the file on disk is untouched.
127
+
128
+ ```ts
129
+ const analysis = new Analysis() // default: unbounded index
130
+ analysis.on('frame', row => { /* new indexed frame */ })
131
+ await analysis.watch('/path/to/growing.pcap') // tails the file; reducers get phase 'live'
132
+
133
+ // optional memory guard for a long-running tail:
134
+ // new Analysis({maxFrames: 1_000_000}) // FIFO-evict the oldest past the cap → bounded memory
135
+ ```
136
+
137
+ ## Browser
138
+
139
+ The facade is environment-agnostic — inject a browser worker channel; the worker is a bundled
140
+ `analysisWorkerBrowser` entry (esbuild + Buffer polyfill), and the source is a `File`/`Blob`:
141
+
142
+ ```ts
143
+ import {Analysis, WebWorkerChannel} from '@netkitty/analysis'
144
+
145
+ const worker = new Worker(/* bundled analysisWorkerBrowser.js */)
146
+ const analysis = new Analysis({}, () => new WebWorkerChannel(worker))
147
+ await analysis.open(file) // file: a File/Blob
148
+ ```
149
+
150
+ node and browser produce field-for-field identical results.
@@ -0,0 +1,146 @@
1
+ <p align="center">
2
+ <img src="https://raw.githubusercontent.com/myq1991/netkitty/main/assets/NetKittyLogo.webp" alt="NetKitty" width="180">
3
+ </p>
4
+
5
+ # @netkitty/analysis
6
+
7
+ 对抓包文件做流式的跨包分析——一个可编程的、类似 Wireshark 的 pcap/pcapng 入口。同一个 `Analysis` 类
8
+ **在 Node 和浏览器里都能用**:读取、解析、部分解码、建立索引这些重活都放在一个后台工作线程里做,所以
9
+ `open()` 不会卡住调用方,`close()` 只要结束这个线程,就能把占用的内存一次性全部释放。
10
+
11
+ > English docs: [README.md](./README.md)
12
+
13
+ 它底层用到 [`@netkitty/codec`](../codec)(协议解码)和 [`@netkitty/pcap-core`](../pcap-core)
14
+ (pcap/pcapng 解析),两者都只处理内存里的字节,可以安全地在浏览器中运行。
15
+
16
+ ## 安装
17
+
18
+ ```bash
19
+ npm i @netkitty/analysis
20
+ # 或者用聚合包:import ... from 'netkitty/analysis'
21
+ ```
22
+
23
+ ## 快速上手(Node)
24
+
25
+ ```ts
26
+ import {Analysis, ConversationsReducer} from '@netkitty/analysis'
27
+
28
+ const analysis = new Analysis()
29
+ await analysis.open('/path/to/capture.pcap') // 读一个完整文件:建好索引,完成后触发 'complete'
30
+
31
+ analysis.frameCount() // 已索引的总帧数
32
+ const rows = await analysis.getFrames(0, 100) // 轻量的帧行(不含解码后的各层)
33
+ const frame = await analysis.getFrame(42) // 单帧,含解码后的各层
34
+ const tcp = await analysis.filter('tcp') // 命中的帧序号
35
+
36
+ const conversations = new ConversationsReducer()
37
+ await analysis.attachReducer(conversations) // 先把已索引的历史帧全部重放,再跟随后续实时帧
38
+ conversations.result() // 随时取当前结果(没有收尾这一步)
39
+
40
+ await analysis.close() // 结束工作线程,释放全部内存
41
+ ```
42
+
43
+ ## 什么是 reducer
44
+
45
+ reducer 是一个可插拔的、"边读边算"的统计器:每来一帧都会喂给它,你随时都能用 `result()` 拿到当前的
46
+ 统计结果(没有"结束/收尾"这一步)。调用 `attachReducer` 时,它会先把已经建好索引的历史帧整个重放一遍,
47
+ 再继续跟着后续的新帧走——所以这个方法一返回,统计结果就已经是完整的("一打开,统计就在那儿")。
48
+
49
+ - **内置(已导出)**:`ConversationsReducer`、`EndpointsReducer`、`TcpStreamReducer`
50
+ (统计重传、重复 ACK、往返时延)。其中会话表和端点表建议直接用快捷方法 `analysis.conversations()`
51
+ 和 `analysis.endpoints()`——它们**全程在工作线程里**扫描索引列算出同样的结果,不重新解码、不逐帧在
52
+ 线程之间搬数据、主线程几乎不干活,所以哪怕上亿帧也很快。
53
+ - **便利工厂**:`reduceReducer(seed, fold)`、`groupByReducer(keyOf, seed, fold)`,用来快速拼一个
54
+ 简单的 reducer。
55
+ - **自定义**:实现 `IAnalysisReducer<T>` 接口——见下面的[编写自定义 reducer](#编写自定义-reducer)。
56
+
57
+ ```ts
58
+ import {reduceReducer, groupByReducer} from '@netkitty/analysis'
59
+
60
+ const totalBytes = reduceReducer(0, (sum, frame) => sum + frame.length)
61
+ const perProtocol = groupByReducer(f => f.topProtocol, 0, count => count + 1)
62
+ ```
63
+
64
+ ### 编写自定义 reducer
65
+
66
+ 一个 reducer 就是实现了 `IAnalysisReducer<T>` 的普通对象或类:
67
+
68
+ ```ts
69
+ interface IAnalysisReducer<T> {
70
+ update(frame: Frame, context: UpdateContext): void // 每来一帧调用一次(先重放历史,再走实时)
71
+ result(): T // 随时取当前结果,没有收尾这一步
72
+ reset(): void // 清空已累积的状态
73
+ readonly needs?: string[] // 可选:声明你会用到哪几层协议
74
+ readonly indexOnly?: boolean // 可选:一条关乎正确性的约定,见下文
75
+ }
76
+ ```
77
+
78
+ - **`update(frame, context)`**:每来一帧调用一次。`frame` 里有 `index`、`timestamp`、`length`、
79
+ `topProtocol`、`conversationKey`,以及 `layers`(解码好的各层协议)。`context` 里有 `index`、
80
+ `total` 和 `phase`——重放历史帧时是 `'replay'`,用 `watch()` 跟读实时流时是 `'live'`。
81
+ - **`result()`**:随时都能调,返回当前这一刻的统计结果,没有收尾这一步。**`reset()`**:清空已累积的
82
+ 状态。
83
+ - 累积状态时请按**会话或端点**来聚合,不要一帧一条地堆。内置 reducer 只记录每个会话首帧和末帧的序号,
84
+ 而不是把所有成员帧列出来,所以就算几亿帧,内存占用也基本是平的。
85
+ - **`needs`(可选)**:声明你真正会读到的协议层,比如 `['ipv4', 'tcp']`。这样工作线程只把这几层的
86
+ 数据回传给你,而不是整棵协议树——纯粹省流量,不影响结果。
87
+ - **`indexOnly`(可选,是一条关乎正确性的约定,不只是性能开关)**:只有当你的 `update` 除了帧的基本
88
+ 信息(`index`、`timestamp`、`length`、`topProtocol`、`conversationKey`)之外,**只读取五元组**——
89
+ 也就是收发双方的地址和端口(`eth` 的 `smac`/`dmac`,`ipv4`/`ipv6` 的 `sip`/`dip`,`tcp`/`udp`
90
+ 的 `srcport`/`dstport`)——时,才可以设成 `true`。设成 `true` 之后,重放历史帧会直接用索引列拼出
91
+ 帧、跳过逐帧重新解码,实测在大文件上快大约 13 倍。⚠️ 但如果你设了 `true`,却又读了更深的字段(比如
92
+ `ipv4.ttl`、`tcp.flags`、TLS 的 SNI、载荷等),这些字段在重放时是**取不到的**,结果会悄悄算错。
93
+ 拿不准就别设,默认走完整解码最稳妥。内置的 `ConversationsReducer` 和 `EndpointsReducer` 设了它
94
+ (只用五元组);`TcpStreamReducer` 没设(因为它要读序列号、确认号、标志位)。
95
+
96
+ ```ts
97
+ import {IAnalysisReducer, Frame, UpdateContext} from '@netkitty/analysis'
98
+
99
+ // 按顶层协议统计帧数。只读基本信息,所以设 indexOnly 是安全的。
100
+ class ProtocolCounter implements IAnalysisReducer<Record<string, number>> {
101
+ readonly indexOnly = true
102
+ #counts: Record<string, number> = {}
103
+ update(frame: Frame, _context: UpdateContext): void {
104
+ this.#counts[frame.topProtocol] = (this.#counts[frame.topProtocol] ?? 0) + 1
105
+ }
106
+ result(): Record<string, number> { return {...this.#counts} }
107
+ reset(): void { this.#counts = {} }
108
+ }
109
+ ```
110
+
111
+ reducer 是跑在**主线程**上的——它的 `update` 是个函数,没办法被复制到工作线程里去执行。这不要紧:
112
+ `update` 本身只是些很轻的加加减减;真正费时的读取和解码早在工作线程里做完了,而 `indexOnly` 和 `needs`
113
+ 决定了其中有多少数据要回传到主线程。如果需要,reducer 还可以顺带继承一个事件发射器,在 `update()`
114
+ 里推送自己的 `progress`/`update` 事件。
115
+
116
+ ## 实时抓包(watch)
117
+
118
+ `watch()` 会持续跟读一个还在不断写入的抓包文件,把每一新帧以 `'live'` 阶段喂给 reducer。索引**默认
119
+ 不设上限**(内存多少由你自己把控)。`maxFrames` 是一个可选的内存护栏:一旦超过这个数,就按先进先出的
120
+ 方式淘汰最老的索引条目(类似 Wireshark 的环形缓冲),让长时间跟读也能把内存控制住——注意这只丢内存里
121
+ 的索引,磁盘上的原始文件不受影响。
122
+
123
+ ```ts
124
+ const analysis = new Analysis() // 默认:索引不设上限
125
+ analysis.on('frame', row => { /* 新索引到的一帧 */ })
126
+ await analysis.watch('/path/to/growing.pcap') // 跟读文件;reducer 收到 'live' 阶段
127
+
128
+ // 长时间跟读时的可选内存护栏:
129
+ // new Analysis({maxFrames: 1_000_000}) // 超出上限就先进先出淘汰最老的帧 → 内存有上限
130
+ ```
131
+
132
+ ## 浏览器
133
+
134
+ `Analysis` 这个入口本身不依赖具体运行环境——你只要给它注入一个浏览器版的工作线程通道就行。这个工作线程
135
+ 是打包好的 `analysisWorkerBrowser` 入口(用 esbuild 打包,并带上 Buffer 垫片),数据源换成 `File`
136
+ 或 `Blob`:
137
+
138
+ ```ts
139
+ import {Analysis, WebWorkerChannel} from '@netkitty/analysis'
140
+
141
+ const worker = new Worker(/* 打包后的 analysisWorkerBrowser.js */)
142
+ const analysis = new Analysis({}, () => new WebWorkerChannel(worker))
143
+ await analysis.open(file) // file 是一个 File/Blob
144
+ ```
145
+
146
+ Node 和浏览器下算出来的结果逐字段完全一致。
@@ -0,0 +1,30 @@
1
+ /**
2
+ * Streaming capture-file analysis — the Analysis facade over a single worker, pluggable reducers, a
3
+ * display filter, and the pluggable backend seams. node (worker_threads) and browser (Web Worker)
4
+ * share one environment-agnostic API, verified field-for-field in both.
5
+ */
6
+ export { Analysis } from './lib/streaming/Analysis';
7
+ export type { AnalysisOptions } from './lib/streaming/types/AnalysisOptions';
8
+ export type { AnalysisSource } from './lib/streaming/types/AnalysisSource';
9
+ export type { AnalysisEvents } from './lib/streaming/types/AnalysisEvents';
10
+ export type { AnalysisPhase } from './lib/streaming/types/AnalysisPhase';
11
+ export type { UpdateContext } from './lib/streaming/types/UpdateContext';
12
+ export type { Frame } from './lib/streaming/types/Frame';
13
+ export type { FrameRow } from './lib/streaming/types/FrameRow';
14
+ export type { FrameIndexRecord } from './lib/streaming/types/FrameIndexRecord';
15
+ export type { IAnalysisReducer } from './lib/streaming/interfaces/IAnalysisReducer';
16
+ export type { IReadBackend } from './lib/streaming/interfaces/IReadBackend';
17
+ export type { IWorkerChannel } from './lib/streaming/interfaces/IWorkerChannel';
18
+ export type { IIndexStore } from './lib/streaming/interfaces/IIndexStore';
19
+ /**
20
+ * Built-in reducers (exported, not baked into Analysis) — attach them for Wireshark-style stats.
21
+ */
22
+ export { ConversationsReducer } from './lib/streaming/reducers/ConversationsReducer';
23
+ export type { ConversationSummary } from './lib/streaming/reducers/ConversationsReducer';
24
+ export { EndpointsReducer } from './lib/streaming/reducers/EndpointsReducer';
25
+ export type { EndpointSummary } from './lib/streaming/reducers/EndpointsReducer';
26
+ export { TcpStreamReducer } from './lib/streaming/reducers/TcpStreamReducer';
27
+ export type { RttSample, TcpStreamDiagnostic } from './lib/streaming/reducers/TcpStreamReducer';
28
+ export { reduceReducer, groupByReducer } from './lib/streaming/reducers/ReducerFactories';
29
+ export { parseFilter, matchesFilter, matchesIndexed, indexableEval } from './lib/streaming/filter/FilterExpression';
30
+ export type { FilterExpression, FilterPredicate } from './lib/streaming/filter/FilterExpression';
package/dist/index.js ADDED
@@ -0,0 +1,27 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.indexableEval = exports.matchesIndexed = exports.matchesFilter = exports.parseFilter = exports.groupByReducer = exports.reduceReducer = exports.TcpStreamReducer = exports.EndpointsReducer = exports.ConversationsReducer = exports.Analysis = void 0;
4
+ /**
5
+ * Streaming capture-file analysis — the Analysis facade over a single worker, pluggable reducers, a
6
+ * display filter, and the pluggable backend seams. node (worker_threads) and browser (Web Worker)
7
+ * share one environment-agnostic API, verified field-for-field in both.
8
+ */
9
+ var Analysis_1 = require("./lib/streaming/Analysis");
10
+ Object.defineProperty(exports, "Analysis", { enumerable: true, get: function () { return Analysis_1.Analysis; } });
11
+ /**
12
+ * Built-in reducers (exported, not baked into Analysis) — attach them for Wireshark-style stats.
13
+ */
14
+ var ConversationsReducer_1 = require("./lib/streaming/reducers/ConversationsReducer");
15
+ Object.defineProperty(exports, "ConversationsReducer", { enumerable: true, get: function () { return ConversationsReducer_1.ConversationsReducer; } });
16
+ var EndpointsReducer_1 = require("./lib/streaming/reducers/EndpointsReducer");
17
+ Object.defineProperty(exports, "EndpointsReducer", { enumerable: true, get: function () { return EndpointsReducer_1.EndpointsReducer; } });
18
+ var TcpStreamReducer_1 = require("./lib/streaming/reducers/TcpStreamReducer");
19
+ Object.defineProperty(exports, "TcpStreamReducer", { enumerable: true, get: function () { return TcpStreamReducer_1.TcpStreamReducer; } });
20
+ var ReducerFactories_1 = require("./lib/streaming/reducers/ReducerFactories");
21
+ Object.defineProperty(exports, "reduceReducer", { enumerable: true, get: function () { return ReducerFactories_1.reduceReducer; } });
22
+ Object.defineProperty(exports, "groupByReducer", { enumerable: true, get: function () { return ReducerFactories_1.groupByReducer; } });
23
+ var FilterExpression_1 = require("./lib/streaming/filter/FilterExpression");
24
+ Object.defineProperty(exports, "parseFilter", { enumerable: true, get: function () { return FilterExpression_1.parseFilter; } });
25
+ Object.defineProperty(exports, "matchesFilter", { enumerable: true, get: function () { return FilterExpression_1.matchesFilter; } });
26
+ Object.defineProperty(exports, "matchesIndexed", { enumerable: true, get: function () { return FilterExpression_1.matchesIndexed; } });
27
+ Object.defineProperty(exports, "indexableEval", { enumerable: true, get: function () { return FilterExpression_1.indexableEval; } });
@@ -0,0 +1,54 @@
1
+ import { AnalysisOptions } from './types/AnalysisOptions';
2
+ import { AnalysisSource } from './types/AnalysisSource';
3
+ import { AnalysisEvents } from './types/AnalysisEvents';
4
+ import { Frame } from './types/Frame';
5
+ import { FrameRow } from './types/FrameRow';
6
+ import { IAnalysisReducer } from './interfaces/IAnalysisReducer';
7
+ import { IWorkerChannel } from './interfaces/IWorkerChannel';
8
+ import { ConversationSummary } from './reducers/ConversationsReducer';
9
+ import { EndpointSummary } from './reducers/EndpointsReducer';
10
+ /**
11
+ * Capture-file analysis facade — a programmable Wireshark-style front door over a pcap/pcapng file.
12
+ * Environment-agnostic: it holds no fs/File/codec at runtime, only talks to a worker channel. Heavy
13
+ * work (read → parse → index → decode) lives in a single worker so open() never blocks the caller and
14
+ * close() releases everything by terminating it. The node channel factory is loaded lazily (require)
15
+ * so this module never statically imports worker_threads; a browser build injects its own factory.
16
+ */
17
+ export declare class Analysis {
18
+ #private;
19
+ constructor(options?: AnalysisOptions, spawnChannel?: () => IWorkerChannel);
20
+ /** Read a bounded capture: build the frame index, emit 'complete'. Resolves once indexing finishes. */
21
+ open(source: AnalysisSource): Promise<void>;
22
+ /**
23
+ * Tail a capture being written live: the worker indexes appended frames continuously and pushes
24
+ * each one here, where it updates frameCount, emits a 'frame' row, and feeds every attached reducer
25
+ * with phase 'live'. Never emits 'complete'. Index memory is bounded by AnalysisOptions.maxFrames.
26
+ */
27
+ watch(source: AnalysisSource): Promise<void>;
28
+ /** Terminate the worker and release the index, cache, and parse state in one shot. */
29
+ close(): Promise<void>;
30
+ frameCount(): number;
31
+ /**
32
+ * Conversation table, computed inside the worker by scanning the index columns — no re-decode, no
33
+ * per-frame cross-thread transfer, no main-thread work. Equivalent to attaching a ConversationsReducer.
34
+ */
35
+ conversations(): Promise<ConversationSummary[]>;
36
+ /** Per-endpoint tx/rx totals, computed inside the worker over the index columns (see conversations()). */
37
+ endpoints(): Promise<EndpointSummary[]>;
38
+ /** A single frame with decoded layers (served through the worker's on-demand re-parse). */
39
+ getFrame(index: number): Promise<Frame | null>;
40
+ /** A range of lightweight rows (no decoded layers), for list display. Half-open [from, to). */
41
+ getFrames(from: number, to: number): Promise<FrameRow[]>;
42
+ /** Evaluate a display filter (v1 subset), returning the matching frame indices. */
43
+ filter(displayFilter: string): Promise<number[]>;
44
+ /**
45
+ * Attach a reducer and replay every already-indexed frame into it (phase 'replay'), so stats are
46
+ * complete the moment attach resolves — Wireshark-style "open the stats and they're already there".
47
+ * Frames are pulled from the worker in bounded batches. (watch live-feed is added with watch().)
48
+ */
49
+ attachReducer(reducer: IAnalysisReducer<unknown>): Promise<void>;
50
+ /** Detach a reducer: stop feeding it and drop the internal reference (its state is the caller's). */
51
+ detachReducer(reducer: IAnalysisReducer<unknown>): void;
52
+ on<E extends keyof AnalysisEvents>(event: E, listener: AnalysisEvents[E]): this;
53
+ off<E extends keyof AnalysisEvents>(event: E, listener: AnalysisEvents[E]): this;
54
+ }
@@ -0,0 +1,159 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.Analysis = void 0;
4
+ //How many frames to pull per replay batch — bounds memory and back-pressures the worker naturally.
5
+ const REPLAY_BATCH = 512;
6
+ /**
7
+ * Capture-file analysis facade — a programmable Wireshark-style front door over a pcap/pcapng file.
8
+ * Environment-agnostic: it holds no fs/File/codec at runtime, only talks to a worker channel. Heavy
9
+ * work (read → parse → index → decode) lives in a single worker so open() never blocks the caller and
10
+ * close() releases everything by terminating it. The node channel factory is loaded lazily (require)
11
+ * so this module never statically imports worker_threads; a browser build injects its own factory.
12
+ */
13
+ class Analysis {
14
+ #options;
15
+ #spawnChannel;
16
+ #channel = null;
17
+ #frameCount = 0;
18
+ #reducers = new Set();
19
+ #listeners = new Map();
20
+ constructor(options = {}, spawnChannel) {
21
+ this.#options = options;
22
+ this.#spawnChannel = spawnChannel !== undefined ? spawnChannel : () => {
23
+ //Lazy require keeps this module environment-agnostic — it never statically imports the
24
+ //node-only worker_threads channel (a browser build injects its own factory instead).
25
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
26
+ const factory = require('./worker/spawnNodeAnalysisChannel');
27
+ return factory.spawnNodeAnalysisChannel();
28
+ };
29
+ }
30
+ /** Read a bounded capture: build the frame index, emit 'complete'. Resolves once indexing finishes. */
31
+ async open(source) {
32
+ const channel = this.#start();
33
+ const result = await channel.request('open', { source: source });
34
+ this.#frameCount = result.frameCount;
35
+ }
36
+ /**
37
+ * Tail a capture being written live: the worker indexes appended frames continuously and pushes
38
+ * each one here, where it updates frameCount, emits a 'frame' row, and feeds every attached reducer
39
+ * with phase 'live'. Never emits 'complete'. Index memory is bounded by AnalysisOptions.maxFrames.
40
+ */
41
+ async watch(source) {
42
+ const channel = this.#start();
43
+ channel.on('frame', (payload) => {
44
+ const frame = payload;
45
+ if (frame.index + 1 > this.#frameCount)
46
+ this.#frameCount = frame.index + 1;
47
+ const row = { index: frame.index, timestamp: frame.timestamp, length: frame.length, topProtocol: frame.topProtocol, conversationKey: frame.conversationKey, info: frame.info };
48
+ this.#emit('frame', row);
49
+ for (const reducer of this.#reducers) {
50
+ reducer.update(frame, { index: frame.index, total: this.#frameCount, phase: 'live' });
51
+ }
52
+ });
53
+ const result = await channel.request('watch', { source: source, maxFrames: this.#options.maxFrames });
54
+ if (result.frameCount > this.#frameCount)
55
+ this.#frameCount = result.frameCount;
56
+ }
57
+ /** Terminate the worker and release the index, cache, and parse state in one shot. */
58
+ async close() {
59
+ if (this.#channel) {
60
+ this.#channel.terminate();
61
+ this.#channel = null;
62
+ }
63
+ this.#frameCount = 0;
64
+ }
65
+ frameCount() {
66
+ return this.#frameCount;
67
+ }
68
+ /**
69
+ * Conversation table, computed inside the worker by scanning the index columns — no re-decode, no
70
+ * per-frame cross-thread transfer, no main-thread work. Equivalent to attaching a ConversationsReducer.
71
+ */
72
+ async conversations() {
73
+ return this.#require().request('conversations');
74
+ }
75
+ /** Per-endpoint tx/rx totals, computed inside the worker over the index columns (see conversations()). */
76
+ async endpoints() {
77
+ return this.#require().request('endpoints');
78
+ }
79
+ /** A single frame with decoded layers (served through the worker's on-demand re-parse). */
80
+ async getFrame(index) {
81
+ return this.#require().request('getFrame', { index: index });
82
+ }
83
+ /** A range of lightweight rows (no decoded layers), for list display. Half-open [from, to). */
84
+ async getFrames(from, to) {
85
+ return this.#require().request('getFrames', { from: from, to: to });
86
+ }
87
+ /** Evaluate a display filter (v1 subset), returning the matching frame indices. */
88
+ async filter(displayFilter) {
89
+ return this.#require().request('filter', { displayFilter: displayFilter });
90
+ }
91
+ /**
92
+ * Attach a reducer and replay every already-indexed frame into it (phase 'replay'), so stats are
93
+ * complete the moment attach resolves — Wireshark-style "open the stats and they're already there".
94
+ * Frames are pulled from the worker in bounded batches. (watch live-feed is added with watch().)
95
+ */
96
+ async attachReducer(reducer) {
97
+ const channel = this.#require();
98
+ this.#reducers.add(reducer);
99
+ const needs = reducer.needs;
100
+ const indexOnly = reducer.indexOnly === true;
101
+ const total = this.#frameCount;
102
+ for (let from = 0; from < total; from += REPLAY_BATCH) {
103
+ const to = Math.min(from + REPLAY_BATCH, total);
104
+ const frames = await channel.request('getFrameBatch', { from: from, to: to, needs: needs, indexOnly: indexOnly });
105
+ for (const frame of frames) {
106
+ const context = { index: frame.index, total: total, phase: 'replay' };
107
+ reducer.update(frame, context);
108
+ }
109
+ }
110
+ }
111
+ /** Detach a reducer: stop feeding it and drop the internal reference (its state is the caller's). */
112
+ detachReducer(reducer) {
113
+ this.#reducers.delete(reducer);
114
+ }
115
+ on(event, listener) {
116
+ let set = this.#listeners.get(event);
117
+ if (!set) {
118
+ set = new Set();
119
+ this.#listeners.set(event, set);
120
+ }
121
+ set.add(listener);
122
+ return this;
123
+ }
124
+ off(event, listener) {
125
+ this.#listeners.get(event)?.delete(listener);
126
+ return this;
127
+ }
128
+ #start() {
129
+ const channel = this.#spawnChannel();
130
+ this.#channel = channel;
131
+ channel.on('progress', (payload) => {
132
+ const p = payload;
133
+ this.#frameCount = p.frames;
134
+ this.#emit('progress', p.bytesRead, p.totalBytes);
135
+ });
136
+ channel.on('complete', (payload) => {
137
+ const c = payload;
138
+ this.#frameCount = c.frameCount;
139
+ this.#emit('complete');
140
+ });
141
+ channel.on('error', (payload) => {
142
+ this.#emit('error', payload instanceof Error ? payload : new Error(String(payload)));
143
+ });
144
+ return channel;
145
+ }
146
+ #require() {
147
+ if (!this.#channel)
148
+ throw new Error('Analysis has no open source; call open() or watch() first');
149
+ return this.#channel;
150
+ }
151
+ #emit(event, ...args) {
152
+ const set = this.#listeners.get(event);
153
+ if (!set)
154
+ return;
155
+ for (const listener of set)
156
+ listener(...args);
157
+ }
158
+ }
159
+ exports.Analysis = Analysis;
@@ -0,0 +1,14 @@
1
+ import { IReadBackend } from '../interfaces/IReadBackend';
2
+ /**
3
+ * Browser IReadBackend over a File/Blob: Blob.slice gives random access (for on-demand re-parsing)
4
+ * and a chunked async iterator (for index building), both via arrayBuffer(). A Blob is immutable, so
5
+ * there is nothing to tail (watch is absent) and no handle to release (close is absent). Works in any
6
+ * environment with the Blob API — node (>=18) included, which is how it is unit-tested.
7
+ */
8
+ export declare class BrowserFileReadBackend implements IReadBackend {
9
+ #private;
10
+ constructor(source: Blob, chunkSize?: number);
11
+ size(): Promise<number>;
12
+ read(offset: number, length: number): Promise<Uint8Array>;
13
+ createStream(): AsyncIterableIterator<Uint8Array>;
14
+ }
@@ -0,0 +1,36 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.BrowserFileReadBackend = void 0;
4
+ /**
5
+ * Browser IReadBackend over a File/Blob: Blob.slice gives random access (for on-demand re-parsing)
6
+ * and a chunked async iterator (for index building), both via arrayBuffer(). A Blob is immutable, so
7
+ * there is nothing to tail (watch is absent) and no handle to release (close is absent). Works in any
8
+ * environment with the Blob API — node (>=18) included, which is how it is unit-tested.
9
+ */
10
+ class BrowserFileReadBackend {
11
+ #blob;
12
+ #chunkSize;
13
+ constructor(source, chunkSize = 1 << 16) {
14
+ this.#blob = source;
15
+ this.#chunkSize = chunkSize;
16
+ }
17
+ async size() {
18
+ return this.#blob.size;
19
+ }
20
+ async read(offset, length) {
21
+ const slice = this.#blob.slice(offset, offset + length);
22
+ return new Uint8Array(await slice.arrayBuffer());
23
+ }
24
+ async *createStream() {
25
+ let position = 0;
26
+ while (position < this.#blob.size) {
27
+ const slice = this.#blob.slice(position, position + this.#chunkSize);
28
+ const bytes = new Uint8Array(await slice.arrayBuffer());
29
+ if (bytes.length === 0)
30
+ break;
31
+ position += bytes.length;
32
+ yield bytes;
33
+ }
34
+ }
35
+ }
36
+ exports.BrowserFileReadBackend = BrowserFileReadBackend;