@h-ai/datapipe 0.1.0-alpha.56 → 0.1.0-alpha.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +80 -0
- package/README.ja.md +80 -0
- package/README.md +4 -0
- package/package.json +5 -5
package/README.en.md
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# @h-ai/datapipe
|
|
2
|
+
|
|
3
|
+
[简体中文](./README.md) | [English](./README.en.md) | [日本語](./README.ja.md)
|
|
4
|
+
|
|
5
|
+
[Framework home](../../README.en.md) · Related modules: [@h-ai/core](../core/README.en.md)
|
|
6
|
+
|
|
7
|
+
A data processing pipeline module for text cleaning (`clean`), chunking (`chunk`), and composable pipelines (`pipeline`). Pure functions; no initialization required.
|
|
8
|
+
|
|
9
|
+
## Features
|
|
10
|
+
|
|
11
|
+
| Feature | Description |
|
|
12
|
+
| -------------------- | ----------------------------------------------------------------------------------------------------- |
|
|
13
|
+
| Text cleaning | Remove HTML tags, URLs, and emails; normalize whitespace; apply custom regex replacements |
|
|
14
|
+
| Text chunking | Sentences, paragraphs, Markdown headings, page breaks, word counts, characters, and custom separators |
|
|
15
|
+
| Pipeline composition | Chain cleaning → transformation → chunking → postprocessing in a single execution |
|
|
16
|
+
|
|
17
|
+
> Security: `customReplacements` and `separator` in `mode: 'custom'` are used as regular expressions. Only provide trusted, reviewed patterns; do not interpolate user input directly into regexes, to avoid ReDoS or unintended broad replacements.
|
|
18
|
+
|
|
19
|
+
## Quick start
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
import { datapipe, HaiDatapipeError } from '@h-ai/datapipe'
|
|
23
|
+
|
|
24
|
+
// 直接清洗
|
|
25
|
+
const cleaned = datapipe.clean('<p>Hello</p>', { removeHtml: true })
|
|
26
|
+
|
|
27
|
+
// 按 Markdown 标题分块
|
|
28
|
+
const chunks = datapipe.chunk(text, { mode: 'markdown', maxSize: 2000 })
|
|
29
|
+
|
|
30
|
+
// 管线模式（链式组合多步操作）
|
|
31
|
+
const result = await datapipe.pipeline()
|
|
32
|
+
.clean({ removeHtml: true, removeUrls: true })
|
|
33
|
+
.transform(text => text.toLowerCase())
|
|
34
|
+
.chunk({ mode: 'paragraph', maxSize: 1000, overlap: 100 })
|
|
35
|
+
.chunkTransform(chunks => chunks.filter(c => c.content.length > 50))
|
|
36
|
+
.run(rawText)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## Chunking modes
|
|
40
|
+
|
|
41
|
+
| Mode | Description |
|
|
42
|
+
| --------- | ------------------------------------- |
|
|
43
|
+
| sentence | Split by sentence |
|
|
44
|
+
| paragraph | Split by paragraph |
|
|
45
|
+
| markdown | Split by Markdown heading |
|
|
46
|
+
| page | Split by page break |
|
|
47
|
+
| word | Split by word count |
|
|
48
|
+
| character | Split by character count |
|
|
49
|
+
| custom | Custom separator (requires separator) |
|
|
50
|
+
|
|
51
|
+
## Error handling
|
|
52
|
+
|
|
53
|
+
All operations return `HaiResult<T>`; check `result.success` for success or failure. Error codes use the string format `hai:datapipe:NNN`.
|
|
54
|
+
|
|
55
|
+
```ts
|
|
56
|
+
const result = datapipe.chunk(text, { mode: 'custom' })
|
|
57
|
+
if (!result.success) {
|
|
58
|
+
switch (result.error.code) {
|
|
59
|
+
case HaiDatapipeError.MISSING_SEPARATOR.code:
|
|
60
|
+
// mode='custom' 时需要提供 separator
|
|
61
|
+
break
|
|
62
|
+
case HaiDatapipeError.CONFIG_ERROR.code:
|
|
63
|
+
// 配置参数校验失败
|
|
64
|
+
break
|
|
65
|
+
case HaiDatapipeError.PROCESSING_ERROR.code:
|
|
66
|
+
// 输入或自定义正则导致处理失败；记录错误并回退到安全默认策略
|
|
67
|
+
break
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Tests
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pnpm --filter @h-ai/datapipe test
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## License
|
|
79
|
+
|
|
80
|
+
Apache-2.0
|
package/README.ja.md
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# @h-ai/datapipe
|
|
2
|
+
|
|
3
|
+
[简体中文](./README.md) | [English](./README.en.md) | [日本語](./README.ja.md)
|
|
4
|
+
|
|
5
|
+
[フレームワークのホーム](../../README.ja.md) · 関連モジュール：[@h-ai/core](../core/README.ja.md)
|
|
6
|
+
|
|
7
|
+
テキストのクリーニング（clean）、分割（chunk）、パイプラインの合成（pipeline）を提供するデータ処理モジュールです。純粋関数で構成され、初期化は不要です。
|
|
8
|
+
|
|
9
|
+
## 主な機能
|
|
10
|
+
|
|
11
|
+
| 機能 | 説明 |
|
|
12
|
+
| ---------------------- | --------------------------------------------------------------------- |
|
|
13
|
+
| テキストのクリーニング | HTML タグ・URL・Email の除去、空白の正規化、独自の正規表現による置換 |
|
|
14
|
+
| テキストの分割 | 文、段落、Markdown 見出し、改ページ、単語数、文字数、独自の区切り文字 |
|
|
15
|
+
| パイプラインの合成 | クリーニング → 変換 → 分割 → 後処理を連結し、一度に実行 |
|
|
16
|
+
|
|
17
|
+
> セキュリティ：`customReplacements` と `mode: 'custom'` の `separator` は正規表現として使用されます。信頼できる検証済みのパターンのみを渡し、ReDoS や意図しない広範囲の置換を防ぐため、ユーザー入力を直接正規表現に組み込まないでください。
|
|
18
|
+
|
|
19
|
+
## クイックスタート
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
import { datapipe, HaiDatapipeError } from '@h-ai/datapipe'
|
|
23
|
+
|
|
24
|
+
// 直接清洗
|
|
25
|
+
const cleaned = datapipe.clean('<p>Hello</p>', { removeHtml: true })
|
|
26
|
+
|
|
27
|
+
// 按 Markdown 标题分块
|
|
28
|
+
const chunks = datapipe.chunk(text, { mode: 'markdown', maxSize: 2000 })
|
|
29
|
+
|
|
30
|
+
// 管线模式（链式组合多步操作）
|
|
31
|
+
const result = await datapipe.pipeline()
|
|
32
|
+
.clean({ removeHtml: true, removeUrls: true })
|
|
33
|
+
.transform(text => text.toLowerCase())
|
|
34
|
+
.chunk({ mode: 'paragraph', maxSize: 1000, overlap: 100 })
|
|
35
|
+
.chunkTransform(chunks => chunks.filter(c => c.content.length > 50))
|
|
36
|
+
.run(rawText)
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
## 分割モード
|
|
40
|
+
|
|
41
|
+
| モード | 説明 |
|
|
42
|
+
| --------- | ------------------------------------ |
|
|
43
|
+
| sentence | 文単位で分割 |
|
|
44
|
+
| paragraph | 段落単位で分割 |
|
|
45
|
+
| markdown | Markdown 見出しで分割 |
|
|
46
|
+
| page | 改ページで分割 |
|
|
47
|
+
| word | 単語数で分割 |
|
|
48
|
+
| character | 文字数で分割 |
|
|
49
|
+
| custom | 独自の区切り文字（separator が必要） |
|
|
50
|
+
|
|
51
|
+
## エラー処理
|
|
52
|
+
|
|
53
|
+
すべての操作は `HaiResult<T>` を返します。`result.success` で成功・失敗を判定します。エラーコードは文字列形式 `hai:datapipe:NNN` です。
|
|
54
|
+
|
|
55
|
+
```ts
|
|
56
|
+
const result = datapipe.chunk(text, { mode: 'custom' })
|
|
57
|
+
if (!result.success) {
|
|
58
|
+
switch (result.error.code) {
|
|
59
|
+
case HaiDatapipeError.MISSING_SEPARATOR.code:
|
|
60
|
+
// mode='custom' 时需要提供 separator
|
|
61
|
+
break
|
|
62
|
+
case HaiDatapipeError.CONFIG_ERROR.code:
|
|
63
|
+
// 配置参数校验失败
|
|
64
|
+
break
|
|
65
|
+
case HaiDatapipeError.PROCESSING_ERROR.code:
|
|
66
|
+
// 输入或自定义正则导致处理失败；记录错误并回退到安全默认策略
|
|
67
|
+
break
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## テスト
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
pnpm --filter @h-ai/datapipe test
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
## License
|
|
79
|
+
|
|
80
|
+
Apache-2.0
|
package/README.md
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@h-ai/datapipe",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.1.0-alpha.
|
|
4
|
+
"version": "0.1.0-alpha.58",
|
|
5
5
|
"description": "Data processing pipeline module for hai-framework: clean, chunk, transform",
|
|
6
6
|
"exports": {
|
|
7
7
|
".": {
|
|
@@ -18,11 +18,11 @@
|
|
|
18
18
|
"dist"
|
|
19
19
|
],
|
|
20
20
|
"engines": {
|
|
21
|
-
"node": "
|
|
21
|
+
"node": "^22.22.2 || ^24.15.0 || >=26.0.0"
|
|
22
22
|
},
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"
|
|
25
|
-
"
|
|
24
|
+
"@h-ai/core": "0.1.0-alpha.58",
|
|
25
|
+
"zod": "^4.6.5"
|
|
26
26
|
},
|
|
27
27
|
"devDependencies": {
|
|
28
28
|
"tsup": "^8.5.1",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"scripts": {
|
|
32
32
|
"build": "tsup",
|
|
33
33
|
"dev": "tsup --watch",
|
|
34
|
-
"typecheck": "
|
|
34
|
+
"typecheck": "node ../../scripts/typecheck.mjs --noEmit",
|
|
35
35
|
"lint": "eslint src/",
|
|
36
36
|
"lint:fix": "eslint src/ --fix",
|
|
37
37
|
"test": "vitest run",
|