@eltjs/chunking 0.0.0-stage → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +302 -2
- package/dist/chunking.d.ts +69 -0
- package/dist/chunking.js +216 -0
- package/package.json +47 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright 2026 ezzabuzaid
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -1,3 +1,303 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @eltjs/chunking
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
`Chunking` is a rule of a computed dataset. It cuts the text of each loaded row into chunks. Each chunk counts at most `maxTokens` tokens, as the tokenizer of the embedding model counts them. Then the `Embedding` rule of [`@eltjs/embedding`](../embedding/README.md) makes one vector for each chunk.
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
npm install @eltjs/chunking gpt-tokenizer
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
Without chunks, `Embedding` makes one vector for each loaded row. A long SharePoint proposal is then one vector. When its text counts more than 8,191 tokens, the limit of OpenAI's `text-embedding-3-small`, it is one failed row, and each pass tries it again ([ADR 0007 of elt-sqlite](../destinations/sqlite/docs/adr/0007-each-pass-tries-an-input-at-most-once.md)). One vector of a long text also loses the meaning of each part of it. `Chunking` is a table rule of elt: it gives many parts for one input ([ADR 0026 of elt](../elt/docs/adr/0026-a-rule-can-give-many-parts-for-one-input.md)). Ezz chose on 2026-11-19 to solve this in the elt core, not with a SQL view that cuts by characters.
|
|
10
|
+
|
|
11
|
+
The words in these documents have one meaning each. See the glossary in [CONTEXT.md](./CONTEXT.md), and the glossary of [elt](../elt/CONTEXT.md) for the words of all of elt.
|
|
12
|
+
|
|
13
|
+
## Cut documents into chunks, then embed each chunk
|
|
14
|
+
|
|
15
|
+
This example works the same for each destination that keeps the parts of a table rule (ADR 0026 of elt): `elt-sqlite` today, then `elt-postgresql` and `elt-sqlserver` (#4736). Only the import of `View` and `ComputedDataset` changes. The [recipe](./docs/recipes/cut-documents-into-chunks-and-embed-each-chunk.md) runs it end to end on SQLite.
|
|
16
|
+
|
|
17
|
+
```ts
|
|
18
|
+
import { countTokens } from 'gpt-tokenizer/encoding/cl100k_base';
|
|
19
|
+
|
|
20
|
+
import { Chunking } from '@eltjs/chunking';
|
|
21
|
+
import { DocumentText } from '@eltjs/elt';
|
|
22
|
+
import { ComputedDataset, View } from '@eltjs/elt-sqlite';
|
|
23
|
+
import { Embedding } from '@eltjs/embedding';
|
|
24
|
+
|
|
25
|
+
// 1. The text of each stored document (a computed dataset that exists today).
|
|
26
|
+
const text = new ComputedDataset({
|
|
27
|
+
into: destination,
|
|
28
|
+
name: 'document_text',
|
|
29
|
+
description: 'The text of each stored document.',
|
|
30
|
+
from: [documents],
|
|
31
|
+
compute: new DocumentText({
|
|
32
|
+
parser: () => parser,
|
|
33
|
+
files: files.directory,
|
|
34
|
+
column: 'documentRef',
|
|
35
|
+
}),
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
// 2. A computed dataset reads a table or a view, never another computed
|
|
39
|
+
// dataset, so a view gives the text that the parser read.
|
|
40
|
+
const readable = new View({
|
|
41
|
+
into: destination,
|
|
42
|
+
name: 'document_text_readable',
|
|
43
|
+
description: 'The text of each document that the parser read.',
|
|
44
|
+
columns: { ref: 'The ref of the stored file.', content: 'Its text.' },
|
|
45
|
+
sql: "SELECT ref, content FROM document_text WHERE status = 'computed'",
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
// 3. One output row for each document, and one part for each chunk.
|
|
49
|
+
const chunks = new ComputedDataset({
|
|
50
|
+
into: destination,
|
|
51
|
+
name: 'document_chunks',
|
|
52
|
+
description: 'The chunks of the text of each document.',
|
|
53
|
+
from: [readable],
|
|
54
|
+
compute: new Chunking({
|
|
55
|
+
tokenizer: { name: 'cl100k_base', count: countTokens },
|
|
56
|
+
maxTokens: 800,
|
|
57
|
+
overlapTokens: 400,
|
|
58
|
+
key: 'ref',
|
|
59
|
+
fields: ['content'],
|
|
60
|
+
}),
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
// 4. A view gives each chunk with its part hash as its key.
|
|
64
|
+
const chunkTexts = new View({
|
|
65
|
+
into: destination,
|
|
66
|
+
name: 'document_chunk_texts',
|
|
67
|
+
description: 'The text of each chunk, by its part hash.',
|
|
68
|
+
columns: {
|
|
69
|
+
id: 'The part hash of the chunk.',
|
|
70
|
+
text: 'The text of the chunk.',
|
|
71
|
+
},
|
|
72
|
+
sql: 'SELECT part_hash AS id, text FROM document_chunks_parts',
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
// 5. One vector for each chunk text.
|
|
76
|
+
const vectors = new ComputedDataset({
|
|
77
|
+
into: destination,
|
|
78
|
+
name: 'document_chunk_vectors',
|
|
79
|
+
description: 'The vector of each chunk text.',
|
|
80
|
+
from: [chunkTexts],
|
|
81
|
+
compute: new Embedding({
|
|
82
|
+
model,
|
|
83
|
+
dimensions: 1536,
|
|
84
|
+
clock: globalThis,
|
|
85
|
+
key: 'id',
|
|
86
|
+
fields: ['text'],
|
|
87
|
+
}),
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
new Pipeline({
|
|
91
|
+
history,
|
|
92
|
+
connections,
|
|
93
|
+
derived: [text, readable, chunks, chunkTexts, vectors],
|
|
94
|
+
});
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
- `tokenizer` is the tokenizer of the embedding model. `text-embedding-3-small` uses `cl100k_base`. The package does not choose a tokenizer, because each model has its own ([ADR 0002](./docs/adr/0002-the-host-s-tokenizer-measures-each-chunk-and-no-chunk-passes-the-limit.md)).
|
|
98
|
+
- `maxTokens` and `overlapTokens` have no default. The example uses the defaults of OpenAI's own file search, 800 and 400 ([ADR 0004](./docs/adr/0004-the-host-gives-the-two-sizes-in-tokens-and-the-overlap-is-at-most-half-a-chunk.md)). Datahub takes its sizes from an eval on its own proposals.
|
|
99
|
+
- `key` and `fields` are as in `Embedding`: the column that identifies a row, and the columns whose text the rule cuts, joined with line breaks. A row with no text in these columns has no chunks.
|
|
100
|
+
- A search joins `document_chunk_vectors` to `document_chunks_parts` on `id = part_hash`, and gets the ref, the order, the text and the offsets of each chunk that matched.
|
|
101
|
+
- The sentence and word boundaries come from `Intl.Segmenter` with the root locale, `und`. Node resolves `und` to the default locale of the process, but V8 gave the same sentences for `und`, `en`, `de` and `ja` (probe, Node 26.13.0).
|
|
102
|
+
|
|
103
|
+
## The signatures
|
|
104
|
+
|
|
105
|
+
```ts
|
|
106
|
+
// @eltjs/elt (ADR 0026 of elt)
|
|
107
|
+
export type TableRuleResult<Part> =
|
|
108
|
+
| { readonly parts: readonly [Part, ...Part[]] }
|
|
109
|
+
| { readonly empty: true }
|
|
110
|
+
| { readonly error: string }
|
|
111
|
+
| { readonly gone: true };
|
|
112
|
+
|
|
113
|
+
export abstract class TableRule<
|
|
114
|
+
Input,
|
|
115
|
+
Columns extends RuleColumns, // the content columns of each part, in its part hash
|
|
116
|
+
Place extends RuleColumns, // the place columns of each part, not in its part hash
|
|
117
|
+
> {
|
|
118
|
+
abstract readonly key: {
|
|
119
|
+
readonly name: string;
|
|
120
|
+
readonly description: string;
|
|
121
|
+
};
|
|
122
|
+
abstract readonly output: Columns;
|
|
123
|
+
abstract readonly place: Place;
|
|
124
|
+
abstract readonly batch: number;
|
|
125
|
+
abstract readonly concurrency: number;
|
|
126
|
+
abstract read(
|
|
127
|
+
row: LoadedRow,
|
|
128
|
+
): { readonly key: string; readonly input: Input } | null;
|
|
129
|
+
abstract identity(input: Input): Promise<string | null>;
|
|
130
|
+
abstract compute(
|
|
131
|
+
inputs: readonly Input[],
|
|
132
|
+
signal?: AbortSignal,
|
|
133
|
+
): Promise<
|
|
134
|
+
readonly TableRuleResult<RuleValues<Columns> & RuleValues<Place>>[]
|
|
135
|
+
>;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// @eltjs/chunking
|
|
139
|
+
export type Tokenizer = {
|
|
140
|
+
// The name of the vocabulary, such as cl100k_base. It is part of the rule identity.
|
|
141
|
+
readonly name: string;
|
|
142
|
+
// How many tokens the model counts in text.
|
|
143
|
+
count(text: string): number;
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
const content = {
|
|
147
|
+
text: { type: 'text', description: '…' },
|
|
148
|
+
tokens: { type: 'integer', description: '…' },
|
|
149
|
+
} as const;
|
|
150
|
+
const place = {
|
|
151
|
+
start: { type: 'integer', description: '…' },
|
|
152
|
+
end: { type: 'integer', description: '…' },
|
|
153
|
+
} as const;
|
|
154
|
+
|
|
155
|
+
export class Chunking extends TableRule<string, typeof content, typeof place> {
|
|
156
|
+
constructor(options: {
|
|
157
|
+
tokenizer: Tokenizer;
|
|
158
|
+
maxTokens: number;
|
|
159
|
+
overlapTokens: number;
|
|
160
|
+
key: string;
|
|
161
|
+
fields: readonly string[];
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
`Chunking` has two content columns, `text` and `tokens`, and two place columns, `start` and `end` ([ADR 0006](./docs/adr/0006-each-chunk-gives-its-offsets-and-its-count-of-tokens.md)). Its batch is 32 inputs and its concurrency is 1: the work is synchronous, so calls side by side gain nothing, and a batch keeps the commits few.
|
|
167
|
+
|
|
168
|
+
## What happens underneath
|
|
169
|
+
|
|
170
|
+
| Call or event | What happens |
|
|
171
|
+
| ------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
172
|
+
| `new Chunking({ … })` | Refuses a size that is not an integer, `maxTokens` below 1, `overlapTokens` below 0, and `overlapTokens` above half of `maxTokens`, with a `RangeError` that names the value |
|
|
173
|
+
| `read(row)` | Gives the key and the text of `fields`, joined with line breaks. `null` when the row has no key or no text |
|
|
174
|
+
| `identity(text)` | `@eltjs/chunking@<version> <tokenizer.name> <maxTokens>/<overlapTokens>`, the same for each input ([ADR 0005](./docs/adr/0005-the-rule-identity-of-a-chunk-is-the-package-version-the-tokenizer-and-the-two-sizes.md)) |
|
|
175
|
+
| `compute(texts)`, a text that fits | One part: the text with its white space at the start and the end removed |
|
|
176
|
+
| `compute(texts)`, a longer text | Cuts at the largest boundary that fits: paragraph, line, sentence, word, grapheme ([ADR 0003](./docs/adr/0003-a-chunk-ends-at-the-largest-boundary-that-fits.md)). Counts the text of each chunk with `tokenizer.count` |
|
|
177
|
+
| `compute(texts)`, overlap | Each chunk after the first starts with the last whole pieces of the chunk before it, up to `overlapTokens` tokens. Each chunk starts after the start of the chunk before it |
|
|
178
|
+
| a refresh writes an output | Writes the output row and its parts in one transaction. A part with the same values keeps its commit. A part beyond the new count goes into the history ([ADR 0026 of elt](../elt/docs/adr/0026-a-rule-can-give-many-parts-for-one-input.md)) |
|
|
179
|
+
| the text of a document changes | Its input hash changes, so the refresh cuts it again. The chunks before the change have the same text, the same part hash, and keep their vectors. The chunks after it only move: their offsets change, and their part hashes do not. Only new chunk texts go to the model |
|
|
180
|
+
| the package is released, or a size changes | The rule identity changes, so the next refresh cuts each text again. Each chunk text that did not change keeps its part hash and its vector |
|
|
181
|
+
| the document is removed | The refresh deletes its output row and its parts. The vector of a part hash that no chunk has any more is deleted by the refresh of the vectors |
|
|
182
|
+
|
|
183
|
+
## The columns
|
|
184
|
+
|
|
185
|
+
The computed dataset `document_chunks` has two reader views. `document_chunks` has one row for each key, with the columns of each computed dataset: the key, `input_hash`, `rule`, `status` and `error`. `document_chunks_parts` has one row for each chunk:
|
|
186
|
+
|
|
187
|
+
| Column | Holds |
|
|
188
|
+
| ----------- | -------------------------------------------------------------------------------------------------------------------------- |
|
|
189
|
+
| the key | the key of the input, with the name that the rule gives, for example `ref` |
|
|
190
|
+
| `seq` | the order of the chunk in its text, from 0 |
|
|
191
|
+
| `part_hash` | the SHA-256 of the content columns of the part, `text` and `tokens`. Two chunks with the same text have the same part hash |
|
|
192
|
+
| `text` | the text of the chunk: a slice of the input, with no white space at the start or the end |
|
|
193
|
+
| `tokens` | the count of tokens of `text`, as `tokenizer.count` gives it |
|
|
194
|
+
| `start` | the offset of the first code point of the chunk in the input, from 0 |
|
|
195
|
+
| `end` | the offset after the last code point of the chunk, so the chunk is the slice from `start` to `end` |
|
|
196
|
+
|
|
197
|
+
A key whose status is `failed` or `empty` has no parts.
|
|
198
|
+
|
|
199
|
+
## When something fails
|
|
200
|
+
|
|
201
|
+
| Failure | What happens | What the caller does |
|
|
202
|
+
| ----------------------------------------- | -------------------------------------------------------------------------------------------------------- | --------------------------------------------- |
|
|
203
|
+
| `RangeError` from the constructor | A size that is not possible, such as an overlap of more than half a chunk | Change the size |
|
|
204
|
+
| One grapheme counts more than `maxTokens` | A rule failure of that input: the output row is `failed`, and the error gives the offset of the grapheme | Give a larger `maxTokens` |
|
|
205
|
+
| `tokenizer.count` throws | A rule failure of each input of the call. The next pass tries again (ADR 0004 of elt) | Fix the tokenizer |
|
|
206
|
+
| A text that the model still refuses | `Embedding` splits its call in halves, and only the row of that chunk fails | Check that the tokenizer is that of the model |
|
|
207
|
+
|
|
208
|
+
## The map
|
|
209
|
+
|
|
210
|
+
| Now | Later | Never |
|
|
211
|
+
| -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------- |
|
|
212
|
+
| `TableRule` and parts in `@eltjs/elt`, and a parts table in each destination (ADR 0026 of elt) | Boundaries from the structure of a document, such as headings and tables, when a `DocumentParser` gives structure (Unstructured's `chunk_by_title`) | A chunk size in characters for a model whose limit is in tokens (Ezz, 2026-11-19) |
|
|
213
|
+
| `part_hash`, so that a dataset that reads the parts keys them by content | `from` names a computed dataset, with no view between them, when a third host writes the same view (ADR 0027 of elt) | A chunker inside `Embedding`, as pgai's vectorizer has its `chunking` inside it (ADR 0001) |
|
|
214
|
+
| `Chunking`: boundaries of Unicode text segmentation, a fit that the host's tokenizer checks, overlap in tokens | Semantic breakpoints from the vectors of sentences (LlamaIndex `SemanticSplitterNodeParser`, .NET `SemanticSimilarityChunker`). This needs model calls | A random id for each chunk (LlamaIndex `uuid4`, .NET `Guid.NewGuid()`): a refresh compares states by key |
|
|
215
|
+
| Integer rule columns, and the `tokens`, `start` and `end` of each chunk (ADR 0006) | | A default chunk size |
|
|
216
|
+
| | A ready `Tokenizer` for OpenAI's vocabularies, when a third host writes the same two lines over `gpt-tokenizer` | A vocabulary that loads from the network at run time (LangChain's `tiktoken.pages.dev`) |
|
|
217
|
+
| | A `locale` for sentence boundaries | A cut inside a character (LangChain's `TokenTextSplitter`) |
|
|
218
|
+
|
|
219
|
+
## The same concepts in other systems
|
|
220
|
+
|
|
221
|
+
| Here | LangChain | LlamaIndex | pgai vectorizer | Unstructured | Haystack | .NET |
|
|
222
|
+
| ------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
223
|
+
| `Chunking` | [`RecursiveCharacterTextSplitter`, `TokenTextSplitter`](https://github.com/langchain-ai/langchainjs/blob/main/libs/langchain-textsplitters/src/text_splitter.ts) | [`SentenceSplitter`](https://github.com/run-llama/llama_index/blob/main/llama-index-core/llama_index/core/node_parser/text/sentence.py) | [`ai.chunking_recursive_character_text_splitter`](https://github.com/timescale/pgai/blob/main/docs/vectorizer/api-reference.md) | [`chunk_by_title`, `basic`](https://github.com/Unstructured-IO/unstructured/blob/main/unstructured/chunking/base.py) | [`DocumentSplitter`](https://github.com/deepset-ai/haystack/blob/main/haystack/components/preprocessors/document_splitter.py) | [`DocumentTokenChunker`](https://github.com/dotnet/extensions/blob/main/src/Libraries/Microsoft.Extensions.DataIngestion/Chunkers/DocumentTokenChunker.cs) |
|
|
224
|
+
| `maxTokens` | `chunkSize`, 1,000 characters (length function `text.length`) | `chunk_size`, 1,024 tokens | `chunk_size`, 800 characters | `max_characters` 500, or `max_tokens` with a `tokenizer` | `split_length`, 200 words; `split_by="token"` also | `MaxTokensPerChunk`, 2,000 tokens |
|
|
225
|
+
| `overlapTokens` | `chunkOverlap`, 200; refuses overlap ≥ size | `chunk_overlap`, 200 tokens | `chunk_overlap`, 400 characters | `overlap` 0, only inside a split element unless `overlap_all` | `split_overlap`, 0 | `OverlapTokens`, 500 |
|
|
226
|
+
| `tokenizer` | `lengthFunction`; `encodingName` `gpt2` for the token splitter | `tokenizer` callable; default tiktoken for `gpt-3.5-turbo` (`cl100k_base`) | none: counts characters; the embedder knows the 8,191-token limit and records the error | `tokenizer`, a tiktoken name | `tokenizer_encoding`, default `o200k_base` | `Tokenizer` of `Microsoft.ML.Tokenizers`, required |
|
|
227
|
+
| Boundaries | `["\n\n", "\n", " ", ""]` | paragraphs (`"\n\n\n"`), NLTK sentences, a regex of `,.;。?!`, words, characters | `["\n\n", "\n", ".", "?", "!", " ", ""]` | the elements of the document: titles, text, tables | words, sentences, pages, lines | the elements of the document, then tokens |
|
|
228
|
+
| An over-size chunk | logs a warning and keeps it (`text_splitter.js`, "Created a chunk of size …") | splits to characters | the model refuses it, and the error stays | splits the element | no limit of tokens, unless `split_by="token"` | splits by tokens |
|
|
229
|
+
| Key of a chunk | none | `uuid4()` (`node_utils.py`, `default_id_func`), with `ref_doc_id` | `(source primary key, chunk_seq)` and `embedding_uuid` | none | the SHA-256 of the content and the metadata, with `source_id` and `split_id` | `Guid.NewGuid()`, with `documentid` |
|
|
230
|
+
| The source changes | the caller decides | `IngestionPipeline` with `UPSERTS` deletes the nodes of the document and adds the new ones | deletes each embedding of the source row, then adds the new chunks (`vectorizer.py`, `_delete_embeddings`) | the caller decides | the caller decides | `VectorStoreWriter` adds the new chunks, then deletes the chunks that the document had |
|
|
231
|
+
|
|
232
|
+
OpenAI's own file search cuts by tokens: `max_chunk_size_tokens` 800 (100 to 4,096) and `chunk_overlap_tokens` 400, at most half of a chunk (the `StaticFileChunkingStrategy` type of the `openai` SDK). In Go, `langchaingo/textsplitter` copies LangChain: `RecursiveCharacter` with `utf8.RuneCountInString` and `TokenSplitter` with tiktoken-go.
|
|
233
|
+
|
|
234
|
+
Each leader with a key for each chunk keeps the order of the chunk in its source (pgai `chunk_seq`, Haystack `split_id`). Each leader that keeps chunks in step with a source replaces all the chunks of the source when it changes. elt does the same, and a part with the same values keeps its commit.
|
|
235
|
+
|
|
236
|
+
## Why no npm package does the job
|
|
237
|
+
|
|
238
|
+
The requirements as checks: (1) each chunk counts at most `maxTokens` tokens of the model's vocabulary; (2) overlap in tokens; (3) a cut at the largest boundary that fits, never inside a character; (4) the same text gives the same chunks; (5) no network at run time; (6) the tokenizer is the caller's. Each candidate was probed in an empty folder on 2026-11-19, Node 26.13.0.
|
|
239
|
+
|
|
240
|
+
| Package | Version, released | Downloads (last week) | Install footprint | Why it fails |
|
|
241
|
+
| -------------------------- | ------------------ | --------------------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
242
|
+
| `@langchain/textsplitters` | 1.0.2, 2026-09-27 | 2,058,170 | 152 KB, and the peer `@langchain/core` (13 packages, with langsmith) | `TokenTextSplitter` decodes windows of tokens: 13 of 22 chunks of a text with accents, emoji and Japanese held U+FFFD (3). It loads the vocabulary from `tiktoken.pages.dev` (5). `RecursiveCharacterTextSplitter` sums the lengths of pieces and only warns when a chunk is too long (1); it has no sentence boundary (3) |
|
|
243
|
+
| `@llamaindex/node-parser` | 2.0.23, 2026-03-09 | 138,928 | 36 packages, `tree-sitter` with an install script | Its nodes are LlamaIndex documents with `uuid4` ids (4); a framework for one function |
|
|
244
|
+
| `@chonkiejs/core` | 0.0.12, 2026-10-03 | 27,088 | 4 packages, a native tree-sitter binary for each platform | Its default tokenizer counts characters. Tokens need `@chonkiejs/token`, which takes `@huggingface/transformers` and loads the tokenizer from the Hugging Face Hub (5). Version 0.0.x |
|
|
245
|
+
| `@mastra/rag` | 2.8.1, 2026-11-17 | 279,441 | 211 packages (AWS Bedrock client and others) | A framework of retrieval; the footprint is for one function |
|
|
246
|
+
| `llm-splitter` | 0.3.0, 2026-08-25 | 20,255 | 68 KB, 1 package | It packs the units of a splitter function, with no boundaries: with a token splitter, 9 of 31 chunks started inside a word (3) |
|
|
247
|
+
| `semantic-chunking` | 3.0.2, 2026-08-14 | 250 | `@huggingface/transformers` and others | Semantic chunking with a local model; not a token fit |
|
|
248
|
+
|
|
249
|
+
No package does all six. The cut is the knowledge of this package. The count is not: `gpt-tokenizer` does it.
|
|
250
|
+
|
|
251
|
+
## Tokenizers for OpenAI's vocabularies in JavaScript
|
|
252
|
+
|
|
253
|
+
| Package | Version, released | Downloads (last week) | Footprint | Probe on 420,000 tokens of `cl100k_base` |
|
|
254
|
+
| ------------------------- | ------------------ | --------------------- | ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- |
|
|
255
|
+
| `gpt-tokenizer` | 4.1.2, 2026-11-10 | 2,721,243 | 1 package, no dependencies, 30 MB (each vocabulary is its own subpath) | 41 ms. `isWithinTokenLimit` answered `false` for 8,191 in 2 ms. Same count as js-tiktoken. MIT |
|
|
256
|
+
| `js-tiktoken` | 1.0.21, 2025-08-09 | 13,130,133 | 2 packages, 21 MB | 481 ms |
|
|
257
|
+
| `tiktoken` | 1.0.22, 2025-08-09 | 2,234,448 | WASM | not probed: a WASM binary to load in each runtime |
|
|
258
|
+
| `@huggingface/tokenizers` | 0.2.0, 2026-09-07 | 4,126,201 | 1 package | runs a `tokenizer.json`; OpenAI ships no such file, so a host must get one |
|
|
259
|
+
| `tokenx` | 2.2.1, 2026-11-14 | 2,546,118 | 2 kB | an estimate ("95%+ average accuracy"): no limit holds on an estimate |
|
|
260
|
+
|
|
261
|
+
`js-tiktoken` names `cl100k_base` for `text-embedding-3-small` (`getEncodingNameForModel`). deepagents (`packages/context/src/lib/estimate.ts`) and datahub already take `gpt-tokenizer`. Node 26 has no tokenizer. It has `Intl.Segmenter`, which this package uses for boundaries.
|
|
262
|
+
|
|
263
|
+
## Who uses what
|
|
264
|
+
|
|
265
|
+
| Package or host | What it takes |
|
|
266
|
+
| -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
|
|
267
|
+
| datahub, `apps/semantic-layer/elt` (backlog #4740) | `Chunking` over `sharepoint_document_text`, then `Embedding` with `text-embedding-3-small`, kept in Azure SQL (#4736) |
|
|
268
|
+
| `@eltjs/elt` | gives `TableRule`, `TableRuleResult`, `LoadedRow` and `RuleColumn`, with its `integer` kind |
|
|
269
|
+
| each destination | keeps the parts of a `TableRule` (ADR 0026 of elt) |
|
|
270
|
+
| `gpt-tokenizer` | taken by the host, not by this package, for a `Tokenizer` of OpenAI's vocabularies |
|
|
271
|
+
|
|
272
|
+
## Who owns each job
|
|
273
|
+
|
|
274
|
+
| Job | Who can know it | Owner |
|
|
275
|
+
| ----------------------------------------------------------------------- | --------------------------------------- | -------------------------------------------- |
|
|
276
|
+
| Which columns hold the key and the text | the host | the host: `key`, `fields` |
|
|
277
|
+
| Which tokenizer the model uses, and its limit | the host, which picks the model | the host: `tokenizer`, `maxTokens` |
|
|
278
|
+
| How many tokens a text counts | the tokenizer library | `gpt-tokenizer`, behind the `Tokenizer` port |
|
|
279
|
+
| Where to cut, how to pack, the overlap, the fit | the domain of chunking | `@eltjs/chunking` |
|
|
280
|
+
| The rule identity of the chunks | this package: its version and its sizes | `@eltjs/chunking` |
|
|
281
|
+
| One input to many parts: store, replace, history, positions | the engine and each destination | `@eltjs/elt` and each destination (ADR 0026) |
|
|
282
|
+
| Which part columns go to the model, a filter, a title before each chunk | the host | the host's `View` (ADR 0027) |
|
|
283
|
+
| The vector of each chunk | the embedding model | `@eltjs/embedding`, unchanged |
|
|
284
|
+
| The rank of a document from its chunks, such as its best chunk (MaxP) | the host | the host's search |
|
|
285
|
+
|
|
286
|
+
## Ezz's rulings (2026-11-19)
|
|
287
|
+
|
|
288
|
+
1. **The name of the engine kind** is `TableRule`, with "part" for each row it gives ([ADR 0026 of elt](../elt/docs/adr/0026-a-rule-can-give-many-parts-for-one-input.md)).
|
|
289
|
+
2. **The order with #4649.** The parts are written one time, in the refresh that moves into elt before the SQL Server destination ([ADR 0012 of elt-sqlserver](../destinations/sqlserver/docs/adr/0012-the-rules-that-the-destinations-share-move-into-elt-before-the-third-destination.md)).
|
|
290
|
+
3. **The key of a chunk for the model** is its part hash, so a chunk whose text did not change keeps its vector.
|
|
291
|
+
4. **The sizes for SharePoint proposals** come from an eval on datahub's own proposals, as for staff search (#3350).
|
|
292
|
+
5. **Offsets and counts of each chunk** come now: `tokens` as a content column, `start` and `end` as place columns, out of the part hash ([ADR 0006](./docs/adr/0006-each-chunk-gives-its-offsets-and-its-count-of-tokens.md)).
|
|
293
|
+
6. **The limit of the overlap** is half of a chunk ([ADR 0004](./docs/adr/0004-the-host-gives-the-two-sizes-in-tokens-and-the-overlap-is-at-most-half-a-chunk.md)).
|
|
294
|
+
|
|
295
|
+
## Decisions
|
|
296
|
+
|
|
297
|
+
- [ADR 0001](./docs/adr/0001-chunking-is-a-rule-of-its-own-package-that-runs-before-the-embedding.md): chunking is a rule of its own package that runs before the embedding.
|
|
298
|
+
- [ADR 0002](./docs/adr/0002-the-host-s-tokenizer-measures-each-chunk-and-no-chunk-passes-the-limit.md): the host's tokenizer measures each chunk, and no chunk passes the limit.
|
|
299
|
+
- [ADR 0003](./docs/adr/0003-a-chunk-ends-at-the-largest-boundary-that-fits.md): a chunk ends at the largest boundary that fits.
|
|
300
|
+
- [ADR 0004](./docs/adr/0004-the-host-gives-the-two-sizes-in-tokens-and-the-overlap-is-at-most-half-a-chunk.md): the host gives the two sizes in tokens, and the overlap is at most half a chunk.
|
|
301
|
+
- [ADR 0005](./docs/adr/0005-the-rule-identity-of-a-chunk-is-the-package-version-the-tokenizer-and-the-two-sizes.md): the rule identity of a chunk is the package version, the tokenizer and the two sizes.
|
|
302
|
+
- [ADR 0006](./docs/adr/0006-each-chunk-gives-its-offsets-and-its-count-of-tokens.md): each chunk gives its offsets and its count of tokens.
|
|
303
|
+
- In elt: [ADR 0026](../elt/docs/adr/0026-a-rule-can-give-many-parts-for-one-input.md), a rule can give many parts for one input, and [ADR 0027](../elt/docs/adr/0027-a-dataset-reads-the-parts-of-a-computed-dataset-through-a-view.md), a dataset reads the parts of a computed dataset through a view.
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { type LoadedRow, type RuleValues, TableRule, type TableRuleResult } from '@eltjs/elt';
|
|
2
|
+
export type Tokenizer = {
|
|
3
|
+
readonly name: string;
|
|
4
|
+
count(text: string): number;
|
|
5
|
+
};
|
|
6
|
+
declare const content: {
|
|
7
|
+
readonly text: {
|
|
8
|
+
readonly type: 'text';
|
|
9
|
+
readonly description: 'The text of the chunk: a slice of the input, with no white space at its start or end.';
|
|
10
|
+
};
|
|
11
|
+
readonly tokens: {
|
|
12
|
+
readonly type: 'integer';
|
|
13
|
+
readonly description: 'How many tokens the text of the chunk counts.';
|
|
14
|
+
};
|
|
15
|
+
};
|
|
16
|
+
declare const place: {
|
|
17
|
+
readonly start: {
|
|
18
|
+
readonly type: 'integer';
|
|
19
|
+
readonly description: 'The offset of the first code point of the chunk in the input, from 0.';
|
|
20
|
+
};
|
|
21
|
+
readonly end: {
|
|
22
|
+
readonly type: 'integer';
|
|
23
|
+
readonly description: 'The offset after the last code point of the chunk in the input.';
|
|
24
|
+
};
|
|
25
|
+
};
|
|
26
|
+
type Chunk = RuleValues<typeof content> & RuleValues<typeof place>;
|
|
27
|
+
export declare class Chunking extends TableRule<string, typeof content, typeof place> {
|
|
28
|
+
#private;
|
|
29
|
+
readonly key: {
|
|
30
|
+
readonly name: string;
|
|
31
|
+
readonly description: string;
|
|
32
|
+
};
|
|
33
|
+
readonly output: {
|
|
34
|
+
readonly text: {
|
|
35
|
+
readonly type: 'text';
|
|
36
|
+
readonly description: 'The text of the chunk: a slice of the input, with no white space at its start or end.';
|
|
37
|
+
};
|
|
38
|
+
readonly tokens: {
|
|
39
|
+
readonly type: 'integer';
|
|
40
|
+
readonly description: 'How many tokens the text of the chunk counts.';
|
|
41
|
+
};
|
|
42
|
+
};
|
|
43
|
+
readonly place: {
|
|
44
|
+
readonly start: {
|
|
45
|
+
readonly type: 'integer';
|
|
46
|
+
readonly description: 'The offset of the first code point of the chunk in the input, from 0.';
|
|
47
|
+
};
|
|
48
|
+
readonly end: {
|
|
49
|
+
readonly type: 'integer';
|
|
50
|
+
readonly description: 'The offset after the last code point of the chunk in the input.';
|
|
51
|
+
};
|
|
52
|
+
};
|
|
53
|
+
readonly batch = 32;
|
|
54
|
+
readonly concurrency = 1;
|
|
55
|
+
constructor({ tokenizer, maxTokens, overlapTokens, key, fields, }: {
|
|
56
|
+
tokenizer: Tokenizer;
|
|
57
|
+
maxTokens: number;
|
|
58
|
+
overlapTokens: number;
|
|
59
|
+
key: string;
|
|
60
|
+
fields: readonly string[];
|
|
61
|
+
});
|
|
62
|
+
read(row: LoadedRow): {
|
|
63
|
+
readonly key: string;
|
|
64
|
+
readonly input: string;
|
|
65
|
+
} | null;
|
|
66
|
+
identity(): Promise<string>;
|
|
67
|
+
compute(texts: readonly string[]): Promise<readonly TableRuleResult<Chunk>[]>;
|
|
68
|
+
}
|
|
69
|
+
export {};
|
package/dist/chunking.js
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { TableRule, errorMessage, } from '@eltjs/elt';
|
|
2
|
+
import manifest from '../package.json' with { type: 'json' };
|
|
3
|
+
const content = {
|
|
4
|
+
text: {
|
|
5
|
+
type: 'text',
|
|
6
|
+
description: 'The text of the chunk: a slice of the input, with no white space at its start or end.',
|
|
7
|
+
},
|
|
8
|
+
tokens: {
|
|
9
|
+
type: 'integer',
|
|
10
|
+
description: 'How many tokens the text of the chunk counts.',
|
|
11
|
+
},
|
|
12
|
+
};
|
|
13
|
+
const place = {
|
|
14
|
+
start: {
|
|
15
|
+
type: 'integer',
|
|
16
|
+
description: 'The offset of the first code point of the chunk in the input, from 0.',
|
|
17
|
+
},
|
|
18
|
+
end: {
|
|
19
|
+
type: 'integer',
|
|
20
|
+
description: 'The offset after the last code point of the chunk in the input.',
|
|
21
|
+
},
|
|
22
|
+
};
|
|
23
|
+
// The ends of the pieces of a text at each boundary, from the largest: a
|
|
24
|
+
// paragraph (two or more line breaks), a line, a sentence, a word and a
|
|
25
|
+
// grapheme (ADR 0003). Each piece keeps its separator, so the pieces join back
|
|
26
|
+
// into the text. Intl.Segmenter applies Unicode text segmentation (UAX #29);
|
|
27
|
+
// the rule asks for the root locale.
|
|
28
|
+
const boundaries = [
|
|
29
|
+
(text) => Array.from(text.matchAll(/\n\s*\n/g), (match) => match.index + match[0].length),
|
|
30
|
+
(text) => Array.from(text.matchAll(/\n/g), (match) => match.index + 1),
|
|
31
|
+
...['sentence', 'word', 'grapheme'].map((granularity) => {
|
|
32
|
+
const segmenter = new Intl.Segmenter('und', { granularity });
|
|
33
|
+
return (text) => Array.from(segmenter.segment(text), ({ index, segment }) => index + segment.length);
|
|
34
|
+
}),
|
|
35
|
+
];
|
|
36
|
+
// The chunks of each loaded row's text, each counting at most maxTokens
|
|
37
|
+
// tokens of the embedding model's tokenizer, so the Embedding rule can make
|
|
38
|
+
// one vector for each chunk (ADR 0001). A chunk ends at the largest boundary
|
|
39
|
+
// that fits, and starts with the last whole pieces of the chunk before it,
|
|
40
|
+
// up to overlapTokens tokens.
|
|
41
|
+
export class Chunking extends TableRule {
|
|
42
|
+
key;
|
|
43
|
+
output = content;
|
|
44
|
+
place = place;
|
|
45
|
+
// Cutting is synchronous work, so calls side by side gain nothing, and a
|
|
46
|
+
// batch keeps the commits few.
|
|
47
|
+
batch = 32;
|
|
48
|
+
concurrency = 1;
|
|
49
|
+
#tokenizer;
|
|
50
|
+
#maxTokens;
|
|
51
|
+
#overlapTokens;
|
|
52
|
+
#fields;
|
|
53
|
+
// tokenizer: the tokenizer of the embedding model. maxTokens: the most
|
|
54
|
+
// tokens of a chunk. overlapTokens: the most tokens a chunk repeats of the
|
|
55
|
+
// chunk before it, at most half of maxTokens (ADR 0004). key: the column
|
|
56
|
+
// that identifies a row. fields: the columns whose text the rule cuts,
|
|
57
|
+
// joined by line breaks; a row with none has no chunks.
|
|
58
|
+
constructor({ tokenizer, maxTokens, overlapTokens, key, fields, }) {
|
|
59
|
+
super();
|
|
60
|
+
if (!Number.isInteger(maxTokens) || maxTokens < 1)
|
|
61
|
+
throw new RangeError(`maxTokens is an integer of 1 or more, not ${maxTokens}.`);
|
|
62
|
+
if (!Number.isInteger(overlapTokens) || overlapTokens < 0)
|
|
63
|
+
throw new RangeError(`overlapTokens is an integer of 0 or more, not ${overlapTokens}.`);
|
|
64
|
+
if (overlapTokens > maxTokens / 2)
|
|
65
|
+
throw new RangeError(`overlapTokens is at most half of maxTokens ${maxTokens}, not ${overlapTokens}.`);
|
|
66
|
+
this.#tokenizer = tokenizer;
|
|
67
|
+
this.#maxTokens = maxTokens;
|
|
68
|
+
this.#overlapTokens = overlapTokens;
|
|
69
|
+
this.key = {
|
|
70
|
+
name: key,
|
|
71
|
+
description: `The ${key} of the row the chunks are cut from.`,
|
|
72
|
+
};
|
|
73
|
+
this.#fields = Object.freeze([...fields]);
|
|
74
|
+
Object.freeze(this);
|
|
75
|
+
}
|
|
76
|
+
read(row) {
|
|
77
|
+
const key = row[this.key.name];
|
|
78
|
+
if (typeof key !== 'string' && typeof key !== 'number')
|
|
79
|
+
return null;
|
|
80
|
+
const text = this.#fields
|
|
81
|
+
.map((field) => row[field])
|
|
82
|
+
.filter((value) => typeof value === 'string' && value.trim() !== '')
|
|
83
|
+
.join('\n');
|
|
84
|
+
return text === '' ? null : { key: String(key), input: text };
|
|
85
|
+
}
|
|
86
|
+
// The code that cuts, the vocabulary and the sizes make every chunk, so
|
|
87
|
+
// they name them all (ADR 0005).
|
|
88
|
+
async identity() {
|
|
89
|
+
return `${manifest.name}@${manifest.version} ${this.#tokenizer.name} ${this.#maxTokens}/${this.#overlapTokens}`;
|
|
90
|
+
}
|
|
91
|
+
async compute(texts) {
|
|
92
|
+
try {
|
|
93
|
+
return texts.map((text) => this.#cut(text));
|
|
94
|
+
}
|
|
95
|
+
catch (failure) {
|
|
96
|
+
// A tokenizer that throws says nothing about one text: each input of
|
|
97
|
+
// the call fails, and the next pass tries again.
|
|
98
|
+
const error = errorMessage(failure);
|
|
99
|
+
return texts.map(() => ({ error }));
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
#cut(text) {
|
|
103
|
+
const pieces = this.#split(text, 0, text.length, 0);
|
|
104
|
+
if ('grapheme' in pieces)
|
|
105
|
+
return {
|
|
106
|
+
error: `The grapheme at offset ${[...text.slice(0, pieces.grapheme)].length} counts ${pieces.tokens} tokens, more than maxTokens ${this.#maxTokens}.`,
|
|
107
|
+
};
|
|
108
|
+
const [first, ...rest] = this.#pack(text, pieces);
|
|
109
|
+
return first === undefined ? { empty: true } : { parts: [first, ...rest] };
|
|
110
|
+
}
|
|
111
|
+
// The pieces of the text between from and to that each fit, cut at the largest
|
|
112
|
+
// boundary that makes them fit. A piece that fits at one boundary is not cut
|
|
113
|
+
// at a smaller one.
|
|
114
|
+
#split(text, from, to, level) {
|
|
115
|
+
const tokens = this.#count(text.slice(from, to));
|
|
116
|
+
if (tokens <= this.#maxTokens)
|
|
117
|
+
return [{ from, to }];
|
|
118
|
+
const ends = boundaries[level];
|
|
119
|
+
// Past the grapheme boundary, the piece is one grapheme.
|
|
120
|
+
if (ends === undefined)
|
|
121
|
+
return { grapheme: from, tokens };
|
|
122
|
+
const pieces = [];
|
|
123
|
+
let start = from;
|
|
124
|
+
for (const end of [...ends(text.slice(from, to)), to - from]) {
|
|
125
|
+
if (from + end <= start)
|
|
126
|
+
continue;
|
|
127
|
+
const split = this.#split(text, start, from + end, level + 1);
|
|
128
|
+
if ('grapheme' in split)
|
|
129
|
+
return split;
|
|
130
|
+
pieces.push(...split);
|
|
131
|
+
start = from + end;
|
|
132
|
+
}
|
|
133
|
+
return pieces;
|
|
134
|
+
}
|
|
135
|
+
// The chunks of the pieces in order: a piece joins the chunk while the chunk
|
|
136
|
+
// fits. The next chunk starts with the last whole pieces of the chunk, up to
|
|
137
|
+
// overlapTokens, that start after its text starts, so each chunk starts
|
|
138
|
+
// after the start of the chunk before it.
|
|
139
|
+
#pack(text, pieces) {
|
|
140
|
+
const chunks = [];
|
|
141
|
+
const offsets = new Offsets(text);
|
|
142
|
+
// The starts of the pieces in the chunk being packed.
|
|
143
|
+
let starts = [];
|
|
144
|
+
for (const { from, to } of pieces) {
|
|
145
|
+
const [start] = starts;
|
|
146
|
+
if (start !== undefined && !this.#fits(text.slice(start, to))) {
|
|
147
|
+
const visible = start + leadingSpace(text.slice(start, from));
|
|
148
|
+
chunks.push(...this.#chunk(text, visible, from, offsets));
|
|
149
|
+
starts = this.#overlap(text, starts.filter((piece) => piece > visible), from, to);
|
|
150
|
+
}
|
|
151
|
+
starts.push(from);
|
|
152
|
+
}
|
|
153
|
+
const [start] = starts;
|
|
154
|
+
if (start !== undefined) {
|
|
155
|
+
const visible = start + leadingSpace(text.slice(start));
|
|
156
|
+
chunks.push(...this.#chunk(text, visible, text.length, offsets));
|
|
157
|
+
}
|
|
158
|
+
return chunks;
|
|
159
|
+
}
|
|
160
|
+
// The last of the starts whose text up to from counts at most overlapTokens,
|
|
161
|
+
// and whose text with the next piece, up to to, fits.
|
|
162
|
+
#overlap(text, starts, from, to) {
|
|
163
|
+
const kept = [];
|
|
164
|
+
for (const start of starts.toReversed()) {
|
|
165
|
+
if (this.#count(text.slice(start, from)) > this.#overlapTokens ||
|
|
166
|
+
!this.#fits(text.slice(start, to)))
|
|
167
|
+
break;
|
|
168
|
+
kept.unshift(start);
|
|
169
|
+
}
|
|
170
|
+
return kept;
|
|
171
|
+
}
|
|
172
|
+
// The chunk of text from start, its first code point that is not white
|
|
173
|
+
// space, to end, with no white space at its end; none when it holds only
|
|
174
|
+
// white space.
|
|
175
|
+
#chunk(text, start, end, offsets) {
|
|
176
|
+
const trimmed = text.slice(start, end).trimEnd();
|
|
177
|
+
if (trimmed === '')
|
|
178
|
+
return [];
|
|
179
|
+
const point = offsets.at(start);
|
|
180
|
+
return [
|
|
181
|
+
{
|
|
182
|
+
text: trimmed,
|
|
183
|
+
tokens: this.#tokenizer.count(trimmed),
|
|
184
|
+
start: point,
|
|
185
|
+
end: point + [...trimmed].length,
|
|
186
|
+
},
|
|
187
|
+
];
|
|
188
|
+
}
|
|
189
|
+
#fits(text) {
|
|
190
|
+
return this.#count(text) <= this.#maxTokens;
|
|
191
|
+
}
|
|
192
|
+
// The tokens of a text as its chunk holds it, with no white space at its
|
|
193
|
+
// start or end.
|
|
194
|
+
#count(text) {
|
|
195
|
+
return this.#tokenizer.count(text.trim());
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
// How many UTF-16 units of white space a text starts with.
|
|
199
|
+
function leadingSpace(text) {
|
|
200
|
+
return text.length - text.trimStart().length;
|
|
201
|
+
}
|
|
202
|
+
// The offset in code points of a place in UTF-16 units of a text, for places
|
|
203
|
+
// asked in order, so the text is read once.
|
|
204
|
+
class Offsets {
|
|
205
|
+
#text;
|
|
206
|
+
#unit = 0;
|
|
207
|
+
#point = 0;
|
|
208
|
+
constructor(text) {
|
|
209
|
+
this.#text = text;
|
|
210
|
+
}
|
|
211
|
+
at(unit) {
|
|
212
|
+
this.#point += [...this.#text.slice(this.#unit, unit)].length;
|
|
213
|
+
this.#unit = unit;
|
|
214
|
+
return this.#point;
|
|
215
|
+
}
|
|
216
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,49 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@eltjs/chunking",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.7.0",
|
|
4
|
+
"description": "A rule of an elt computed dataset: it cuts the text of each loaded row into chunks that each count at most a number of tokens of the embedding model's tokenizer, so each chunk can get its own vector.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "https://github.com/ezzabuzaid/elt",
|
|
9
|
+
"directory": "packages/chunking"
|
|
10
|
+
},
|
|
11
|
+
"bugs": {
|
|
12
|
+
"url": "https://github.com/ezzabuzaid/elt/issues"
|
|
13
|
+
},
|
|
14
|
+
"homepage": "https://github.com/ezzabuzaid/elt/tree/main/packages/chunking#readme",
|
|
15
|
+
"keywords": [
|
|
16
|
+
"elt",
|
|
17
|
+
"chunking",
|
|
18
|
+
"text-splitter",
|
|
19
|
+
"tokens",
|
|
20
|
+
"embedding",
|
|
21
|
+
"rag"
|
|
22
|
+
],
|
|
23
|
+
"type": "module",
|
|
24
|
+
"exports": {
|
|
25
|
+
"./package.json": "./package.json",
|
|
26
|
+
".": {
|
|
27
|
+
"types": "./dist/chunking.d.ts",
|
|
28
|
+
"default": "./dist/chunking.js"
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
"files": [
|
|
32
|
+
"dist",
|
|
33
|
+
"!dist/spec/**",
|
|
34
|
+
"!**/*.tsbuildinfo",
|
|
35
|
+
"!**/*.test.*"
|
|
36
|
+
],
|
|
37
|
+
"publishConfig": {
|
|
38
|
+
"access": "public"
|
|
39
|
+
},
|
|
40
|
+
"dependencies": {
|
|
41
|
+
"@eltjs/elt": "^0.7.0"
|
|
42
|
+
},
|
|
43
|
+
"devDependencies": {
|
|
44
|
+
"@eltjs/elt-sqlite": "^0.7.0",
|
|
45
|
+
"@zukhruf/fs": "^0.8.31",
|
|
46
|
+
"@zukhruf/sqlite-open": "^0.8.31",
|
|
47
|
+
"gpt-tokenizer": "^4.1.2"
|
|
48
|
+
}
|
|
49
|
+
}
|