glost-processor 0.7.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +262 -262
- package/package.json +4 -4
- package/src/__benchmarks__/stream-processor.bench.d.ts +8 -0
- package/src/__benchmarks__/stream-processor.bench.d.ts.map +1 -0
- package/src/__benchmarks__/stream-processor.bench.js +202 -0
- package/src/__benchmarks__/stream-processor.bench.js.map +1 -0
- package/src/index.d.ts +59 -0
- package/src/index.d.ts.map +1 -0
- package/src/index.js +57 -0
- package/src/index.js.map +1 -0
- package/src/processor.d.ts +256 -0
- package/src/processor.d.ts.map +1 -0
- package/src/processor.js +482 -0
- package/src/processor.js.map +1 -0
- package/src/stream-processor.d.ts +145 -0
- package/src/stream-processor.d.ts.map +1 -0
- package/src/stream-processor.js +245 -0
- package/src/stream-processor.js.map +1 -0
- package/src/types.d.ts +167 -0
- package/src/types.d.ts.map +1 -0
- package/src/types.js +9 -0
- package/src/types.js.map +1 -0
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* GLOST Stream Processor
|
|
3
|
+
*
|
|
4
|
+
* Streaming variant of GLOSTProcessor that yields processed sentence
|
|
5
|
+
* batches progressively using AsyncGenerator. Keeps the full document
|
|
6
|
+
* out of memory between chunks.
|
|
7
|
+
*
|
|
8
|
+
* Document-level transforms (extensions with streamingSupport !== 'chunk')
|
|
9
|
+
* run once on the full document before streaming begins. Chunk-compatible
|
|
10
|
+
* extensions (streamingSupport === 'chunk') then run on each batch.
|
|
11
|
+
*
|
|
12
|
+
* @packageDocumentation
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```typescript
|
|
16
|
+
* import { GLOSTStreamProcessor } from "glost-processor";
|
|
17
|
+
*
|
|
18
|
+
* const processor = new GLOSTStreamProcessor()
|
|
19
|
+
* .use(transcription)
|
|
20
|
+
* .use(translation);
|
|
21
|
+
*
|
|
22
|
+
* for await (const chunk of processor.stream(document)) {
|
|
23
|
+
* console.log(chunk.sentences, chunk.isLast);
|
|
24
|
+
* }
|
|
25
|
+
* ```
|
|
26
|
+
*
|
|
27
|
+
* @since 0.7.0
|
|
28
|
+
*/
|
|
29
|
+
import { processGLOSTWithExtensionsAsync, processGLOSTChunkAsync, extensionRegistry, } from "glost-extensions";
|
|
30
|
+
// ============================================================================
|
|
31
|
+
// GLOSTStreamProcessor
|
|
32
|
+
// ============================================================================
|
|
33
|
+
/**
|
|
34
|
+
* Streaming processor for GLOST documents
|
|
35
|
+
*
|
|
36
|
+
* Mirrors the `GLOSTProcessor` API (`.use()`, `.freeze()`) but adds a
|
|
37
|
+
* `.stream()` method that returns an `AsyncGenerator<ProcessedChunk>`.
|
|
38
|
+
*
|
|
39
|
+
* @since 0.7.0
|
|
40
|
+
*/
|
|
41
|
+
export class GLOSTStreamProcessor {
|
|
42
|
+
plugins = [];
|
|
43
|
+
options = {};
|
|
44
|
+
frozen = false;
|
|
45
|
+
/**
|
|
46
|
+
* Create a new stream processor instance
|
|
47
|
+
*
|
|
48
|
+
* @param options - Initial processor options
|
|
49
|
+
*/
|
|
50
|
+
constructor(options = {}) {
|
|
51
|
+
this.options = { ...options };
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Add a plugin, preset, or extension to the pipeline
|
|
55
|
+
*
|
|
56
|
+
* @param spec - Plugin function, extension object, preset, or ID
|
|
57
|
+
* @param options - Plugin options
|
|
58
|
+
* @returns This processor for chaining
|
|
59
|
+
*/
|
|
60
|
+
use(spec, options) {
|
|
61
|
+
this.assertNotFrozen();
|
|
62
|
+
if (this.isPreset(spec)) {
|
|
63
|
+
return this.usePreset(spec);
|
|
64
|
+
}
|
|
65
|
+
this.plugins.push({ spec, options });
|
|
66
|
+
return this;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Freeze the processor
|
|
70
|
+
*
|
|
71
|
+
* Returns a frozen processor that cannot be modified. Useful for
|
|
72
|
+
* reusing the same pipeline configuration across multiple documents.
|
|
73
|
+
*
|
|
74
|
+
* @returns A frozen copy of this processor
|
|
75
|
+
*/
|
|
76
|
+
freeze() {
|
|
77
|
+
const frozen = new GLOSTStreamProcessor(this.options);
|
|
78
|
+
frozen.plugins = [...this.plugins];
|
|
79
|
+
frozen.frozen = true;
|
|
80
|
+
return frozen;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Stream a document as progressive sentence batches
|
|
84
|
+
*
|
|
85
|
+
* Processing phases:
|
|
86
|
+
* 1. All extensions with `streamingSupport !== 'chunk'` (i.e. `'none'`
|
|
87
|
+
* or `'full'`, or unset) run their `transform`, `visit`, and
|
|
88
|
+
* `enhanceMetadata` hooks on the **full** document.
|
|
89
|
+
* 2. The resulting document is split into sentence batches.
|
|
90
|
+
* 3. For each batch, extensions with `streamingSupport === 'chunk'`
|
|
91
|
+
* run their `visit` and `enhanceMetadata` hooks.
|
|
92
|
+
* 4. A `ProcessedChunk` is yielded.
|
|
93
|
+
*
|
|
94
|
+
* Cancellation: break out of the `for await` loop at any time. The
|
|
95
|
+
* generator will stop without processing remaining chunks.
|
|
96
|
+
*
|
|
97
|
+
* @param document - GLOST document to stream
|
|
98
|
+
* @param streamOptions - Streaming options (batchSize etc.)
|
|
99
|
+
* @yields `ProcessedChunk` objects in document order
|
|
100
|
+
*
|
|
101
|
+
* @example
|
|
102
|
+
* ```typescript
|
|
103
|
+
* for await (const chunk of processor.stream(doc, { batchSize: 20 })) {
|
|
104
|
+
* console.log(`para ${chunk.paragraphIndex} chunk ${chunk.chunkIndex}`);
|
|
105
|
+
* if (chunk.isLast) console.log("done");
|
|
106
|
+
* }
|
|
107
|
+
* ```
|
|
108
|
+
*/
|
|
109
|
+
async *stream(document, streamOptions) {
|
|
110
|
+
const batchSize = streamOptions?.batchSize ?? 50;
|
|
111
|
+
const extensions = await this.resolveExtensions();
|
|
112
|
+
// Split extensions into doc-level and chunk-level
|
|
113
|
+
const docExtensions = extensions.filter((e) => e.streamingSupport !== "chunk");
|
|
114
|
+
const chunkExtensions = extensions.filter((e) => e.streamingSupport === "chunk");
|
|
115
|
+
// Phase 1: run doc-level transforms on the full document
|
|
116
|
+
let processedDoc = document;
|
|
117
|
+
if (docExtensions.length > 0) {
|
|
118
|
+
const { data: _data, ...extOptions } = this.options;
|
|
119
|
+
const result = await processGLOSTWithExtensionsAsync(processedDoc, docExtensions, extOptions);
|
|
120
|
+
processedDoc = result.document;
|
|
121
|
+
}
|
|
122
|
+
// Phase 2: collect all sentences grouped by paragraph index
|
|
123
|
+
const paragraphSentences = collectSentencesByParagraph(processedDoc);
|
|
124
|
+
if (paragraphSentences.length === 0) {
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
const allChunks = [];
|
|
128
|
+
for (let pIdx = 0; pIdx < paragraphSentences.length; pIdx++) {
|
|
129
|
+
const sentences = paragraphSentences[pIdx];
|
|
130
|
+
let chunkIndex = 0;
|
|
131
|
+
for (let offset = 0; offset < sentences.length; offset += batchSize) {
|
|
132
|
+
allChunks.push({
|
|
133
|
+
paragraphIndex: pIdx,
|
|
134
|
+
chunkIndex,
|
|
135
|
+
sentences: sentences.slice(offset, offset + batchSize),
|
|
136
|
+
});
|
|
137
|
+
chunkIndex++;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
const totalChunks = allChunks.length;
|
|
141
|
+
// Phase 3: yield each chunk, optionally running chunk-level extensions
|
|
142
|
+
for (let i = 0; i < totalChunks; i++) {
|
|
143
|
+
const descriptor = allChunks[i];
|
|
144
|
+
let processedSentences = descriptor.sentences;
|
|
145
|
+
if (chunkExtensions.length > 0) {
|
|
146
|
+
const { data: _data, ...extOptions } = this.options;
|
|
147
|
+
processedSentences = await processGLOSTChunkAsync(processedSentences, chunkExtensions, extOptions);
|
|
148
|
+
}
|
|
149
|
+
yield {
|
|
150
|
+
sentences: processedSentences,
|
|
151
|
+
paragraphIndex: descriptor.paragraphIndex,
|
|
152
|
+
chunkIndex: descriptor.chunkIndex,
|
|
153
|
+
isLast: i === totalChunks - 1,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
// =====================================================================
|
|
158
|
+
// Private helpers
|
|
159
|
+
// =====================================================================
|
|
160
|
+
usePreset(preset) {
|
|
161
|
+
for (const entry of preset.plugins) {
|
|
162
|
+
if (Array.isArray(entry)) {
|
|
163
|
+
const [plugin, opts] = entry;
|
|
164
|
+
this.use(plugin, opts);
|
|
165
|
+
}
|
|
166
|
+
else {
|
|
167
|
+
this.use(entry);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return this;
|
|
171
|
+
}
|
|
172
|
+
async resolveExtensions() {
|
|
173
|
+
const result = [];
|
|
174
|
+
for (const { spec, options } of this.plugins) {
|
|
175
|
+
const ext = await this.resolvePlugin(spec, options);
|
|
176
|
+
if (ext) {
|
|
177
|
+
result.push(ext);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
return result;
|
|
181
|
+
}
|
|
182
|
+
async resolvePlugin(spec, options) {
|
|
183
|
+
if (typeof spec === "string") {
|
|
184
|
+
const ext = extensionRegistry.get(spec);
|
|
185
|
+
if (!ext) {
|
|
186
|
+
throw new Error(`Plugin "${spec}" not found in registry`);
|
|
187
|
+
}
|
|
188
|
+
return ext;
|
|
189
|
+
}
|
|
190
|
+
if (typeof spec === "function") {
|
|
191
|
+
const result = spec(options);
|
|
192
|
+
return result ?? null;
|
|
193
|
+
}
|
|
194
|
+
return spec;
|
|
195
|
+
}
|
|
196
|
+
isPreset(spec) {
|
|
197
|
+
return (spec !== null &&
|
|
198
|
+
typeof spec === "object" &&
|
|
199
|
+
"plugins" in spec &&
|
|
200
|
+
Array.isArray(spec.plugins));
|
|
201
|
+
}
|
|
202
|
+
assertNotFrozen() {
|
|
203
|
+
if (this.frozen) {
|
|
204
|
+
throw new Error("Cannot modify frozen stream processor");
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
// ============================================================================
|
|
209
|
+
// Internal helpers
|
|
210
|
+
// ============================================================================
|
|
211
|
+
/**
|
|
212
|
+
* Collect all sentences from a GLOSTRoot, grouped by paragraph index.
|
|
213
|
+
*
|
|
214
|
+
* Only `SentenceNode` children of `ParagraphNode` children are
|
|
215
|
+
* collected. Sentences that appear directly under the root (without a
|
|
216
|
+
* wrapping paragraph) are collected as a single synthetic group at
|
|
217
|
+
* index 0.
|
|
218
|
+
*
|
|
219
|
+
* @internal
|
|
220
|
+
*/
|
|
221
|
+
function collectSentencesByParagraph(document) {
|
|
222
|
+
const groups = [];
|
|
223
|
+
// Sentences that sit directly under the root (no paragraph wrapper)
|
|
224
|
+
const rootSentences = [];
|
|
225
|
+
for (const child of document.children) {
|
|
226
|
+
if (child.type === "ParagraphNode" && "children" in child) {
|
|
227
|
+
const para = child;
|
|
228
|
+
const sentences = para.children.filter((c) => typeof c === "object" &&
|
|
229
|
+
c !== null &&
|
|
230
|
+
c.type === "SentenceNode");
|
|
231
|
+
if (sentences.length > 0) {
|
|
232
|
+
groups.push(sentences);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
else if (child.type === "SentenceNode") {
|
|
236
|
+
rootSentences.push(child);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
// Prepend root-level sentences as paragraph 0 (if any)
|
|
240
|
+
if (rootSentences.length > 0) {
|
|
241
|
+
groups.unshift(rootSentences);
|
|
242
|
+
}
|
|
243
|
+
return groups;
|
|
244
|
+
}
|
|
245
|
+
//# sourceMappingURL=stream-processor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"stream-processor.js","sourceRoot":"","sources":["stream-processor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAIH,OAAO,EACL,+BAA+B,EAC/B,sBAAsB,EACtB,iBAAiB,GAClB,MAAM,kBAAkB,CAAC;AAiD1B,+EAA+E;AAC/E,uBAAuB;AACvB,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,OAAO,oBAAoB;IACvB,OAAO,GAAmD,EAAE,CAAC;IAC7D,OAAO,GAAqB,EAAE,CAAC;IAC/B,MAAM,GAAG,KAAK,CAAC;IAEvB;;;;OAIG;IACH,YAAY,UAA4B,EAAE;QACxC,IAAI,CAAC,OAAO,GAAG,EAAE,GAAG,OAAO,EAAE,CAAC;IAChC,CAAC;IAED;;;;;;OAMG;IACH,GAAG,CAAC,IAAyB,EAAE,OAAiB;QAC9C,IAAI,CAAC,eAAe,EAAE,CAAC;QAEvB,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;;OAOG;IACH,MAAM;QACJ,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtD,MAAM,CAAC,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,MAAyC,CAAC,MAAM,GAAG,IAAI,CAAC;QACzD,OAAO,MAA0C,CAAC;IACpD,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;IACH,KAAK,CAAC,CAAC,MAAM,CACX,QAAmB,EACnB,aAA6B;QAE7B,MAAM,SAAS,GAAG,aAAa,EAAE,SAAS,IAAI,EAAE,CAAC;QACjD,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAElD,kDAAkD;QAClD,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,OAAO,CACtC,CAAC;QACF,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,CACvC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,OAAO,CACtC,CAAC;QAEF,yDAAyD;QACzD,IAAI,YAAY,GAAG,QAAQ,CAAC;QAC5B,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAE7B,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAClC,IAAI,CAAC,OAA0B,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,+BAA+B,CAClD,YAAY,EACZ,aAAa,EACb,UAAU,CACX,CAAC;YACF,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC;QACjC,CAAC;QAED,4DAA4D;QAC5D,MAAM,kBAAkB,GAAG,2BAA2B,CAAC,YAAY,CAAC,CAAC;QAErE,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,OAAO;QACT,CAAC;QAUD,MAAM,SAAS,GAAsB,EAAE,CAAC;QAExC,KACE,IAAI,IAAI,GAAG,CAAC,EACZ,IAAI,GAAG,kBAAkB,CAAC,MAAM,EAChC,IAAI,EAAE,EACN,CAAC;YACD,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAE,CAAC;YAC5C,IAAI,UAAU,GAAG,CAAC,CAAC;YAEnB,KACE,IAAI,MAAM,GAAG,CAAC,EACd,MAAM,GAAG,SAAS,CAAC,MAAM,EACzB,MAAM,IAAI,SAAS,EACnB,CAAC;gBACD,SAAS,CAAC,IAAI,CAAC;oBACb,cAAc,EAAE,IAAI;oBACpB,UAAU;oBACV,SAAS,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;iBACvD,CAAC,CAAC;gBACH,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC;QAErC,uEAAuE;QACvE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAE,CAAC;YAEjC,IAAI,kBAAkB,GAAG,UAAU,CAAC,SAAS,CAAC;YAE9C,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAE/B,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,UAAU,EAAE,GAClC,IAAI,CAAC,OAA0B,CAAC;gBAClC,kBAAkB,GAAG,MAAM,sBAAsB,CAC/C,kBAAkB,EAClB,eAAe,EACf,UAAU,CACX,CAAC;YACJ,CAAC;YAED,MAAM;gBACJ,SAAS,EAAE,kBAAkB;gBAC7B,cAAc,EAAE,UAAU,CAAC,cAAc;gBACzC,UAAU,EAAE,UAAU,CAAC,UAAU;gBACjC,MAAM,EAAE,CAAC,KAAK,WAAW,GAAG,CAAC;aAC9B,CAAC;QACJ,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,kBAAkB;IAClB,wEAAwE;IAEhE,SAAS,CAAC,MAAc;QAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,KAAK,CAAC;gBAC7B,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,MAAM,MAAM,GAAqB,EAAE,CAAC;QACpC,KAAK,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACpD,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,IAAgB,EAChB,OAAiB;QAEjB,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,MAAM,IAAI,KAAK,CAAC,WAAW,IAAI,yBAAyB,CAAC,CAAC;YAC5D,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;QAED,IAAI,OAAO,IAAI,KAAK,UAAU,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAI,IAAkD,CAChE,OAAO,CACR,CAAC;YACF,OAAO,MAAM,IAAI,IAAI,CAAC;QACxB,CAAC;QAED,OAAO,IAAsB,CAAC;IAChC,CAAC;IAEO,QAAQ,CAAC,IAAa;QAC5B,OAAO,CACL,IAAI,KAAK,IAAI;YACb,OAAO,IAAI,KAAK,QAAQ;YACxB,SAAS,IAAK,IAAe;YAC7B,KAAK,CAAC,OAAO,CAAE,IAAe,CAAC,OAAO,CAAC,CACxC,CAAC;IACJ,CAAC;IAEO,eAAe;QACrB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;CACF;AAgBD,+EAA+E;AAC/E,mBAAmB;AACnB,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,SAAS,2BAA2B,CAClC,QAAmB;IAEnB,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,oEAAoE;IACpE,MAAM,aAAa,GAAoB,EAAE,CAAC;IAE1C,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,IAAI,KAAK,eAAe,IAAI,UAAU,IAAI,KAAK,EAAE,CAAC;YAC1D,MAAM,IAAI,GAAG,KAA8C,CAAC;YAC5D,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CACpC,CAAC,CAAC,EAAsB,EAAE,CACxB,OAAO,CAAC,KAAK,QAAQ;gBACrB,CAAC,KAAK,IAAI;gBACT,CAAsB,CAAC,IAAI,KAAK,cAAc,CAClD,CAAC;YACF,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACzC,aAAa,CAAC,IAAI,CAAC,KAAiC,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/src/types.d.ts
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Processor Types
|
|
3
|
+
*
|
|
4
|
+
* Type definitions for the unified-style GLOST processor.
|
|
5
|
+
*
|
|
6
|
+
* @packageDocumentation
|
|
7
|
+
*/
|
|
8
|
+
import type { GLOSTRoot } from "glost-core";
|
|
9
|
+
import type { GLOSTExtension, ProcessorOptions as ExtensionProcessorOptions } from "glost-extensions";
|
|
10
|
+
/**
|
|
11
|
+
* Plugin function signature
|
|
12
|
+
*
|
|
13
|
+
* A plugin is a function that returns an extension or modifies the processor.
|
|
14
|
+
* Similar to remark/unified plugins.
|
|
15
|
+
*
|
|
16
|
+
* @example
|
|
17
|
+
* ```typescript
|
|
18
|
+
* const myPlugin: Plugin = (options) => {
|
|
19
|
+
* return {
|
|
20
|
+
* id: "my-plugin",
|
|
21
|
+
* name: "My Plugin",
|
|
22
|
+
* transform: (tree) => tree
|
|
23
|
+
* };
|
|
24
|
+
* };
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
export type Plugin<TOptions = any> = (options?: TOptions) => GLOSTExtension | void;
|
|
28
|
+
/**
|
|
29
|
+
* Plugin specification
|
|
30
|
+
*
|
|
31
|
+
* Can be a plugin function, extension object, or string ID.
|
|
32
|
+
*/
|
|
33
|
+
export type PluginSpec = Plugin | GLOSTExtension | string;
|
|
34
|
+
/**
|
|
35
|
+
* Preset definition
|
|
36
|
+
*
|
|
37
|
+
* A preset is a collection of plugins with their options.
|
|
38
|
+
* Similar to babel presets.
|
|
39
|
+
*
|
|
40
|
+
* @example
|
|
41
|
+
* ```typescript
|
|
42
|
+
* const preset: Preset = {
|
|
43
|
+
* id: "language-learning",
|
|
44
|
+
* name: "Language Learning",
|
|
45
|
+
* description: "Full language learning stack",
|
|
46
|
+
* plugins: [
|
|
47
|
+
* ["transcription", { scheme: "ipa" }],
|
|
48
|
+
* ["translation", { target: "en" }],
|
|
49
|
+
* ["frequency"],
|
|
50
|
+
* ]
|
|
51
|
+
* };
|
|
52
|
+
* ```
|
|
53
|
+
*/
|
|
54
|
+
export interface Preset {
|
|
55
|
+
/** Unique preset identifier */
|
|
56
|
+
id: string;
|
|
57
|
+
/** Human-readable name */
|
|
58
|
+
name: string;
|
|
59
|
+
/** Optional description */
|
|
60
|
+
description?: string;
|
|
61
|
+
/** Plugins to apply with their options */
|
|
62
|
+
plugins: Array<PluginSpec | [PluginSpec, any]>;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Processing hook types
|
|
66
|
+
*/
|
|
67
|
+
export type BeforeHook = (document: GLOSTRoot, pluginId: string) => void | Promise<void>;
|
|
68
|
+
export type AfterHook = (document: GLOSTRoot, pluginId: string) => void | Promise<void>;
|
|
69
|
+
export type ErrorHook = (error: Error, pluginId: string) => void;
|
|
70
|
+
export type SkipHook = (pluginId: string, reason: string) => void;
|
|
71
|
+
export type ProgressHook = (stats: ProgressStats) => void;
|
|
72
|
+
/**
|
|
73
|
+
* Progress statistics
|
|
74
|
+
*/
|
|
75
|
+
export interface ProgressStats {
|
|
76
|
+
/** Total number of plugins */
|
|
77
|
+
total: number;
|
|
78
|
+
/** Number of plugins completed */
|
|
79
|
+
completed: number;
|
|
80
|
+
/** Current plugin being processed */
|
|
81
|
+
current?: string;
|
|
82
|
+
/** Processing start time */
|
|
83
|
+
startTime: number;
|
|
84
|
+
/** Elapsed time in ms */
|
|
85
|
+
elapsed: number;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Processing hooks
|
|
89
|
+
*/
|
|
90
|
+
export interface ProcessorHooks {
|
|
91
|
+
before: Map<string, BeforeHook[]>;
|
|
92
|
+
after: Map<string, AfterHook[]>;
|
|
93
|
+
onError: ErrorHook[];
|
|
94
|
+
onSkip: SkipHook[];
|
|
95
|
+
onProgress: ProgressHook[];
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Processor options
|
|
99
|
+
*/
|
|
100
|
+
export interface ProcessorOptions extends ExtensionProcessorOptions {
|
|
101
|
+
/** Data storage for sharing state between plugins */
|
|
102
|
+
data?: Map<string, any>;
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Processing result with detailed metadata
|
|
106
|
+
*/
|
|
107
|
+
export interface ProcessingResult {
|
|
108
|
+
/** The processed document */
|
|
109
|
+
document: GLOSTRoot;
|
|
110
|
+
/** Processing metadata */
|
|
111
|
+
metadata: {
|
|
112
|
+
/** Plugins that were applied */
|
|
113
|
+
appliedPlugins: string[];
|
|
114
|
+
/** Plugins that were skipped */
|
|
115
|
+
skippedPlugins: string[];
|
|
116
|
+
/** Processing errors */
|
|
117
|
+
errors: ProcessingError[];
|
|
118
|
+
/** Processing warnings */
|
|
119
|
+
warnings: ProcessingWarning[];
|
|
120
|
+
/** Processing statistics */
|
|
121
|
+
stats: ProcessingStats;
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Processing error details
|
|
126
|
+
*/
|
|
127
|
+
export interface ProcessingError {
|
|
128
|
+
/** Plugin that caused the error */
|
|
129
|
+
plugin: string;
|
|
130
|
+
/** Processing phase */
|
|
131
|
+
phase: "transform" | "visit" | "enhance";
|
|
132
|
+
/** Error message */
|
|
133
|
+
message: string;
|
|
134
|
+
/** Error stack trace */
|
|
135
|
+
stack?: string;
|
|
136
|
+
/** Whether the error is recoverable */
|
|
137
|
+
recoverable: boolean;
|
|
138
|
+
/** Original error object */
|
|
139
|
+
error: Error;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Processing warning details
|
|
143
|
+
*/
|
|
144
|
+
export interface ProcessingWarning {
|
|
145
|
+
/** Plugin that issued the warning */
|
|
146
|
+
plugin: string;
|
|
147
|
+
/** Warning message */
|
|
148
|
+
message: string;
|
|
149
|
+
/** Warning severity */
|
|
150
|
+
severity: "low" | "medium" | "high";
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Processing statistics
|
|
154
|
+
*/
|
|
155
|
+
export interface ProcessingStats {
|
|
156
|
+
/** Total processing time in ms */
|
|
157
|
+
totalTime: number;
|
|
158
|
+
/** Time per plugin in ms */
|
|
159
|
+
timing: Map<string, number>;
|
|
160
|
+
/** Total nodes processed */
|
|
161
|
+
nodesProcessed: number;
|
|
162
|
+
/** Processing start timestamp */
|
|
163
|
+
startTime: number;
|
|
164
|
+
/** Processing end timestamp */
|
|
165
|
+
endTime: number;
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,gBAAgB,IAAI,yBAAyB,EAAE,MAAM,kBAAkB,CAAC;AAEtG;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,MAAM,MAAM,CAAC,QAAQ,GAAG,GAAG,IAAI,CACnC,OAAO,CAAC,EAAE,QAAQ,KACf,cAAc,GAAG,IAAI,CAAC;AAE3B;;;;GAIG;AACH,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,cAAc,GAAG,MAAM,CAAC;AAE1D;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,WAAW,MAAM;IACrB,+BAA+B;IAC/B,EAAE,EAAE,MAAM,CAAC;IAEX,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAC;IAEb,2BAA2B;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,0CAA0C;IAC1C,OAAO,EAAE,KAAK,CAAC,UAAU,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;CAChD;AAED;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACzF,MAAM,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AACxF,MAAM,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;AACjE,MAAM,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;AAClE,MAAM,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,8BAA8B;IAC9B,KAAK,EAAE,MAAM,CAAC;IAEd,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAElB,qCAAqC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAC;IAElB,yBAAyB;IACzB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAClC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IAChC,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,UAAU,EAAE,YAAY,EAAE,CAAC;CAC5B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAiB,SAAQ,yBAAyB;IACjE,qDAAqD;IACrD,IAAI,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,6BAA6B;IAC7B,QAAQ,EAAE,SAAS,CAAC;IAEpB,0BAA0B;IAC1B,QAAQ,EAAE;QACR,gCAAgC;QAChC,cAAc,EAAE,MAAM,EAAE,CAAC;QAEzB,gCAAgC;QAChC,cAAc,EAAE,MAAM,EAAE,CAAC;QAEzB,wBAAwB;QACxB,MAAM,EAAE,eAAe,EAAE,CAAC;QAE1B,0BAA0B;QAC1B,QAAQ,EAAE,iBAAiB,EAAE,CAAC;QAE9B,4BAA4B;QAC5B,KAAK,EAAE,eAAe,CAAC;KACxB,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,mCAAmC;IACnC,MAAM,EAAE,MAAM,CAAC;IAEf,uBAAuB;IACvB,KAAK,EAAE,WAAW,GAAG,OAAO,GAAG,SAAS,CAAC;IAEzC,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAEhB,wBAAwB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,uCAAuC;IACvC,WAAW,EAAE,OAAO,CAAC;IAErB,4BAA4B;IAC5B,KAAK,EAAE,KAAK,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IAEf,sBAAsB;IACtB,OAAO,EAAE,MAAM,CAAC;IAEhB,uBAAuB;IACvB,QAAQ,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;CACrC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAElB,4BAA4B;IAC5B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5B,4BAA4B;IAC5B,cAAc,EAAE,MAAM,CAAC;IAEvB,iCAAiC;IACjC,SAAS,EAAE,MAAM,CAAC;IAElB,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;CACjB"}
|
package/src/types.js
ADDED
package/src/types.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["types.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG"}
|