@kaeawc/auto-mobile 0.0.87 → 0.0.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,6 +5,7 @@
5
5
  [![DeepScan grade](https://deepscan.io/api/teams/5165/projects/8526/branches/103329/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=5165&pid=8526&bid=103329)
6
6
  [![Known Vulnerabilities](https://snyk.io/test/github/Borewit/strtok3/badge.svg?targetFile=package.json)](https://snyk.io/test/github/Borewit/strtok3?targetFile=package.json)
7
7
  [![Codacy Badge](https://api.codacy.com/project/badge/Grade/59dd6795e61949fb97066ca52e6097ef)](https://www.codacy.com/app/Borewit/strtok3?utm_source=github.com&utm_medium=referral&utm_content=Borewit/strtok3&utm_campaign=Badge_Grade)
8
+ [![Codecov](https://codecov.io/gh/Borewit/strtok3/branch/master/graph/badge.svg)](https://codecov.io/gh/Borewit/strtok3)
8
9
  # strtok3
9
10
 
10
11
  A promise-based streaming [*tokenizer*](#tokenizer-object) for [Node.js](https://nodejs.org) and browsers.
@@ -45,7 +46,7 @@ For TypeScript CommonJS backward compatibility, you can use [load-esm](https://g
45
46
  Use Node.js 24 LTS to build the project and install development dependencies.
46
47
  This build requirement is separate from the Node.js ≥ 18 runtime requirement in `package.json`.
47
48
  CI builds once with Node.js 24, then runs the compiled JavaScript unit tests across the Node.js compatibility matrix and Bun 1.2.x, 1.3.x, and 1.4.x.
48
- Coverage is collected in a separate Node.js 24 job.
49
+ Coverage is collected in a separate Node.js 24 job and uploaded to Codecov.
49
50
 
50
51
  For local development, running the TypeScript tests with `tsx` requires Node.js 18.19+ (18.x) or Node.js ≥ 20.6.0.
51
52
 
@@ -65,15 +66,18 @@ If you find this project useful and would like to support its development, consi
65
66
  Use one of the methods to instantiate an [*abstract tokenizer*](#tokenizer-object):
66
67
  - [fromBlob](#fromblob-function)
67
68
  - [fromBuffer](#frombuffer-function)
68
- - [fromFile](#fromfile-function)*
69
- - [fromStream](#fromstream-function)*
69
+ - [fromFile](#fromfile-function)
70
+ - [fromStream](#fromstream-function)
70
71
  - [fromWebStream](#fromwebstream-function)
72
+ - [createTokenizerView](#createtokenizerview-function)
71
73
 
72
74
  > [!NOTE]
73
75
  > `fromFile` requires Node.js file-system APIs, and `fromStream` requires a Node.js readable stream.
74
76
  > In browsers, use `fromBlob`, `fromBuffer`, or `fromWebStream`.
75
77
 
76
78
  All methods return a [`Tokenizer`](#tokenizer-object), either directly or via a promise.
79
+ The `strtok3/core` entry point exports all factories except `fromFile` and can be used
80
+ without Node.js file-system APIs. In browsers, `strtok3` resolves to this core entry point.
77
81
 
78
82
  #### `fromBlob()` function
79
83
 
@@ -141,13 +145,13 @@ parse();
141
145
  Creates a [*tokenizer*](#tokenizer-object) from a local file.
142
146
 
143
147
  ```ts
144
- function fromFile(sourceFilePath: string, options?: ITokenizerOptions): Promise<FileTokenizer>
148
+ function fromFile(sourceFilePath: string, options?: IFromFileOptions): Promise<FileTokenizer>
145
149
  ```
146
150
 
147
151
  | Parameter | Type | Description |
148
152
  |----------------|----------|----------------------------|
149
153
  | sourceFilePath | `string` | Path to file to read from |
150
- | options | [ITokenizerOptions](#itokenizeroptions-interface) | Optional tokenizer options |
154
+ | options | [IFromFileOptions](#ifromfileoptions-interface) | Optional file tokenizer options |
151
155
 
152
156
  > [!NOTE]
153
157
  > - Only available for Node.js engines
@@ -248,6 +252,83 @@ async function parse() {
248
252
  parse();
249
253
  ```
250
254
 
255
+ #### `createTokenizerView()` function
256
+
257
+ Borrow a finite range of an existing tokenizer without buffering it or taking
258
+ ownership of its input. Available from both `strtok3` and `strtok3/core`.
259
+
260
+ ```ts
261
+ function createTokenizerView(source: ITokenizer, length: number): ITokenizerView
262
+ ```
263
+
264
+ The view starts at `source.position`, with local `position` zero and
265
+ `fileInfo.size` equal to `length`. Creation does not read or skip bytes. The size
266
+ is the declared range, so it can exceed the actual remaining input. The outer
267
+ file's MIME type, path, and URL are not copied to the view.
268
+
269
+ ```js
270
+ import { createTokenizerView, fromBuffer } from 'strtok3';
271
+ import { UINT8 } from 'token-types';
272
+
273
+ const source = fromBuffer(new Uint8Array([10, 20, 30, 40, 50]));
274
+ await source.ignore(1);
275
+ const chunk = createTokenizerView(source, 3); // source bytes [20, 30, 40]
276
+
277
+ console.log(await chunk.readToken(UINT8)); // 20; chunk.position = 1
278
+ const child = chunk.createView(1); // same as createTokenizerView(chunk, 1)
279
+ console.log(await child.readToken(UINT8)); // 30; chunk.position = 2
280
+ await child.close(); // source stays open, at position 3
281
+
282
+ console.log(chunk.remaining); // 1
283
+ await chunk.consumeRemainder(); // exactly consume 40, or throw on truncation
284
+ await chunk.close();
285
+ console.log(await source.readToken(UINT8)); // 50
286
+ await source.close();
287
+ ```
288
+
289
+ All read and peek positions, including explicit offsets and `setPosition()`, are
290
+ relative to the view's start. Nested views must fit inside their parent's
291
+ remaining range. A view exposes `setPosition()` and reports random access only
292
+ when its source supports it; a stream with a known size remains forward-only.
293
+ The overload for an `IRandomAccessTokenizer` returns an
294
+ `IRandomAccessTokenizerView`, including for nested `createView()` calls.
295
+
296
+ A request crossing the declared boundary rejects with `TokenizerBoundaryError`,
297
+ a `RangeError` with `position`, `length`, and `size` properties. This applies to
298
+ reads, peeks, tokens, and skips, even with `mayBeLess: true`. Invalid lengths,
299
+ offsets, and unsafe offset arithmetic reject before allocating token buffers or
300
+ consuming input. The source's token allocation and stream lookahead limits still
301
+ apply through nested views.
302
+
303
+ A request inside the view that encounters truncated underlying input rejects
304
+ with `EndOfStreamError`. Buffer reads and peeks with `mayBeLess: true` may return
305
+ fewer bytes within the requested range; token reads and peeks always require a
306
+ complete token. A failed exact read may have consumed part of the input; the
307
+ shared cursor reflects the bytes consumed. Peeks do not advance that cursor.
308
+
309
+ `ignore()` preserves the source's skip behavior, which may return a shorter
310
+ count or reject at EOF. Use `consumeRemainder()` when the entire declared
311
+ remainder must exist: it returns the number consumed, or rejects with
312
+ `EndOfStreamError`. On random-access sources it verifies the final byte and
313
+ advances the cursor without reading the whole remainder. On streams it drains
314
+ the remainder using the source's bounded skip buffers.
315
+
316
+ The source, parent, child, and sibling views share one cursor. Use them
317
+ sequentially, and close a child before resuming its parent. `close()` is
318
+ idempotent, does not drain the remainder, and never closes or aborts the source.
319
+ After a partially consumed child closes, the parent resumes at that same byte,
320
+ not at the child's declared end. Call `child.consumeRemainder()` before closing
321
+ when the parent should resume after the entire child range. Closed views reject
322
+ further operations and keep their last local position; closing a parent view
323
+ also prevents its children from being used.
324
+
325
+ Overlapping operations through a view or its parent/child chain are rejected;
326
+ `close()` also rejects while that view has an active operation. Direct operations
327
+ on the original source, or separate views created from it, cannot be coordinated
328
+ by a view, so callers must keep those sequential too. Explicit `abort()`
329
+ propagates to the shared source to cancel pending input and affects its other
330
+ users; it also closes the view.
331
+
251
332
  ### `Tokenizer` object
252
333
  The *tokenizer* is an abstraction of a [stream](https://nodejs.org/api/stream.html), file or [Uint8Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Uint8Array), allowing _reading_ or _peeking_ from the stream.
253
334
  It can also be translated in chunked reads, as done in [@tokenizer/http](https://github.com/Borewit/tokenizer-http);
@@ -264,9 +345,18 @@ It can also be translated in chunked reads, as done in [@tokenizer/http](https:/
264
345
  _Read_ methods advance the stream pointer, while _peek_ methods do not.
265
346
 
266
347
  There are two kinds of functions:
267
- 1. *read* methods: used to read a *token* or bytes into a buffer from the [*tokenizer*](#tokenizer-object). The position of the *tokenizer-stream* will advance with the number of bytes read.
348
+ 1. *read* methods: used to read a *token* or bytes into a buffer from the [*tokenizer*](#tokenizer-object). After a successful read, `tokenizer.position` is the starting position of the read plus the number of bytes read. An explicit position can skip ahead or, for random-access tokenizers, move backwards.
268
349
  2. *peek* methods: same as the read, but it will *not* advance the pointer. It allows to read (peek) ahead.
269
350
 
351
+ Reads and peeks require the full requested length unless `mayBeLess: true` is passed to
352
+ `readBuffer` or `peekBuffer`. Otherwise, insufficient input rejects with the exported
353
+ `EndOfStreamError`. Token methods always require the full token.
354
+
355
+ A failed exact stream read can consume partial input and advance `tokenizer.position`
356
+ by the number of bytes consumed. A failed exact stream peek leaves `tokenizer.position`
357
+ unchanged and retains the partial bytes for subsequent reads. Caller-provided buffers
358
+ may contain partial data even when the operation rejects.
359
+
270
360
  #### `readBuffer` function
271
361
 
272
362
  Read data from the _tokenizer_ into provided "buffer" (`Uint8Array`).
@@ -299,6 +389,12 @@ peekBuffer(uint8Array: Uint8Array, options?: IReadChunkOptions): Promise<number>
299
389
 
300
390
  Return value `Promise<number>` Promise with number of bytes read. The number of bytes read may be less if the `mayBeLess` flag was set.
301
391
 
392
+ For streams with a known size, a nonempty peek starting at or beyond that size
393
+ rejects with `EndOfStreamError`, or returns zero with `mayBeLess: true`, before
394
+ allocating lookahead or reading input. This also applies through tokenizer views.
395
+ Peeks starting inside the advertised input retain their partial-read behavior
396
+ when they cross EOF. Zero-length peeks remain no-ops.
397
+
302
398
  #### `readToken` function
303
399
 
304
400
  Read a *token* from the tokenizer-stream.
@@ -342,6 +438,8 @@ readNumber(token: IToken<number>): Promise<number>
342
438
  | token | [IToken<number>](#token-object) | Numeric token to read from the tokenizer-stream. |
343
439
 
344
440
  A promise resolving to a numeric value read and decoded from the *tokenizer-stream*.
441
+ Uses a reusable 8-byte buffer, so `token.len` must be at most 8 bytes.
442
+ For larger tokens or values such as `bigint`, use `readToken` instead.
345
443
 
346
444
  #### `peekNumber` function
347
445
 
@@ -353,10 +451,11 @@ peekNumber(token: IToken<number>): Promise<number>
353
451
 
354
452
  Returns a promise resolving to the decoded numeric value.
355
453
  Like `readNumber`, rejects with `EndOfStreamError` if the token cannot be read in full.
454
+ The same 8-byte token limit applies; use `peekToken` for larger tokens or values such as `bigint`.
356
455
 
357
456
  #### `ignore` function
358
457
 
359
- Advance the offset pointer with the token number of bytes provided.
458
+ Advance the offset pointer by the given number of bytes.
360
459
 
361
460
  ```ts
362
461
  ignore(length: number): Promise<number>
@@ -364,10 +463,14 @@ ignore(length: number): Promise<number>
364
463
 
365
464
  | Parameter | Type | Description |
366
465
  |------------|--------|------------------------------------------------------------------|
367
- | length | number | Non-negative number of bytes to ignore. Advances `tokenizer.position`. |
466
+ | length | number | Non-negative safe integer number of bytes to ignore. Advances `tokenizer.position`. |
368
467
 
369
468
  Returns a promise resolving to the number of bytes ignored.
370
469
  Buffer, blob, and file tokenizers stop at the known file size. Stream tokenizers reject with `EndOfStreamError` if the requested bytes are unavailable.
470
+ When a stream has a known size, skips beyond that size reject before allocating
471
+ a skip buffer or consuming input, including skips through tokenizer views.
472
+ If the size is unknown, or the stream ends before its advertised size, a failed
473
+ skip may consume available bytes before rejecting; `position` reflects those bytes.
371
474
 
372
475
  #### `close` function
373
476
  Clean up resources, such as closing a file pointer if applicable.
@@ -378,6 +481,17 @@ close(): Promise<void>
378
481
 
379
482
  Await this method when finished with a tokenizer. For Node.js streams, it does not destroy the underlying stream; the caller remains responsible for its lifecycle.
380
483
 
484
+ For buffer, blob, and file tokenizers, repeated or concurrent `close()` calls
485
+ share one cleanup operation and invoke `onClose` at most once. If cleanup or the
486
+ callback fails, subsequent calls reject with the same error. Calling `abort()`
487
+ before `close()` does not suppress cleanup or the callback.
488
+
489
+ Closing or aborting a Node.js stream tokenizer removes its reader-retaining
490
+ stream listeners. After the source ends, shared error and close guards may remain
491
+ until the source emits either event, including after tokenizer closure; these
492
+ guards do not retain the tokenizer. The caller remains responsible for handling
493
+ subsequent source errors.
494
+
381
495
  For Web streams, `close()` releases the reader lock. A BYOB reader leaves the source uncancelled, allowing another reader to continue reading it. A default reader cancels the source before releasing the lock, discarding remaining data. Bytes already read into tokenizer buffers, including peeked bytes, are not returned to the source.
382
496
 
383
497
  #### `abort` function
@@ -390,6 +504,10 @@ abort(): Promise<void>
390
504
 
391
505
  This is a no-op for buffer, blob, and file tokenizers.
392
506
 
507
+ For Node.js streams, pending reads reject with the exported `AbortError`, and buffered
508
+ peek data is discarded. The underlying stream is not destroyed. Calling `close()` or
509
+ aborting the configured `AbortSignal` has the same effect on pending Node.js stream reads.
510
+
393
511
  For Web streams, `abort()` releases a BYOB reader's lock without cancelling the source. A default reader cancels the source but keeps its lock until `close()` is called. An `AbortSignal` invokes the same abort behavior.
394
512
 
395
513
  #### `supportsRandomAccess` function
@@ -401,6 +519,25 @@ supportsRandomAccess(): boolean
401
519
  Returns `true` for buffer, blob, and file tokenizers, and `false` for stream tokenizers.
402
520
  When random access is supported, read and peek methods can use positions before `tokenizer.position`.
403
521
 
522
+ #### `isRandomAccessTokenizer` helper
523
+
524
+ Exported from both `strtok3` and `strtok3/core`:
525
+
526
+ ```ts
527
+ import { isRandomAccessTokenizer, type ITokenizer } from 'strtok3';
528
+
529
+ function skipToEnd(tokenizer: ITokenizer): void {
530
+ if (isRandomAccessTokenizer(tokenizer)) {
531
+ tokenizer.setPosition(tokenizer.fileInfo.size);
532
+ }
533
+ }
534
+ ```
535
+
536
+ Narrows `ITokenizer` to `IRandomAccessTokenizer` by checking `supportsRandomAccess()`,
537
+ a callable `setPosition`, and a numeric file size. A known size alone does not imply
538
+ random access. Existing third-party implementations can keep returning a boolean
539
+ from `supportsRandomAccess()` without changing their interface.
540
+
404
541
  #### `setPosition` function
405
542
 
406
543
  Available on buffer, blob, and file tokenizers (`IRandomAccessTokenizer`).
@@ -409,7 +546,10 @@ Available on buffer, blob, and file tokenizers (`IRandomAccessTokenizer`).
409
546
  setPosition(position: number): void
410
547
  ```
411
548
 
412
- Sets the current byte offset without reading data.
549
+ Sets the current byte offset without reading data. The position must be a
550
+ non-negative safe integer; invalid positions throw `RangeError` without changing
551
+ the cursor. Seeking past EOF is allowed. At or past EOF, `ignore()` returns zero
552
+ without moving the cursor backward, subject to safe end-position arithmetic.
413
553
 
414
554
  #### `Tokenizer` attributes
415
555
 
@@ -471,16 +611,51 @@ try {
471
611
  }
472
612
  ```
473
613
 
614
+ ### `IFromFileOptions` interface
615
+
616
+ Extends [ITokenizerOptions](#itokenizeroptions-interface) for `fromFile()`.
617
+
618
+ | Attribute | Type | Description |
619
+ |-----------|------|-------------|
620
+ | regularFileOnly | `boolean` | When `true`, opens with `O_RDONLY \| O_NONBLOCK` and checks that the opened handle refers to a regular file. Defaults to `false`, preserving existing behavior. |
621
+
622
+ Use this option when non-regular inputs, such as FIFOs, should be rejected without waiting for a writer. Validation uses the opened handle, so replacing the pathname before opening does not bypass the check. Symbolic links to regular files are accepted.
623
+
624
+ If an opened handle is not a regular file, `fromFile()` closes it and rejects with the exported `NotRegularFileError`. Its `path` property identifies the requested path. Opening errors, stat errors, and abort errors continue to propagate; any handle acquired by the factory is closed if setup fails. After success, the caller must close the tokenizer.
625
+
626
+ ```js
627
+ import { fromFile, NotRegularFileError } from 'strtok3';
628
+
629
+ try {
630
+ const tokenizer = await fromFile('somefile.bin', { regularFileOnly: true });
631
+ try {
632
+ // Parse the regular file.
633
+ } finally {
634
+ await tokenizer.close();
635
+ }
636
+ } catch (error) {
637
+ if (!(error instanceof NotRegularFileError)) {
638
+ throw error;
639
+ }
640
+ // Handle an input that is not a regular file.
641
+ }
642
+ ```
643
+
474
644
  ### `IReadChunkOptions` interface
475
645
 
476
646
  Each attribute is optional:
477
647
 
478
648
  | Attribute | Type | Description |
479
649
  |-----------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
480
- | length | number | Requested number of bytes to read or peek. Defaults to the target buffer length. |
481
- | position | number | Byte offset to read or peek from. Defaults to [tokenizer.position](#tokenizer-attributes) when omitted. Earlier positions require random access. |
650
+ | length | number | Requested number of bytes to read or peek. Must be a non-negative safe integer no larger than the target buffer length. Defaults to the target buffer length. |
651
+ | position | number | Byte offset to read or peek from. Must be a non-negative safe integer. Defaults to [tokenizer.position](#tokenizer-attributes) when omitted. Earlier positions require random access. |
482
652
  | mayBeLess | boolean | If `true`, allows fewer than the requested number of bytes to be read or peeked instead of throwing `EndOfStreamError`. Defaults to `false`. |
483
653
 
654
+ Invalid lengths or positions, including a `position + length` that exceeds the safe
655
+ integer range, reject with `RangeError`. Requesting an earlier position on a tokenizer
656
+ without random access rejects with `Error`. Data is written starting at index 0 of the
657
+ target buffer. To write at an offset, pass `buffer.subarray(offset)` as the target buffer.
658
+
484
659
  Example usage:
485
660
  ```js
486
661
  const bytesPeeked = await tokenizer.peekBuffer(buffer, {mayBeLess: true});
@@ -11,7 +11,7 @@ export class AbstractTokenizer {
11
11
  * @protected
12
12
  */
13
13
  constructor(options) {
14
- this.numBuffer = new Uint8Array(8);
14
+ this.numBuffer = new Uint8Array(10); // Includes the 80-bit floating-point tokens.
15
15
  /**
16
16
  * Tokenizer-stream position
17
17
  */
@@ -106,7 +106,7 @@ export class AbstractTokenizer {
106
106
  this.validateLength(length, 'ignore length');
107
107
  this.validateLength(this.position + length, 'end position');
108
108
  if (this.fileInfo.size !== undefined) {
109
- const bytesLeft = this.fileInfo.size - this.position;
109
+ const bytesLeft = Math.max(0, this.fileInfo.size - this.position);
110
110
  if (length > bytesLeft) {
111
111
  this.position += bytesLeft;
112
112
  return bytesLeft;
@@ -115,8 +115,16 @@ export class AbstractTokenizer {
115
115
  this.position += length;
116
116
  return length;
117
117
  }
118
- async close() {
119
- this.removeAbortListener();
118
+ close() {
119
+ if (!this.closePromise) {
120
+ this.removeAbortListener();
121
+ // Cache completion before invoking cleanup hooks or caller callbacks.
122
+ this.closePromise = Promise.resolve().then(() => this.closeOnce());
123
+ }
124
+ return this.closePromise;
125
+ }
126
+ /** Perform cleanup once, sharing completion and the first error across close calls. */
127
+ async closeOnce() {
120
128
  await this.abort();
121
129
  await this.onClose?.();
122
130
  }
@@ -51,6 +51,7 @@ export class BlobTokenizer extends AbstractTokenizer {
51
51
  return true;
52
52
  }
53
53
  setPosition(position) {
54
+ this.validateLength(position, 'position');
54
55
  this.position = position;
55
56
  }
56
57
  }
@@ -31,7 +31,7 @@ export class BufferTokenizer extends AbstractTokenizer {
31
31
  */
32
32
  async peekBuffer(uint8Array, options) {
33
33
  const normOptions = this.normalizeOptions(uint8Array, options);
34
- const bytes2read = Math.min(this.uint8Array.length - normOptions.position, normOptions.length);
34
+ const bytes2read = Math.max(0, Math.min(this.uint8Array.length - normOptions.position, normOptions.length));
35
35
  if (!normOptions.mayBeLess && bytes2read < normOptions.length) {
36
36
  throw new EndOfStreamError();
37
37
  }
@@ -45,6 +45,7 @@ export class BufferTokenizer extends AbstractTokenizer {
45
45
  return true;
46
46
  }
47
47
  setPosition(position) {
48
+ this.validateLength(position, 'position');
48
49
  this.position = position;
49
50
  }
50
51
  }
@@ -1,7 +1,9 @@
1
- import { open as fsOpen } from 'node:fs/promises';
1
+ import { constants } from 'node:fs';
2
+ import fs, {} from 'node:fs/promises';
2
3
  import { throwIfAborted } from './AbortSignal.js';
3
4
  import { AbstractTokenizer } from './AbstractTokenizer.js';
4
5
  import { getMaxBufferSize } from './BufferSizeError.js';
6
+ import { NotRegularFileError } from './NotRegularFileError.js';
5
7
  import { EndOfStreamError } from './stream/index.js';
6
8
  export class FileTokenizer extends AbstractTokenizer {
7
9
  /**
@@ -12,9 +14,13 @@ export class FileTokenizer extends AbstractTokenizer {
12
14
  static async fromFile(sourceFilePath, options) {
13
15
  getMaxBufferSize(options);
14
16
  throwIfAborted(options?.abortSignal);
15
- const fileHandle = await fsOpen(sourceFilePath, 'r');
17
+ const flags = options?.regularFileOnly ? constants.O_RDONLY | constants.O_NONBLOCK : 'r';
18
+ const fileHandle = await fs.open(sourceFilePath, flags);
16
19
  try {
17
20
  const stat = await fileHandle.stat();
21
+ if (options?.regularFileOnly && !stat.isFile()) {
22
+ throw new NotRegularFileError(sourceFilePath);
23
+ }
18
24
  return new FileTokenizer(fileHandle, {
19
25
  ...options,
20
26
  fileInfo: { ...options?.fileInfo, path: sourceFilePath, size: stat.size }
@@ -63,12 +69,12 @@ export class FileTokenizer extends AbstractTokenizer {
63
69
  }
64
70
  return res.bytesRead;
65
71
  }
66
- async close() {
67
- this.removeAbortListener();
72
+ async closeOnce() {
68
73
  await this.fileHandle.close();
69
- return super.close();
74
+ await super.closeOnce();
70
75
  }
71
76
  setPosition(position) {
77
+ this.validateLength(position, 'position');
72
78
  this.position = position;
73
79
  }
74
80
  supportsRandomAccess() {
@@ -0,0 +1,8 @@
1
+ /** Thrown when regularFileOnly is enabled and the opened handle is not a regular file. */
2
+ export class NotRegularFileError extends Error {
3
+ constructor(path) {
4
+ super(`Not a regular file: ${path}`);
5
+ this.path = path;
6
+ this.name = 'NotRegularFileError';
7
+ }
8
+ }
@@ -32,9 +32,10 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
32
32
  return 0;
33
33
  }
34
34
  let bytesRead = 0;
35
+ // Account for partial input before reporting an exact-read failure.
35
36
  while (bytesRead < normOptions.length) {
36
37
  const chunkLength = Math.min(this.maxBufferSize, normOptions.length - bytesRead);
37
- const count = await this.streamReader.read(uint8Array.subarray(bytesRead, bytesRead + chunkLength), normOptions.mayBeLess);
38
+ const count = await this.streamReader.read(uint8Array.subarray(bytesRead, bytesRead + chunkLength), true);
38
39
  bytesRead += count;
39
40
  this.position += count;
40
41
  if (count < chunkLength) {
@@ -57,6 +58,13 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
57
58
  if (normOptions.length === 0) {
58
59
  return 0;
59
60
  }
61
+ // A known-empty range needs neither lookahead allocation nor source I/O.
62
+ if (this.fileInfo.size !== undefined && normOptions.position >= this.fileInfo.size) {
63
+ if (normOptions.mayBeLess) {
64
+ return 0;
65
+ }
66
+ throw new EndOfStreamError();
67
+ }
60
68
  const lookaheadSize = normOptions.length + normOptions.position - this.position;
61
69
  this.validateBufferSize(lookaheadSize);
62
70
  let bytesRead = 0;
@@ -75,7 +83,11 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
75
83
  }
76
84
  if (normOptions.length > 0) {
77
85
  try {
78
- bytesRead = await this.streamReader.peek(uint8Array.subarray(0, normOptions.length), normOptions.mayBeLess);
86
+ // The reader retains this buffer. Keep it independent of caller-owned memory,
87
+ // including partial bytes retained when an exact peek encounters truncation.
88
+ const peekBuffer = new Uint8Array(normOptions.length);
89
+ bytesRead = await this.streamReader.peek(peekBuffer, true);
90
+ uint8Array.set(peekBuffer.subarray(0, bytesRead));
79
91
  }
80
92
  catch (err) {
81
93
  if (options?.mayBeLess && err instanceof EndOfStreamError) {
@@ -98,6 +110,9 @@ export class ReadStreamTokenizer extends AbstractTokenizer {
98
110
  }
99
111
  this.validateLength(length, 'ignore length');
100
112
  this.validateLength(this.position + length, 'end position');
113
+ if (length > 0 && this.fileInfo.size !== undefined && length > this.fileInfo.size - this.position) {
114
+ throw new EndOfStreamError();
115
+ }
101
116
  const bufSize = Math.min(ignoreChunkSize, this.maxBufferSize, length);
102
117
  const buf = new Uint8Array(bufSize);
103
118
  let totBytesRead = 0;
@@ -0,0 +1,10 @@
1
+ /** A request exceeds a tokenizer view, rather than encountering truncated input. */
2
+ export class TokenizerBoundaryError extends RangeError {
3
+ constructor(position, length, size) {
4
+ super(`Range at position ${position} with length ${length} exceeds tokenizer view size ${size}`);
5
+ this.position = position;
6
+ this.length = length;
7
+ this.size = size;
8
+ this.name = 'TokenizerBoundaryError';
9
+ }
10
+ }