@jarenjs/core 0.75.0 → 0.83.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +4 -0
- package/README.md +15 -0
- package/dist/types/range/index.d.ts +81 -0
- package/dist/types/retry.d.ts +43 -0
- package/dist/types/schedule.d.ts +43 -0
- package/dist/types/search/config.d.ts +69 -0
- package/dist/types/search/index.d.ts +145 -0
- package/dist/types/search/vocabulary.d.ts +9 -0
- package/dist/types/virtual/index.d.ts +86 -0
- package/docs/SCHEDULING.md +62 -0
- package/docs/SEARCH.md +158 -0
- package/docs/VIRTUAL.md +40 -0
- package/package.json +22 -2
- package/src/range/index.js +125 -0
- package/src/retry.js +80 -0
- package/src/schedule.js +156 -0
- package/src/search/config.js +65 -0
- package/src/search/index.js +360 -0
- package/src/search/vocabulary.js +39 -0
- package/src/virtual/index.js +140 -0
package/docs/SEARCH.md
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# Resident lexical search
|
|
2
|
+
|
|
3
|
+
`@jarenjs/core/search` is an opt-in, dependency-free index. It compiles a
|
|
4
|
+
versioned definition, owns resident postings and field statistics, and returns
|
|
5
|
+
ranked string IDs. It imports no database, AI engine, worker or DOM code.
|
|
6
|
+
|
|
7
|
+
```js
|
|
8
|
+
import { compileLexical } from '@jarenjs/core/search';
|
|
9
|
+
const lexical = compileLexical({
|
|
10
|
+
version: 1, fields: ['title', 'sku'], prefix: true, fuzzy: 0.15,
|
|
11
|
+
combineWith: 'AND', normalization: 'application-decoded-text/1',
|
|
12
|
+
});
|
|
13
|
+
const index = lexical.create();
|
|
14
|
+
index.rebuild([{ id: 'tea', title: 'Green tea', sku: '00120' }],
|
|
15
|
+
{ generation: 1, sourceRevision: 'catalog-revision-1' });
|
|
16
|
+
const result = index.search('gren tea', { limit: 20 });
|
|
17
|
+
index.dispose();
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Definition and compatibility
|
|
21
|
+
|
|
22
|
+
The definition is closed: `version`, `fields`, `profile`, `prefix`, `fuzzy`,
|
|
23
|
+
`combineWith`, `boost`, `normalization` and `limits`. Fields are distinct names;
|
|
24
|
+
documents carry a nonempty string `id` and string fields. Missing/null fields
|
|
25
|
+
index as empty text. Numbers are refused rather than silently stripping leading
|
|
26
|
+
zeros. Applications decode HTML before indexing and give that policy a stable
|
|
27
|
+
`normalization` identity. Changing this identity invalidates snapshots.
|
|
28
|
+
|
|
29
|
+
The default profile, `minisearch-7.2.0-cold`, uses lowercase UTF-16 text,
|
|
30
|
+
Unicode punctuation/separator plus CR/LF token boundaries, preserved accents,
|
|
31
|
+
and BM25+ with k=1.2, b=0.7 and delta=0.5. Tabs and symbols follow the reference
|
|
32
|
+
tokenizer, not a different whitespace tokenizer. Field lengths count distinct
|
|
33
|
+
unprocessed tokens. Query token repetitions add their score repeatedly, while
|
|
34
|
+
the final quality multiplier counts distinct query terms. Empty/punctuation-only
|
|
35
|
+
queries return a complete empty result. A numeric identifier remains text, but
|
|
36
|
+
prefix/fuzzy matching can deliberately include nearby identifiers; exact identifier
|
|
37
|
+
lookup should use `prefix: false, fuzzy: 0` or the authoritative row lookup.
|
|
38
|
+
|
|
39
|
+
Fuzzy distance is `min(6, round(queryToken.length * fuzzy))`, measured by
|
|
40
|
+
Levenshtein distance over UTF-16 units. Prefix expansion wins a fuzzy overlap.
|
|
41
|
+
Exact, prefix and fuzzy weights are respectively 1, 0.375 and 0.45, with the
|
|
42
|
+
reference's length/distance attenuation. Field `boost` values are finite positive
|
|
43
|
+
numbers. Term combination is `AND` or `OR`.
|
|
44
|
+
|
|
45
|
+
Equal-score results in the cold profile preserve first matching posting order
|
|
46
|
+
under the last AND term (first union occurrence for OR). Vocabulary prefix
|
|
47
|
+
traversal is reverse depth-first and fuzzy traversal is forward depth-first,
|
|
48
|
+
both derived deterministically from authoritative document order. An update
|
|
49
|
+
reconstructs these orders from the current documents, so it agrees with a fresh
|
|
50
|
+
build; this is deliberately independent of a reference engine's deleted-posting
|
|
51
|
+
history. Snapshots preserve the same ties. The retained reference's JSON reload
|
|
52
|
+
can change ties: this divergence is published, never silently normalized.
|
|
53
|
+
The explicit `lexical-key/1` profile instead breaks score ties by string ID.
|
|
54
|
+
It is a declared ordering choice, not cold-profile parity.
|
|
55
|
+
|
|
56
|
+
## Results and completeness
|
|
57
|
+
|
|
58
|
+
`search(text, options)` returns `state`, `hits: [{id, score}]`, `total`,
|
|
59
|
+
`hasMore`, `continuation`, `generation`, `sourceRevision`, `identity` and `used`.
|
|
60
|
+
`complete` certifies that the complete membership was evaluated; `limit` bounds
|
|
61
|
+
the returned page, and `total` still counts every matched, filtered document.
|
|
62
|
+
`budget-exhausted`, `invalidated` and `error` return no hits and `total: null`.
|
|
63
|
+
They never look like complete empty searches.
|
|
64
|
+
|
|
65
|
+
Options include `limit`, smaller `credits: {work, expansions, candidates}`,
|
|
66
|
+
`sourceRevision`, `filter(hit)`, `compare(a,b)`, `onMatch(hit)`, `query` and
|
|
67
|
+
`after`. Filtering and `onMatch` run before sorting and slicing, so counts and
|
|
68
|
+
facets do not filter a truncated top-k. A custom comparator changes ordering
|
|
69
|
+
only. Callbacks are trusted synchronous application code; their own CPU,
|
|
70
|
+
allocations and side effects are outside engine credits. A callback that mutates
|
|
71
|
+
or disposes the index invalidates the pending result before publication.
|
|
72
|
+
|
|
73
|
+
A continuation identifies the final score and stable ID plus exact text,
|
|
74
|
+
request identity, configuration and source revision. Supply it as `after`.
|
|
75
|
+
Changed identities or missing cursor members invalidate the request. Every page
|
|
76
|
+
re-evaluates the complete bounded membership; this is resident pagination, not a
|
|
77
|
+
claim of indexed seek performance. Query/host layers supply `query` for the
|
|
78
|
+
filter, facet and ordering identity. Never reuse it with different callbacks.
|
|
79
|
+
|
|
80
|
+
## Updates and resource credits
|
|
81
|
+
|
|
82
|
+
`rebuild(rows, identity)`, `update({put, remove}, identity)` and `clear(identity)`
|
|
83
|
+
prepare before publishing. Failures leave the published generation intact.
|
|
84
|
+
Lower generations invalidate; an equal generation with different content refuses.
|
|
85
|
+
Equal content and source revision is a no-op: zero changes, unchanged generation,
|
|
86
|
+
no additional postings and no tombstones. A higher source revision can publish
|
|
87
|
+
unchanged indexed text, allowing a change in non-indexed authoritative fields to
|
|
88
|
+
invalidate query results. Explicit generations are nonnegative safe integers.
|
|
89
|
+
|
|
90
|
+
`SEARCH_LIMITS` declares finite document, source-byte, retained-index,
|
|
91
|
+
temporary-byte, token/posting, vocabulary, field/token/query, candidate,
|
|
92
|
+
expansion, output and work ceilings. `stats()` reports retained document,
|
|
93
|
+
posting, vocabulary, source and accounted-index quantities. Accounted bytes
|
|
94
|
+
are a conservative logical allocation model; they are not measurements of a
|
|
95
|
+
JavaScript VM's heap or its garbage collector. Temporary reservations include
|
|
96
|
+
the old published index, staged documents/postings, vocabulary construction and
|
|
97
|
+
snapshot decoding. Credits are checked before publication; source/token-sized
|
|
98
|
+
temporary allocations are reserved before tokenization. No deleted content is
|
|
99
|
+
retained for later vacuuming.
|
|
100
|
+
|
|
101
|
+
`rebuildAsync(rows, {yield, signal, onProgress, ...identity})` and
|
|
102
|
+
`updateAsync(changes, request)` use the same transactional preparation, yields in bounded work batches, and fences cancellation
|
|
103
|
+
and superseded requests before publication. A single document or posting reorder
|
|
104
|
+
that cannot fit one batch refuses. `yield` must yield to the host's event loop
|
|
105
|
+
when responsiveness is required; resolving a promise alone does not do that.
|
|
106
|
+
An injected worker host additionally enforces its process/VM memory ceiling.
|
|
107
|
+
One index's logical limits cannot bound another index or a caller-owned catalog.
|
|
108
|
+
|
|
109
|
+
## Snapshots and ownership
|
|
110
|
+
|
|
111
|
+
`snapshot()` produces a JSON string in `jaren-lexical/1`. It contains an explicit
|
|
112
|
+
complete marker, exact configuration identity, source revision, generation,
|
|
113
|
+
ordered derived field text and stable ordinals. A content checksum detects
|
|
114
|
+
accidental corruption; it does not authenticate untrusted storage.
|
|
115
|
+
`restore(string, {sourceRevision, generation?})` validates before atomic
|
|
116
|
+
publication. Corruption, incompatible configuration, stale source, partial
|
|
117
|
+
format and allocation limits return `rebuild-required` with a reason. Restore
|
|
118
|
+
reconstructs postings through the same engine; it is not a zero-cost mmap load.
|
|
119
|
+
An identical second restore reports zero changes.
|
|
120
|
+
|
|
121
|
+
The authoritative source owns revision validation. An index snapshot is a
|
|
122
|
+
discardable derived cache, never a second catalog. The db adapter publishes it
|
|
123
|
+
atomically, and the app search resource owns an injected worker and drains it
|
|
124
|
+
on disposal. `dispose()` is idempotent, clears resident references, rejects new
|
|
125
|
+
work and fences unfinished builds.
|
|
126
|
+
|
|
127
|
+
## Measured qualification
|
|
128
|
+
|
|
129
|
+
Run `npm run benchmark:lexical` for isolated retained-reference and native
|
|
130
|
+
measurements over both frozen synthetic consumers. The runner compares complete
|
|
131
|
+
membership, scores, cold ties and native reload results for every labelled query.
|
|
132
|
+
It reports reference reload divergence and slower native paths beside gains.
|
|
133
|
+
Heap/RSS qualification belongs to the named Node host and its enforced heap
|
|
134
|
+
ceiling; browser layout and real consumer relevance are separate evidence.
|
|
135
|
+
|
|
136
|
+
<!--fact:lexical.measurements-->
|
|
137
|
+
|
|
138
|
+
Measured on v24.19.0, linux/x64, AMD Ryzen 9 5900HX with Radeon Graphics.
|
|
139
|
+
|
|
140
|
+
| Consumer / engine | Rows | Cold / warm ms | Query p95 ms | One update ms | Snapshot gzip bytes | Sampled heap / RSS high-water MiB | V8 heap ceiling MiB |
|
|
141
|
+
|---|---:|---|---:|---:|---:|---|---:|
|
|
142
|
+
| catalog / reference | 10000 | 130.99 / 50.10 | 11.97 | 0.45 | 370591 | 95.97 / 190.46 | 240.00 |
|
|
143
|
+
| catalog / native | 10000 | 203.59 / 201.37 | 10.89 | 63.60 | 121054 | 104.08 / 208.89 | 240.00 |
|
|
144
|
+
| archive-stock / reference | 75000 | 1124.97 / 578.43 | 168.33 | 0.55 | 2820312 | 559.14 / 697.04 | 752.00 |
|
|
145
|
+
| archive-stock / native | 75000 | 1500.68 / 1775.38 | 98.43 | 681.98 | 920811 | 579.94 / 726.00 | 752.00 |
|
|
146
|
+
|
|
147
|
+
| Consumer | Native logical index MiB | Peak update accounted MiB | Native teardown ms / remaining handles | Membership / order / score / native reload differences | Reference reload tie changes |
|
|
148
|
+
|---|---:|---:|---|---|---:|
|
|
149
|
+
| catalog | 24.58 | 35.37 | 0.04 / 0 | 0 / 0 / 0 / 0 | 1002 |
|
|
150
|
+
| archive-stock | 186.22 | 267.03 | 0.03 / 0 | 0 / 0 / 0 / 0 | 6002 |
|
|
151
|
+
|
|
152
|
+
Browser gzip: native 5606 bytes; reference 5874 bytes.
|
|
153
|
+
|
|
154
|
+
Separate Node processes, identical source rows and queries, five query samples each; startup includes source generation, cold build, snapshot and warm restore. An earlier unconstrained native run exceeded the larger corpus heap and RSS ceilings; the explicit host heap settings are required for this qualification. RSS is the OS process high-water mark; heap is sampled after operations. peakHeapBytes is the V8 hard heap ceiling (a conservative bound, not an observed peak); both engines run with the same max-old-space-size derived from the frozen heap budget, reserving 64 MiB for young space and explicitly limiting each semi-space to 16 MiB. Unconstrained GC is not a bounded host. Reference indexBytes is serialized JSON; native indexBytes is conservative logical retained allocation, so those columns are not the same metric. Both retain a cold and a restored index during query qualification.
|
|
155
|
+
|
|
156
|
+
Native build, reload and update costs exceed the reference; source-bound snapshots compress better. Cold tie compatibility deliberately differs from reference reload ordering. These synthetic results qualify the named bounded host, not downstream relevance or universal latency.
|
|
157
|
+
|
|
158
|
+
<!--/fact-->
|
package/docs/VIRTUAL.md
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Virtual geometry
|
|
2
|
+
|
|
3
|
+
Import `fixedRange`, `createVirtualAxis`, `virtualIndices` and
|
|
4
|
+
`logicalScrollOffset` from `@jarenjs/core/virtual`. This opt-in entry has no DOM,
|
|
5
|
+
view, app, database or worker dependency.
|
|
6
|
+
|
|
7
|
+
`fixedRange({count, size, viewport, offset=0, overscan=0})` returns a half-open
|
|
8
|
+
`{start,end,offset,extent}`. It clamps offsets to the logical extent, does constant
|
|
9
|
+
work, and returns an empty range for empty or hidden viewports. Count and overscan
|
|
10
|
+
are nonnegative safe integers; size is finite and positive. Invalid geometry
|
|
11
|
+
throws `RangeError`. Nonfinite offsets and viewports normalize to zero.
|
|
12
|
+
|
|
13
|
+
For a fixed axis the mounted base range has at most
|
|
14
|
+
`ceil(viewport / size) + 1 + 2 * overscan` rows, clamped by count. The extra row
|
|
15
|
+
covers fractional alignment. `virtualIndices(range,pins,count,pinBudget)` merges
|
|
16
|
+
unique valid pins and returns `budget-exhausted / pin-credits` when additional
|
|
17
|
+
pins exceed their limit. It never silently truncates the requested viewport.
|
|
18
|
+
|
|
19
|
+
`createVirtualAxis({count,estimateSize,maxMeasurements=256,maxBytes=32768})`
|
|
20
|
+
adds sparse sizes through `measure(index,key,size)`. Keys are stable strings.
|
|
21
|
+
The cache evicts oldest measurements and estimates their sizes again. Sorted
|
|
22
|
+
sparse entries and prefix summaries cost O(M) memory, where M is the measurement
|
|
23
|
+
credit, and rebuild only when measurements or identity mappings change. Position
|
|
24
|
+
lookups search those summaries; measured range lookup searches logical positions
|
|
25
|
+
without allocating or enumerating count items. Fixed ranges take the constant
|
|
26
|
+
path when no measurement is retained. `stats()` reports retained measurements,
|
|
27
|
+
accounted bytes and summaries; actual JS allocation overhead is measured separately.
|
|
28
|
+
|
|
29
|
+
`update({count,estimateSize,indexOf})` remaps retained keys after insertion,
|
|
30
|
+
deletion or reordering. An estimate change invalidates previous measurements.
|
|
31
|
+
`anchor(offset,keyAt,query)` captures key plus intra-row offset and query identity.
|
|
32
|
+
`restore(anchor,indexOf,query)` re-resolves the key, or uses the clamped previous
|
|
33
|
+
logical position when it disappeared; a changed query starts at zero. The index
|
|
34
|
+
in this transient fallback is never an entity identifier. The collection retains
|
|
35
|
+
this anchor before the host mutates source order.
|
|
36
|
+
|
|
37
|
+
`position`, `size`, `indexAt`, `extent`, `range`, `clear` and idempotent `dispose`
|
|
38
|
+
complete the axis API. A disposed axis retains no measurements or summaries.
|
|
39
|
+
`logicalScrollOffset` normalizes LTR, negative RTL, reverse RTL and default RTL
|
|
40
|
+
coordinates at a DOM adapter boundary. Core does not choose a browser scroll ceiling.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@jarenjs/core",
|
|
3
3
|
"private": false,
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.83.3",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.js",
|
|
7
7
|
"types": "./dist/types/index.d.ts",
|
|
@@ -32,6 +32,10 @@
|
|
|
32
32
|
"core"
|
|
33
33
|
],
|
|
34
34
|
"exports": {
|
|
35
|
+
"./search": {
|
|
36
|
+
"types": "./dist/types/search/index.d.ts",
|
|
37
|
+
"default": "./src/search/index.js"
|
|
38
|
+
},
|
|
35
39
|
".": {
|
|
36
40
|
"types": "./dist/types/index.d.ts",
|
|
37
41
|
"default": "./src/index.js"
|
|
@@ -172,7 +176,23 @@
|
|
|
172
176
|
"types": "./dist/types/convert/*.d.ts",
|
|
173
177
|
"default": "./src/convert/*.js"
|
|
174
178
|
},
|
|
175
|
-
"./package.json": "./package.json"
|
|
179
|
+
"./package.json": "./package.json",
|
|
180
|
+
"./virtual": {
|
|
181
|
+
"types": "./dist/types/virtual/index.d.ts",
|
|
182
|
+
"default": "./src/virtual/index.js"
|
|
183
|
+
},
|
|
184
|
+
"./range": {
|
|
185
|
+
"types": "./dist/types/range/index.d.ts",
|
|
186
|
+
"default": "./src/range/index.js"
|
|
187
|
+
},
|
|
188
|
+
"./retry": {
|
|
189
|
+
"types": "./dist/types/retry.d.ts",
|
|
190
|
+
"default": "./src/retry.js"
|
|
191
|
+
},
|
|
192
|
+
"./schedule": {
|
|
193
|
+
"types": "./dist/types/schedule.d.ts",
|
|
194
|
+
"default": "./src/schedule.js"
|
|
195
|
+
}
|
|
176
196
|
},
|
|
177
197
|
"scripts": {
|
|
178
198
|
"build": "npm run build:types",
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** Shared structural range protocol and resident-array provider. */
|
|
3
|
+
import { isJsonValue } from '../object.js';
|
|
4
|
+
import { resolveRuntime } from '../runtime.js';
|
|
5
|
+
|
|
6
|
+
/** JSON wire byte cost. @param {any} value */
|
|
7
|
+
export const rangeBytes = (value) => new TextEncoder().encode(JSON.stringify(value)).byteLength;
|
|
8
|
+
/** Echo request identity. @param {any} request */
|
|
9
|
+
export const rangeIdentity = (request) => Object.fromEntries(['generation', 'requestId', 'query', 'snapshot'].map((key) => [key, request[key]]));
|
|
10
|
+
/** Validate finite request credits. @param {any} credits */
|
|
11
|
+
export const validRangeCredits = (credits) => ['pages', 'rows', 'bytes', 'work'].every((key) => Number.isSafeInteger(credits?.[key]) && credits[key] >= 0);
|
|
12
|
+
/** Validate a half-open logical range. @param {any} range */
|
|
13
|
+
export const validLogicalRange = (range) => range && Number.isSafeInteger(range.start) && Number.isSafeInteger(range.end) && range.start >= 0 && range.end >= range.start;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* A resident-array provider. The immutable source copy and key index are reported separately
|
|
17
|
+
* from requested pages; virtual DOM does not make this resident source bounded by page credits.
|
|
18
|
+
* @param {any[]} initialRows @param {any} [options]
|
|
19
|
+
*/
|
|
20
|
+
export function createArrayRangeProvider(initialRows, options = {}) {
|
|
21
|
+
const source = options.source ?? resolveRuntime(options.runtime).uuid();
|
|
22
|
+
const query = options.query ?? source, keyOf = options.keyOf ?? ((row) => String(row.id));
|
|
23
|
+
let snapshot = options.snapshot ?? `${source}-v1`, revision = 0, disposed = false, ticket = 0, generation = -1;
|
|
24
|
+
let rows = [], sizes = [], indices = new Map(), sourceBytes = 0;
|
|
25
|
+
const pending = new Set(), observers = new Set(), cursors = new Map();
|
|
26
|
+
const maxPages = options.maxPages ?? 4, maxRows = options.maxRows ?? 256, maxBytes = options.maxBytes ?? 262144;
|
|
27
|
+
const maxInFlight = options.maxInFlight ?? 2;
|
|
28
|
+
for (const limit of [maxPages, maxRows, maxBytes, maxInFlight])
|
|
29
|
+
if (!Number.isSafeInteger(limit) || limit <= 0) throw new RangeError('Invalid provider credits');
|
|
30
|
+
function load(input) {
|
|
31
|
+
if (!Array.isArray(input) || !isJsonValue(input)) throw new TypeError('Array provider rows must be JSON');
|
|
32
|
+
const copy = structuredClone(input), index = new Map();
|
|
33
|
+
copy.forEach((row, i) => { const key = keyOf(row);
|
|
34
|
+
if (typeof key !== 'string' || index.has(key)) throw new TypeError('Array keys must be unique strings');
|
|
35
|
+
index.set(key, i); });
|
|
36
|
+
// Returned rows are copies, so callers cannot mutate the complete source snapshot.
|
|
37
|
+
rows = copy; indices = index; sizes = rows.map(rangeBytes); sourceBytes = sizes.reduce((n, size) => n + size, 2 + Math.max(0, rows.length - 1));
|
|
38
|
+
}
|
|
39
|
+
load(initialRows);
|
|
40
|
+
const capabilities = Object.freeze({ seekIndex: options.seekIndex !== false, seekKey: true, continuation: true,
|
|
41
|
+
exactTotal: options.exactTotal !== false, live: true, completeExport: true });
|
|
42
|
+
const provider = {
|
|
43
|
+
query, get snapshot() { return snapshot; }, capabilities,
|
|
44
|
+
indexOf(key) { return indices.get(key) ?? -1; },
|
|
45
|
+
stats() { return { rows: rows.length, bytes: sourceBytes, pages: cursors.size, pending: pending.size, subscriptions: observers.size }; },
|
|
46
|
+
replace(next, identity) {
|
|
47
|
+
if (disposed) throw new Error('Provider disposed');
|
|
48
|
+
if (typeof identity !== 'string' || identity === snapshot) throw new TypeError('Replacement needs a new source snapshot');
|
|
49
|
+
load(next); snapshot = identity; cursors.clear(); ticket++;
|
|
50
|
+
const event = { type: 'reset', revision: ++revision, query, snapshot };
|
|
51
|
+
for (const fn of observers) { try { fn(event); } catch { /* Subscribers cannot suppress sibling invalidations. */ } }
|
|
52
|
+
},
|
|
53
|
+
subscribe(fn) {
|
|
54
|
+
if (disposed) throw new Error('Provider disposed');
|
|
55
|
+
if (observers.size >= 8) throw new RangeError('Subscription credits');
|
|
56
|
+
observers.add(fn); return () => observers.delete(fn);
|
|
57
|
+
},
|
|
58
|
+
request(input, signal) {
|
|
59
|
+
const request = structuredClone(input), identity = rangeIdentity(request), used = { pages: 0, rows: 0, bytes: 0, work: 0 };
|
|
60
|
+
const refuse = (state, reason) => ({ ...identity, state, reason, used });
|
|
61
|
+
if (disposed) return Promise.resolve(refuse('error', 'disposed'));
|
|
62
|
+
if (signal?.aborted) return Promise.resolve(refuse('error', 'cancelled'));
|
|
63
|
+
if (pending.size >= maxInFlight) return Promise.resolve(refuse('budget-exhausted', 'in-flight'));
|
|
64
|
+
if (!Number.isSafeInteger(request.generation) || request.generation < 0 || typeof request.requestId !== 'string')
|
|
65
|
+
return Promise.resolve(refuse('error', 'invalid-identity'));
|
|
66
|
+
if (request.generation < generation) return Promise.resolve(refuse('invalidated', 'query-changed'));
|
|
67
|
+
generation = request.generation;
|
|
68
|
+
const current = ++ticket;
|
|
69
|
+
const run = async () => {
|
|
70
|
+
await Promise.resolve();
|
|
71
|
+
if (disposed) return refuse('error', 'disposed');
|
|
72
|
+
if (signal?.aborted) return refuse('error', 'cancelled');
|
|
73
|
+
if (current !== ticket || request.query !== query || request.snapshot !== snapshot) return refuse('invalidated', 'source-changed');
|
|
74
|
+
if (!validRangeCredits(request.credits)) return refuse('error', 'invalid-credits');
|
|
75
|
+
let range = request.range;
|
|
76
|
+
if (request.continuation != null) {
|
|
77
|
+
range = cursors.get(request.continuation);
|
|
78
|
+
if (!range) return refuse('invalidated', 'continuation-changed');
|
|
79
|
+
}
|
|
80
|
+
if (!validLogicalRange(range)) return refuse('error', 'invalid-range');
|
|
81
|
+
const length = range.end - range.start;
|
|
82
|
+
if (!request.continuation && range.start > 0 && !capabilities.seekIndex) return refuse('error', 'unsupported-seek');
|
|
83
|
+
if (request.credits.pages < 1 || request.credits.rows < length || request.credits.work < length || request.credits.bytes < 2
|
|
84
|
+
|| length > maxRows) return refuse('budget-exhausted', 'credits');
|
|
85
|
+
const end = Math.min(range.end, rows.length), count = Math.max(0, end - range.start);
|
|
86
|
+
let bytes = 2 + Math.max(0, count - 1);
|
|
87
|
+
for (let i = range.start; i < end; i++) bytes += sizes[i];
|
|
88
|
+
used.work = count; used.pages = 1;
|
|
89
|
+
if (bytes > request.credits.bytes || bytes > maxBytes) return refuse('budget-exhausted', 'credits');
|
|
90
|
+
const items = structuredClone(rows.slice(range.start, end));
|
|
91
|
+
let continuation = null;
|
|
92
|
+
if (length && range.end < rows.length) {
|
|
93
|
+
continuation = `${snapshot}:${current}`;
|
|
94
|
+
while (cursors.size >= maxPages) cursors.delete(cursors.keys().next().value);
|
|
95
|
+
cursors.set(continuation, { start: range.end, end: range.end + length });
|
|
96
|
+
}
|
|
97
|
+
return { ...identity, state: 'ready', rows: items, keys: items.map(keyOf), continuation,
|
|
98
|
+
total: capabilities.exactTotal ? { kind: 'known', value: rows.length } : { kind: 'unknown' },
|
|
99
|
+
used: { pages: 1, rows: items.length, bytes, work: items.length } };
|
|
100
|
+
};
|
|
101
|
+
const promise = run().finally(() => pending.delete(promise)); pending.add(promise); return promise;
|
|
102
|
+
},
|
|
103
|
+
async *export(request, signal) {
|
|
104
|
+
const { pageRows = 64, pageBytes = maxBytes } = request;
|
|
105
|
+
if (!Number.isSafeInteger(pageRows) || pageRows <= 0 || pageRows > maxRows || !Number.isSafeInteger(pageBytes) || pageBytes < 2)
|
|
106
|
+
throw new RangeError('Invalid export credits');
|
|
107
|
+
const total = rows.length;
|
|
108
|
+
for (let start = 0; start < total; start += pageRows) {
|
|
109
|
+
await Promise.resolve();
|
|
110
|
+
if (disposed || signal?.aborted || request.query !== query || request.snapshot !== snapshot) throw new Error('Incomplete snapshot export');
|
|
111
|
+
const end = Math.min(total, start + pageRows);
|
|
112
|
+
let bytes = 2 + Math.max(0, end - start - 1);
|
|
113
|
+
for (let i = start; i < end; i++) bytes += sizes[i];
|
|
114
|
+
if (bytes > Math.min(maxBytes, pageBytes)) throw new RangeError('Export byte credits');
|
|
115
|
+
const items = structuredClone(rows.slice(start, end));
|
|
116
|
+
yield { state: 'ready', rows: items, keys: items.map(keyOf), query, snapshot };
|
|
117
|
+
}
|
|
118
|
+
if (disposed || signal?.aborted || request.query !== query || request.snapshot !== snapshot) throw new Error('Incomplete snapshot export');
|
|
119
|
+
yield { state: 'complete', total, query, snapshot };
|
|
120
|
+
},
|
|
121
|
+
async dispose() { disposed = true; ticket++; observers.clear(); await Promise.allSettled([...pending]);
|
|
122
|
+
cursors.clear(); rows = []; sizes = []; indices.clear(); sourceBytes = 0; },
|
|
123
|
+
};
|
|
124
|
+
return provider;
|
|
125
|
+
}
|
package/src/retry.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** Shared backoff arithmetic, abortable waits and an explicit dispatch budget. */
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Calculate a delay; strict never shortens the server's minimum wait.
|
|
6
|
+
* Compatibility policies retain the published clients' distinct jitter shapes.
|
|
7
|
+
* @param {{ policy?: 'strict' | 'ai-compat' | 'contract-compat', baseMs?: number, maxMs?: number, random?: () => number }} options
|
|
8
|
+
* @param {number} attempt - failed attempt, counted from one
|
|
9
|
+
* @param {number} [retryAfterMs]
|
|
10
|
+
* @returns {number}
|
|
11
|
+
*/
|
|
12
|
+
export function backoffDelay(options, attempt, retryAfterMs = undefined) {
|
|
13
|
+
const { policy = 'strict', baseMs = 500, maxMs = 8000, random = Math.random } = options;
|
|
14
|
+
const cap = Math.min(maxMs, baseMs * 2 ** (attempt - 1));
|
|
15
|
+
if (retryAfterMs !== undefined)
|
|
16
|
+
return policy === 'ai-compat' ? Math.min(maxMs, retryAfterMs) : Math.max(0, retryAfterMs);
|
|
17
|
+
if (policy === 'contract-compat') return cap + Math.floor(random() * 250);
|
|
18
|
+
return cap * (policy === 'ai-compat' ? 0.5 + 0.5 * random() : random());
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Parse only the selected header dialect. HTTP accepts seconds or HTTP-date;
|
|
23
|
+
* milliseconds accepts nonnegative numbers only; none ignores the header.
|
|
24
|
+
* @param {string | null | undefined} raw
|
|
25
|
+
* @param {{ dialect?: 'http' | 'milliseconds' | 'none', now?: number }} [options]
|
|
26
|
+
* @returns {number | undefined}
|
|
27
|
+
*/
|
|
28
|
+
export function parseRetryAfter(raw, { dialect = 'http', now = Date.now() } = {}) {
|
|
29
|
+
if (raw == null || raw === '' || dialect === 'none') return undefined;
|
|
30
|
+
const number = Number(raw);
|
|
31
|
+
if (Number.isFinite(number) && number >= 0) return number * (dialect === 'milliseconds' ? 1 : 1000);
|
|
32
|
+
if (dialect !== 'http') return undefined;
|
|
33
|
+
const date = Date.parse(raw);
|
|
34
|
+
return Number.isNaN(date) ? undefined : Math.max(0, date - now);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** @param {AbortSignal | null | undefined} signal @returns {any} */
|
|
38
|
+
export function abortError(signal) {
|
|
39
|
+
if (signal?.reason !== undefined) return signal.reason;
|
|
40
|
+
const error = new Error('The operation was aborted.');
|
|
41
|
+
error.name = 'AbortError';
|
|
42
|
+
return error;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Abortable timer with listener cleanup on either settlement.
|
|
46
|
+
* @param {number} ms @param {AbortSignal} [signal] @returns {Promise<void>} */
|
|
47
|
+
export function sleep(ms, signal = undefined) {
|
|
48
|
+
return new Promise((resolve, reject) => {
|
|
49
|
+
if (signal?.aborted) { reject(abortError(signal)); return; }
|
|
50
|
+
const onAbort = () => {
|
|
51
|
+
clearTimeout(timer);
|
|
52
|
+
reject(abortError(signal));
|
|
53
|
+
};
|
|
54
|
+
const timer = setTimeout(() => {
|
|
55
|
+
signal?.removeEventListener('abort', onAbort);
|
|
56
|
+
resolve();
|
|
57
|
+
}, ms);
|
|
58
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* A host-private total budget reused across SDK, workflow and job callbacks.
|
|
64
|
+
* The single-attempt transport takes a credit immediately before dispatch.
|
|
65
|
+
* @param {number} attempts
|
|
66
|
+
* @param {'safe-read' | 'provider-idempotent' | 'single-send'} [safety]
|
|
67
|
+
*/
|
|
68
|
+
export function createAttemptBudget(attempts, safety = 'safe-read') {
|
|
69
|
+
if (!Number.isSafeInteger(attempts) || attempts < 1
|
|
70
|
+
|| !['safe-read', 'provider-idempotent', 'single-send'].includes(safety))
|
|
71
|
+
throw new TypeError('attempt budget needs positive total attempts and an explicit safety category');
|
|
72
|
+
const limit = safety === 'single-send' ? 1 : attempts;
|
|
73
|
+
let used = 0;
|
|
74
|
+
return Object.freeze({
|
|
75
|
+
safety,
|
|
76
|
+
get used() { return used; },
|
|
77
|
+
get remaining() { return limit - used; },
|
|
78
|
+
take() { if (used >= limit) return false; used++; return true; },
|
|
79
|
+
});
|
|
80
|
+
}
|
package/src/schedule.js
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** Bounded fair admission, with per-scope spacing and drained shutdown. */
|
|
3
|
+
import { sleep as defaultSleep } from './retry.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* @typedef {Object} ScheduleOptions
|
|
7
|
+
* @property {number} [concurrency]
|
|
8
|
+
* @property {number} [maxQueue]
|
|
9
|
+
* @property {number} [spacingMs]
|
|
10
|
+
* @property {number} [maxScopes]
|
|
11
|
+
* @property {() => number} [now]
|
|
12
|
+
* @property {(ms: number, signal?: AbortSignal) => Promise<void>} [sleep]
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Round-robin admission among ready scopes; FIFO inside each scope. Closing
|
|
17
|
+
* stops admission immediately and waits for admitted work, including workers
|
|
18
|
+
* which ignore cancellation. A host must keep resources until close settles.
|
|
19
|
+
* @param {ScheduleOptions} [options]
|
|
20
|
+
*/
|
|
21
|
+
export function createScheduler(options = {}) {
|
|
22
|
+
const { concurrency = 4, maxQueue = 64, maxScopes = 256, spacingMs = 0, now = Date.now, sleep = defaultSleep } = options;
|
|
23
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1
|
|
24
|
+
|| !Number.isSafeInteger(maxQueue) || maxQueue < 1
|
|
25
|
+
|| !Number.isSafeInteger(maxScopes) || maxScopes < 1
|
|
26
|
+
|| !Number.isFinite(spacingMs) || spacingMs < 0
|
|
27
|
+
|| typeof now !== 'function' || typeof sleep !== 'function')
|
|
28
|
+
throw new TypeError('scheduler needs finite concurrency, queue and spacing bounds and clock/sleep functions');
|
|
29
|
+
/** @type {any[]} */
|
|
30
|
+
const queue = [];
|
|
31
|
+
/** @type {Map<string, number>} */
|
|
32
|
+
const ready = new Map();
|
|
33
|
+
const turns = new Map();
|
|
34
|
+
let turn = 0;
|
|
35
|
+
let lastScope = '';
|
|
36
|
+
let active = 0;
|
|
37
|
+
let closed = false;
|
|
38
|
+
let scheduled = false;
|
|
39
|
+
/** @type {AbortController | null} */
|
|
40
|
+
let wake = null;
|
|
41
|
+
/** @type {Array<() => void>} */
|
|
42
|
+
const drained = [];
|
|
43
|
+
|
|
44
|
+
function kick() {
|
|
45
|
+
if (scheduled) return;
|
|
46
|
+
scheduled = true;
|
|
47
|
+
queueMicrotask(pump);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function pump() {
|
|
51
|
+
scheduled = false;
|
|
52
|
+
wake?.abort();
|
|
53
|
+
wake = null;
|
|
54
|
+
let at = now();
|
|
55
|
+
function prune() {
|
|
56
|
+
at = now();
|
|
57
|
+
for (let i = queue.length - 1; i >= 0; i--) {
|
|
58
|
+
const item = queue[i];
|
|
59
|
+
const reason = closed ? 'closed' : item.signal?.aborted ? 'cancelled' : item.deadline <= at ? 'deadline' : null;
|
|
60
|
+
if (reason !== null) {
|
|
61
|
+
queue.splice(i, 1);
|
|
62
|
+
item.cleanup();
|
|
63
|
+
item.reject(new Error(reason));
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
prune();
|
|
68
|
+
while (active < concurrency && queue.length) {
|
|
69
|
+
prune();
|
|
70
|
+
let index = -1;
|
|
71
|
+
let oldest = Infinity;
|
|
72
|
+
for (let i = 0; i < queue.length; i++) {
|
|
73
|
+
const item = queue[i];
|
|
74
|
+
const served = turns.get(item.scope) ?? (item.scope === lastScope ? turn : 0);
|
|
75
|
+
if ((ready.get(item.scope) ?? 0) <= at && served < oldest) { index = i; oldest = served; }
|
|
76
|
+
}
|
|
77
|
+
if (index < 0) break;
|
|
78
|
+
const [item] = queue.splice(index, 1);
|
|
79
|
+
item.cleanup();
|
|
80
|
+
lastScope = item.scope;
|
|
81
|
+
turns.set(item.scope, ++turn);
|
|
82
|
+
ready.set(item.scope, at + spacingMs);
|
|
83
|
+
active++;
|
|
84
|
+
// Invoke in this turn, so no cancellation microtask can slip between
|
|
85
|
+
// admission and the worker's own dispatch/authority check.
|
|
86
|
+
let answer;
|
|
87
|
+
try { answer = item.worker(); }
|
|
88
|
+
catch (error) { answer = Promise.reject(error); }
|
|
89
|
+
Promise.resolve(answer).then(item.resolve, item.reject).finally(() => {
|
|
90
|
+
active--;
|
|
91
|
+
kick();
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
// Spacing state for inactive scopes expires, bounding retention by the
|
|
95
|
+
// rate window and active/queued scopes rather than the lifetime of a host.
|
|
96
|
+
for (const [scope, time] of ready) if (time <= at && !queue.some((item) => item.scope === scope)) ready.delete(scope);
|
|
97
|
+
for (const scope of turns.keys()) if (!queue.some((item) => item.scope === scope)) turns.delete(scope);
|
|
98
|
+
if (queue.length) {
|
|
99
|
+
let next = Math.min(...queue.map((item) => item.deadline));
|
|
100
|
+
if (active < concurrency) next = Math.min(next, ...queue.map((item) => ready.get(item.scope) ?? at));
|
|
101
|
+
if (Number.isFinite(next)) {
|
|
102
|
+
const controller = new AbortController();
|
|
103
|
+
wake = controller;
|
|
104
|
+
Promise.resolve().then(() => sleep(Math.max(0, next - now()), controller.signal)).then(() => {
|
|
105
|
+
if (!controller.signal.aborted) kick();
|
|
106
|
+
}, (error) => {
|
|
107
|
+
if (controller.signal.aborted) return;
|
|
108
|
+
closed = true;
|
|
109
|
+
for (const item of queue.splice(0)) { item.cleanup(); item.reject(error); }
|
|
110
|
+
kick();
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
if (!active && !queue.length) for (const resolve of drained.splice(0)) resolve();
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
return Object.freeze({
|
|
118
|
+
/** @template T @param {() => T | Promise<T>} worker
|
|
119
|
+
* @param {{ scope?: string, signal?: AbortSignal, deadline?: number }} [context]
|
|
120
|
+
* @returns {Promise<T>} */
|
|
121
|
+
run(worker, { scope = '', signal, deadline = Infinity } = {}) {
|
|
122
|
+
if (typeof worker !== 'function' || typeof scope !== 'string' || typeof deadline !== 'number' || Number.isNaN(deadline))
|
|
123
|
+
return Promise.reject(new TypeError('scheduler needs a worker, scope and deadline'));
|
|
124
|
+
const reason = closed ? 'closed' : signal?.aborted ? 'cancelled' : deadline <= now() ? 'deadline'
|
|
125
|
+
: queue.length >= maxQueue ? 'queue-full' : null;
|
|
126
|
+
if (reason !== null) return Promise.reject(new Error(reason));
|
|
127
|
+
for (const [name, time] of ready) if (time <= now()) ready.delete(name);
|
|
128
|
+
const scopes = new Set([...ready.keys(), ...queue.map((item) => item.scope)]);
|
|
129
|
+
if (!scopes.has(scope) && scopes.size >= maxScopes) return Promise.reject(new Error('scope-limit'));
|
|
130
|
+
return new Promise((resolve, reject) => {
|
|
131
|
+
signal?.addEventListener('abort', kick, { once: true });
|
|
132
|
+
queue.push({ worker, scope, signal, deadline, resolve, reject,
|
|
133
|
+
cleanup: () => signal?.removeEventListener('abort', kick) });
|
|
134
|
+
kick();
|
|
135
|
+
});
|
|
136
|
+
},
|
|
137
|
+
/** A server observation delays all following work in its scope.
|
|
138
|
+
* @param {string} scope @param {number} delayMs */
|
|
139
|
+
observe(scope, delayMs) {
|
|
140
|
+
if (typeof scope !== 'string' || !Number.isFinite(delayMs) || delayMs < 0)
|
|
141
|
+
throw new TypeError('rate observation needs a scope and a nonnegative delay');
|
|
142
|
+
if (closed) return;
|
|
143
|
+
for (const [name, time] of ready) if (time <= now()) ready.delete(name);
|
|
144
|
+
if (!ready.has(scope) && ready.size >= maxScopes) throw new Error('scope-limit');
|
|
145
|
+
ready.set(scope, Math.max(ready.get(scope) ?? 0, now() + delayMs));
|
|
146
|
+
kick();
|
|
147
|
+
},
|
|
148
|
+
/** @returns {Promise<void>} */
|
|
149
|
+
close() {
|
|
150
|
+
closed = true;
|
|
151
|
+
kick();
|
|
152
|
+
return new Promise((resolve) => { drained.push(resolve); });
|
|
153
|
+
},
|
|
154
|
+
stats: () => ({ active, queued: queue.length, closed }),
|
|
155
|
+
});
|
|
156
|
+
}
|