@jarenjs/db 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +397 -0
- package/README.md +218 -0
- package/dist/types/algebra.d.ts +133 -0
- package/dist/types/app.d.ts +49 -0
- package/dist/types/capture.d.ts +85 -0
- package/dist/types/cli.d.ts +2 -0
- package/dist/types/dag-job.d.ts +40 -0
- package/dist/types/ddl.d.ts +170 -0
- package/dist/types/dialect.d.ts +130 -0
- package/dist/types/dialects/sqlite.d.ts +9 -0
- package/dist/types/driver.d.ts +128 -0
- package/dist/types/drivers/bun.d.ts +47 -0
- package/dist/types/drivers/node.d.ts +37 -0
- package/dist/types/drivers/wasm.d.ts +65 -0
- package/dist/types/emit-model.d.ts +44 -0
- package/dist/types/emit.d.ts +72 -0
- package/dist/types/entity.d.ts +23 -0
- package/dist/types/errors.d.ts +165 -0
- package/dist/types/graph.d.ts +28 -0
- package/dist/types/index.d.ts +35 -0
- package/dist/types/jobs.d.ts +134 -0
- package/dist/types/live.d.ts +62 -0
- package/dist/types/migrate.d.ts +163 -0
- package/dist/types/model.d.ts +36 -0
- package/dist/types/patch-sql.d.ts +37 -0
- package/dist/types/plan.d.ts +119 -0
- package/dist/types/profile.d.ts +80 -0
- package/dist/types/query.d.ts +100 -0
- package/dist/types/residual.d.ts +50 -0
- package/dist/types/store.d.ts +53 -0
- package/dist/types/tracker.d.ts +43 -0
- package/dist/types/typed.d.ts +15 -0
- package/dist/types/types.d.ts +26 -0
- package/dist/types/udf.d.ts +70 -0
- package/dist/types/window.d.ts +52 -0
- package/docs/JOBS-FORMAT.md +218 -0
- package/docs/LIVE-FORMAT.md +348 -0
- package/docs/MIGRATION-FORMAT.md +302 -0
- package/docs/MODEL-FORMAT.md +928 -0
- package/package.json +81 -0
- package/schemas/jaren-migration.draft-07.schema.json +144 -0
- package/schemas/jaren-migration.schema.json +144 -0
- package/schemas/jaren-model.draft-07.schema.json +149 -0
- package/schemas/jaren-model.schema.json +149 -0
- package/src/algebra.js +105 -0
- package/src/app.js +108 -0
- package/src/capture.js +584 -0
- package/src/cli.js +264 -0
- package/src/dag-job.js +86 -0
- package/src/ddl.js +588 -0
- package/src/dialect.js +297 -0
- package/src/dialects/sqlite.js +175 -0
- package/src/driver.js +419 -0
- package/src/drivers/bun.js +101 -0
- package/src/drivers/node.js +93 -0
- package/src/drivers/wasm.js +178 -0
- package/src/emit-model.js +208 -0
- package/src/emit.js +393 -0
- package/src/entity.js +367 -0
- package/src/errors.js +173 -0
- package/src/graph.js +101 -0
- package/src/index.js +64 -0
- package/src/jobs.js +507 -0
- package/src/live.js +899 -0
- package/src/migrate.js +1411 -0
- package/src/model.js +476 -0
- package/src/patch-sql.js +150 -0
- package/src/plan.js +1038 -0
- package/src/profile.js +131 -0
- package/src/query.js +1010 -0
- package/src/residual.js +91 -0
- package/src/store.js +1422 -0
- package/src/tracker.js +776 -0
- package/src/typed.js +19 -0
- package/src/types.js +36 -0
- package/src/udf.js +132 -0
- package/src/window.js +125 -0
- package/types/app.d.ts +36 -0
- package/types/bun.d.ts +9 -0
- package/types/index.d.ts +592 -0
- package/types/node.d.ts +15 -0
- package/types/typed.d.ts +108 -0
- package/types/wasm.d.ts +5 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The query surface over one collection: the D2 provider
|
|
3
|
+
* (`execute(document, options)` — how a linq chain runs here with no
|
|
4
|
+
* import edge), the streaming cursor (`query`), and `explain()`.
|
|
5
|
+
*
|
|
6
|
+
* The statement cache is a CALLER of the core primitives:
|
|
7
|
+
* `createSemanticCache` keyed by the whole discriminating tuple —
|
|
8
|
+
* document plus collection, dialect, strictness, pushdown and profile.
|
|
9
|
+
* The identity is the tuple's COMPLETE serialization, never a
|
|
10
|
+
* fingerprint of it: a 32-bit content hash collides after tens of
|
|
11
|
+
* thousands of documents, and a collision here answers one query with
|
|
12
|
+
* another query's plan and rows. `store.stats()` exposes hits, misses
|
|
13
|
+
* and evictions so the cache is proven rather than assumed.
|
|
14
|
+
*
|
|
15
|
+
* Bind-time diversion: if any referenced external is missing or not a
|
|
16
|
+
* string or finite number, the call runs the always-compilable set
|
|
17
|
+
* residual over the full collection instead of the native statement —
|
|
18
|
+
* SQLite cannot bind a boolean, a `null` needs Jaren's semantics, and
|
|
19
|
+
* a missing external must raise the ENGINE's error, not a driver's.
|
|
20
|
+
*/
|
|
21
|
+
/**
|
|
22
|
+
* The store-wide query state shared by every collection's engine: one
|
|
23
|
+
* bounded statement cache, its counters, the UDF registration set, and
|
|
24
|
+
* the store's resolved registered operators (Ring 2 — `{ functions,
|
|
25
|
+
* extensions }` or `null`), threaded to every engine that builds a
|
|
26
|
+
* residual.
|
|
27
|
+
* @param {number} [bound]
|
|
28
|
+
* @param {{ functions?: any, extensions?: any } | null} [operators]
|
|
29
|
+
* @returns {any}
|
|
30
|
+
*/
|
|
31
|
+
export declare function createQueryState(bound?: number, operators?: {
|
|
32
|
+
functions?: any;
|
|
33
|
+
extensions?: any;
|
|
34
|
+
} | null): any;
|
|
35
|
+
/**
|
|
36
|
+
* The query engine for one collection.
|
|
37
|
+
* @param {{ connection: any, state: any, collection: any,
|
|
38
|
+
* physicalPlan: any, profile?: any }} context - `collection` is the
|
|
39
|
+
* normalized collection; `physicalPlan` is the DDL plan
|
|
40
|
+
* (columns, indexes); `profile` is the store-level normalized
|
|
41
|
+
* profile, if one was opened with
|
|
42
|
+
* @returns {{ execute: Function, query: Function, explain: Function }}
|
|
43
|
+
*/
|
|
44
|
+
export declare function createQueryEngine(context: {
|
|
45
|
+
connection: any;
|
|
46
|
+
state: any;
|
|
47
|
+
collection: any;
|
|
48
|
+
physicalPlan: any;
|
|
49
|
+
profile?: any;
|
|
50
|
+
}): {
|
|
51
|
+
execute: Function;
|
|
52
|
+
query: Function;
|
|
53
|
+
explain: Function;
|
|
54
|
+
};
|
|
55
|
+
/** The default include depth bound (D14: printed, never silent). */
|
|
56
|
+
export declare const INCLUDE_DEPTH_DEFAULT = 3;
|
|
57
|
+
/**
|
|
58
|
+
* The store-level entity query engine: documents over the
|
|
59
|
+
* multi-entity root (`$.<Entity>[*]` bindings), planned to guarded
|
|
60
|
+
* selections and INNER equijoins, with the set residual running the
|
|
61
|
+
* whole document over the fetched root — the same honesty contract as
|
|
62
|
+
* phase A.
|
|
63
|
+
* @param {{ connection: any, entities: Map<string, any>, mapping: any,
|
|
64
|
+
* state: any }} context
|
|
65
|
+
* @returns {any}
|
|
66
|
+
*/
|
|
67
|
+
export declare function createEntityQueryEngine(context: {
|
|
68
|
+
connection: any;
|
|
69
|
+
entities: Map<string, any>;
|
|
70
|
+
mapping: any;
|
|
71
|
+
state: any;
|
|
72
|
+
}): any;
|
|
73
|
+
/**
|
|
74
|
+
* The one-statement graph loader: `entity.load(spec)` compiles an
|
|
75
|
+
* include tree to correlated subqueries projected as JSON — one
|
|
76
|
+
* statement regardless of depth (asserted by a counting driver in the
|
|
77
|
+
* tests, because N+1 is a test, not a promise). Per-relation `where`,
|
|
78
|
+
* `orderBy` and `take` are applied INSIDE the subquery; depth is
|
|
79
|
+
* bounded with the default printed in the refusal; cycles in the
|
|
80
|
+
* specification are rejected; and keyset pagination is chosen over a
|
|
81
|
+
* growing OFFSET whenever the top-level ordering is a single unique
|
|
82
|
+
* column, with the choice reported by `explainLoad`.
|
|
83
|
+
* @param {{ connection: any, entities: Map<string, any>, mapping: any,
|
|
84
|
+
* state: any }} context
|
|
85
|
+
* @param {string} entityName
|
|
86
|
+
* @returns {any}
|
|
87
|
+
*/
|
|
88
|
+
export declare function createLoadEngine(context: {
|
|
89
|
+
connection: any;
|
|
90
|
+
entities: Map<string, any>;
|
|
91
|
+
mapping: any;
|
|
92
|
+
state: any;
|
|
93
|
+
}, entityName: string): any;
|
|
94
|
+
export type ParamCollector = {
|
|
95
|
+
literal?: any;
|
|
96
|
+
external?: string;
|
|
97
|
+
}[];
|
|
98
|
+
/**
|
|
99
|
+
* @typedef {{ literal?: any, external?: string }[]} ParamCollector
|
|
100
|
+
*/
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Residual compilation: the part of a query that stays in
|
|
3
|
+
* JavaScript is a REAL compiled Jaren query — the same engine, the
|
|
4
|
+
* same semantics — never a reimplementation.
|
|
5
|
+
*
|
|
6
|
+
* Two modes (ARCHITECTURE.md):
|
|
7
|
+
*
|
|
8
|
+
* - `set` — the whole original document compiled once, run over the
|
|
9
|
+
* materialized candidate array. Re-applying pushed conjuncts is
|
|
10
|
+
* idempotent, so SQL-side narrowing never changes the answer.
|
|
11
|
+
* - `row` — only the projection stayed behind: each fetched row runs
|
|
12
|
+
* `{ $for: { it: '$[*]' }, $return: [ <ret> ] }` over the one-row
|
|
13
|
+
* array; the array wrapper packs the item sequence so an
|
|
14
|
+
* array-VALUED item stays unambiguous, and the per-row results
|
|
15
|
+
* concatenate in row order (streamable).
|
|
16
|
+
*/
|
|
17
|
+
/**
|
|
18
|
+
* Compile the whole document for set-mode evaluation. A profile's
|
|
19
|
+
* engine limits ride into the compilation so the JavaScript portion is
|
|
20
|
+
* bounded by the engine's own enforcement.
|
|
21
|
+
* @param {any} document
|
|
22
|
+
* @param {any} [limits]
|
|
23
|
+
* @param {{ functions?: any, extensions?: any } | null} [operators] -
|
|
24
|
+
* the store's registered operators, so the residual can evaluate them
|
|
25
|
+
* @returns {(candidates: any[], externals: any) => any}
|
|
26
|
+
*/
|
|
27
|
+
export declare function compileSetResidual(document: any, limits?: any, operators?: {
|
|
28
|
+
functions?: any;
|
|
29
|
+
extensions?: any;
|
|
30
|
+
} | null): (candidates: any[], externals: any) => any;
|
|
31
|
+
/**
|
|
32
|
+
* Compile the per-row projection for row-mode evaluation.
|
|
33
|
+
* @param {any} returnExpression - The document's raw `$return` value
|
|
34
|
+
* @param {any} [limits]
|
|
35
|
+
* @param {{ functions?: any, extensions?: any } | null} [operators]
|
|
36
|
+
* @returns {(row: any, externals: any) => any[]} the row's items
|
|
37
|
+
*/
|
|
38
|
+
export declare function compileRowResidual(returnExpression: any, limits?: any, operators?: {
|
|
39
|
+
functions?: any;
|
|
40
|
+
extensions?: any;
|
|
41
|
+
} | null): (row: any, externals: any) => any[];
|
|
42
|
+
/**
|
|
43
|
+
* Map a flat item array onto the engine's result shape: an empty
|
|
44
|
+
* sequence is `undefined`, a singleton is the item, anything longer is
|
|
45
|
+
* the array (probed engine behaviour, pinned by the differential
|
|
46
|
+
* tests).
|
|
47
|
+
* @param {any[]} items
|
|
48
|
+
* @returns {any}
|
|
49
|
+
*/
|
|
50
|
+
export declare function sequenceResult(items: any[]): any;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The store: `openStore(model, options)` normalizes the model
|
|
3
|
+
* document, opens a database through the injected driver, applies (or
|
|
4
|
+
* verifies) the physical shape, and gives transactional,
|
|
5
|
+
* schema-validated reads and writes.
|
|
6
|
+
*
|
|
7
|
+
* The public surface is ASYNCHRONOUS — every method returns a promise
|
|
8
|
+
* — because the browser's OPFS story forces it regardless of any other
|
|
9
|
+
* backend. Where the driver is synchronous the same operations exist
|
|
10
|
+
* without the promise under `store.sync`, present only there — never a
|
|
11
|
+
* throwing stub — so portable code pays one declared microsecond and
|
|
12
|
+
* code that wants it back opts in knowingly. Internally everything
|
|
13
|
+
* composes through the sync-capable {@link chain}, so the async
|
|
14
|
+
* surface allocates exactly one promise per call, not one per step.
|
|
15
|
+
*
|
|
16
|
+
* Writes validate through an injected `compileSchema` hook with the
|
|
17
|
+
* `compileTypeTest` signature; absent, writes are unvalidated and
|
|
18
|
+
* `store.capabilities.validated === false` — a declared downgrade,
|
|
19
|
+
* never a silent one. `@jarenjs/validate` is never imported here.
|
|
20
|
+
*/
|
|
21
|
+
/** The model format version this store implements. */
|
|
22
|
+
export declare const MODEL_VERSION = "0.1";
|
|
23
|
+
/**
|
|
24
|
+
* Normalize and check a model document. Every failure is `JD0005` with
|
|
25
|
+
* a `docPath` into the model.
|
|
26
|
+
* @param {any} model
|
|
27
|
+
* @returns {Map<string, any>} collection name -> normalized collection
|
|
28
|
+
*/
|
|
29
|
+
export declare function normalizeModel(model: any): Map<string, any>;
|
|
30
|
+
/**
|
|
31
|
+
* Open (or create) a store described by a model document.
|
|
32
|
+
* @param {any} model - A `jaren-model` document (the 0.1 subset)
|
|
33
|
+
* @param {{ driver: any, path?: string, compileSchema?: Function,
|
|
34
|
+
* busyTimeout?: number, queueTimeout?: number, journalMode?: string,
|
|
35
|
+
* statementCacheBound?: number, profile?: any, operators?: any,
|
|
36
|
+
* functions?: any, extensions?: any,
|
|
37
|
+
* readOnly?: boolean }} options
|
|
38
|
+
* @returns {Promise<any>}
|
|
39
|
+
*/
|
|
40
|
+
export declare function openStore(model: any, options: {
|
|
41
|
+
driver: any;
|
|
42
|
+
path?: string;
|
|
43
|
+
compileSchema?: Function;
|
|
44
|
+
busyTimeout?: number;
|
|
45
|
+
queueTimeout?: number;
|
|
46
|
+
journalMode?: string;
|
|
47
|
+
statementCacheBound?: number;
|
|
48
|
+
profile?: any;
|
|
49
|
+
operators?: any;
|
|
50
|
+
functions?: any;
|
|
51
|
+
extensions?: any;
|
|
52
|
+
readOnly?: boolean;
|
|
53
|
+
}): Promise<any>;
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The unit of work (§11): copy-on-write change tracking and
|
|
3
|
+
* minimal writes. Materialised entities are plain, DEEP-FROZEN JSON —
|
|
4
|
+
* no proxies anywhere — and the tracker retains exactly ONE reference
|
|
5
|
+
* per tracked entity: the frozen document itself is the snapshot.
|
|
6
|
+
* Mutation is replacement (`put(next)`); `saveChanges()` diffs
|
|
7
|
+
* snapshot against current with the suite's own diff engine and plans
|
|
8
|
+
* the MINIMAL set of parameterised statements: scalar/epoch/foreign-
|
|
9
|
+
* key column writes, `jsonb_set`/`jsonb_remove` chains for document
|
|
10
|
+
* paths, join-table synchronisation for many-to-many members, and a
|
|
11
|
+
* counted whole-row fallback for anything untranslatable.
|
|
12
|
+
*
|
|
13
|
+
* Ordering never violates a foreign key mid-transaction: inserts run
|
|
14
|
+
* parent-first, deletes child-first, updates in between, join rows
|
|
15
|
+
* after both endpoints exist. A foreign-key cycle among the entities
|
|
16
|
+
* being inserted or deleted is `JD0040`, reported, never a deadlock.
|
|
17
|
+
* The whole save is one transaction; the tracker is mutated ONLY
|
|
18
|
+
* after commit, so a failed save leaves it exactly as it was and a
|
|
19
|
+
* retry is possible.
|
|
20
|
+
*/
|
|
21
|
+
/** Rows per batched INSERT: bounded by the portable parameter budget. */
|
|
22
|
+
export declare const BATCH_PARAM_BUDGET = 900;
|
|
23
|
+
export declare const BATCH_ROW_BOUND = 100;
|
|
24
|
+
/**
|
|
25
|
+
* Deep-freeze a JSON value in place and return it. Idempotent; shared
|
|
26
|
+
* substructure (a graph load's children) freezes once.
|
|
27
|
+
* @template T
|
|
28
|
+
* @param {T} value
|
|
29
|
+
* @returns {T}
|
|
30
|
+
*/
|
|
31
|
+
export declare function deepFreeze<T>(value: T): T;
|
|
32
|
+
/**
|
|
33
|
+
* The store-level unit of work.
|
|
34
|
+
* @param {{ connection: any, entities: Map<string, any>, mapping: any,
|
|
35
|
+
* coreFor: (name: string) => any }} context
|
|
36
|
+
* @returns {any}
|
|
37
|
+
*/
|
|
38
|
+
export declare function createTracker(context: {
|
|
39
|
+
connection: any;
|
|
40
|
+
entities: Map<string, any>;
|
|
41
|
+
mapping: any;
|
|
42
|
+
coreFor: (name: string) => any;
|
|
43
|
+
}): any;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The typed-store binding: a zero-cost identity whose ONLY job
|
|
3
|
+
* is carrying the generated `EntityMetaMap` into the type system —
|
|
4
|
+
* `typedStore<EntityMetaMap>(store)` narrows `entity(name)` to the
|
|
5
|
+
* generated document/input shapes and widens `load` results by their
|
|
6
|
+
* include specification. The runtime is the store it was given; every
|
|
7
|
+
* guarantee lives in `types/typed.d.ts` and the generated artifact.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Bind a store to its generated entity metadata. Identity at runtime.
|
|
11
|
+
* @template E
|
|
12
|
+
* @param {any} store - an opened store whose model generated `E`
|
|
13
|
+
* @returns {any}
|
|
14
|
+
*/
|
|
15
|
+
export declare function typedStore<E>(store: any): any;
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Schema-derived path typing: the collection's JSON Schema is the
|
|
3
|
+
* planner's type source (there is no engine-side inference in this
|
|
4
|
+
* phase — the store's own contract carries the types). Where the
|
|
5
|
+
* schema says nothing, the answer is `'unknown'` and the planner emits
|
|
6
|
+
* the defensive form; the guards in the truth table make `'unknown'`
|
|
7
|
+
* safe, so typing here only ever IMPROVES a plan (column choice), it
|
|
8
|
+
* never weakens one.
|
|
9
|
+
*/
|
|
10
|
+
/**
|
|
11
|
+
* The planner-facing type of a member path.
|
|
12
|
+
* @param {any} schema - The collection's document schema
|
|
13
|
+
* @param {({ name: string } | { index: number })[]} segments
|
|
14
|
+
* @returns {'string' | 'integer' | 'number' | 'boolean' | 'unknown'}
|
|
15
|
+
*/
|
|
16
|
+
export declare function typeOfPath(schema: any, segments: ({
|
|
17
|
+
name: string;
|
|
18
|
+
} | {
|
|
19
|
+
index: number;
|
|
20
|
+
})[]): 'string' | 'integer' | 'number' | 'boolean' | 'unknown';
|
|
21
|
+
/**
|
|
22
|
+
* Whether a planner type is numeric (SQL comparison family).
|
|
23
|
+
* @param {string} type
|
|
24
|
+
* @returns {boolean}
|
|
25
|
+
*/
|
|
26
|
+
export declare function isNumericType(type: string): boolean;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The UDF escape hatch (capability-gated): between native SQL
|
|
3
|
+
* and pulling rows sits registering a compiled predicate conjunct as a
|
|
4
|
+
* DETERMINISTIC function used in the WHERE clause. Only fragments the
|
|
5
|
+
* analysis proves deterministic and side-effect-free qualify — no
|
|
6
|
+
* externals (their values change per call, and a deterministic
|
|
7
|
+
* function must not close over changing state), no host functions, no
|
|
8
|
+
* collations. Registration is keyed by the fragment's COLLISION-FREE
|
|
9
|
+
* structural identity, so identical fragments share one registration
|
|
10
|
+
* per store and different fragments never do. A fingerprint could not
|
|
11
|
+
* decide this: two colliding fragments would claim one SQL function
|
|
12
|
+
* name, the second would silently reuse the first's predicate, and the
|
|
13
|
+
* WHERE clause would filter on the wrong condition. The SQL identifier
|
|
14
|
+
* still derives from a short fingerprint — names must be short — but a
|
|
15
|
+
* fingerprint clash between DIFFERENT identities is disambiguated with
|
|
16
|
+
* a suffix rather than collapsed.
|
|
17
|
+
*
|
|
18
|
+
* WHERE-clause use only: an INDEX over a registered function would
|
|
19
|
+
* make the database unwritable from any connection that has not
|
|
20
|
+
* registered the identical function — that schema-dependency hazard is
|
|
21
|
+
* why the model format declares no UDF-expression indexes.
|
|
22
|
+
*
|
|
23
|
+
* Ring 3 extends the hatch to registry `pushable:'scalar'`
|
|
24
|
+
* operators: a predicate fragment that uses a registered scalar operator
|
|
25
|
+
* (`$sqrt`, `$pow`, …) compiles WITH the store's `{ functions,
|
|
26
|
+
* extensions }` and pushes as the same deterministic UDF — SQLite drives
|
|
27
|
+
* the row iteration and the operator runs inside the callback, instead
|
|
28
|
+
* of every candidate crossing into the residual. The determinism rule is
|
|
29
|
+
* unchanged (no externals, no collations); a registered operator the
|
|
30
|
+
* pack did NOT mark `pushable:'scalar'` (a whole-series `$npv`, an
|
|
31
|
+
* aggregate) stays the residual — Ring 2 keeps it correct.
|
|
32
|
+
*/
|
|
33
|
+
/**
|
|
34
|
+
* Decide whether a raw predicate fragment qualifies for the hatch, and
|
|
35
|
+
* build its registration if so.
|
|
36
|
+
* @param {any} fragment - The raw conjunct (a JSON query expression
|
|
37
|
+
* over `$it`)
|
|
38
|
+
* @param {{ functions?: any, extensions?: any, pushableScalar?: Set<string> } | null}
|
|
39
|
+
* [operators] - the store's registered operators (Ring 3); only its
|
|
40
|
+
* `pushable:'scalar'` subset is admitted. `null`/absent keeps the
|
|
41
|
+
* original engine-internal-only rule (no host function pushes).
|
|
42
|
+
* @returns {{ key: string, name: string,
|
|
43
|
+
* compile: () => (docText: string) => number } | null}
|
|
44
|
+
*/
|
|
45
|
+
export declare function deterministicFragment(fragment: any, operators?: {
|
|
46
|
+
functions?: any;
|
|
47
|
+
extensions?: any;
|
|
48
|
+
pushableScalar?: Set<string>;
|
|
49
|
+
} | null): {
|
|
50
|
+
key: string;
|
|
51
|
+
name: string;
|
|
52
|
+
compile: () => (docText: string) => number;
|
|
53
|
+
} | null;
|
|
54
|
+
/**
|
|
55
|
+
* Register a qualified fragment once per store, and answer the SQL name
|
|
56
|
+
* to call. Identity is the fragment's structural key, so the same
|
|
57
|
+
* fragment registers once and two different fragments always get two
|
|
58
|
+
* different functions — even when their short names fingerprint alike,
|
|
59
|
+
* which the suffix resolves.
|
|
60
|
+
* @param {any} connection
|
|
61
|
+
* @param {Map<string, string>} registered - The store's registrations,
|
|
62
|
+
* fragment identity → the SQL function name it owns
|
|
63
|
+
* @param {{ key: string, name: string, compile: () => Function }} fragment
|
|
64
|
+
* @returns {string} the function name to call in the WHERE clause
|
|
65
|
+
*/
|
|
66
|
+
export declare function registerFragment(connection: any, registered: Map<string, string>, fragment: {
|
|
67
|
+
key: string;
|
|
68
|
+
name: string;
|
|
69
|
+
compile: () => Function;
|
|
70
|
+
}): string;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file The maintained sorted window for `orderBy` (+ optional
|
|
3
|
+
* `limit`) live queries — LIVE-FORMAT §7's window row. The structure
|
|
4
|
+
* keeps EVERY matching row in sorted order (which is what makes a
|
|
5
|
+
* delete inside the visible window answerable without a re-query: the
|
|
6
|
+
* successor is already here), and the visible result is the first
|
|
7
|
+
* `limit` entries. Ties are broken by the collection key token,
|
|
8
|
+
* always ascending — the total order LIVE-FORMAT §9 promises.
|
|
9
|
+
*
|
|
10
|
+
* String comparison is CODEPOINT order (SQLite BINARY over UTF-8 —
|
|
11
|
+
* probed, never assumed), which `<` on JS strings gets wrong for
|
|
12
|
+
* astral-vs-BMP pairs; `compareCodepoint` walks code points.
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Compare two strings in Unicode code-point order.
|
|
16
|
+
* @param {string} a
|
|
17
|
+
* @param {string} b
|
|
18
|
+
* @returns {number}
|
|
19
|
+
*/
|
|
20
|
+
export declare function compareCodepoint(a: string, b: string): number;
|
|
21
|
+
export type WindowEntry = {
|
|
22
|
+
token: string;
|
|
23
|
+
sortValues: any[];
|
|
24
|
+
item: any;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* @typedef {{ token: string, sortValues: any[], item: any }} WindowEntry
|
|
28
|
+
*/
|
|
29
|
+
/**
|
|
30
|
+
* A maintained sorted set of rows under the declared order terms plus
|
|
31
|
+
* the key-token tiebreaker.
|
|
32
|
+
* @param {{ desc: boolean, emptyGreatest: boolean }[]} terms
|
|
33
|
+
* @param {number | null} limit - visible size; `null` = everything
|
|
34
|
+
*/
|
|
35
|
+
export declare function createSortedWindow(terms: {
|
|
36
|
+
desc: boolean;
|
|
37
|
+
emptyGreatest: boolean;
|
|
38
|
+
}[], limit: number | null): {
|
|
39
|
+
compare: (a: WindowEntry, b: WindowEntry) => number;
|
|
40
|
+
size: () => number;
|
|
41
|
+
/** The visible slice: the first `limit` entries (all, unbounded). */
|
|
42
|
+
visible: () => WindowEntry[];
|
|
43
|
+
/**
|
|
44
|
+
* Insert a row; the token MUST not be present.
|
|
45
|
+
* @param {string} token
|
|
46
|
+
* @param {any[]} sortValues
|
|
47
|
+
* @param {any} item
|
|
48
|
+
*/
|
|
49
|
+
insert(token: string, sortValues: any[], item: any): void;
|
|
50
|
+
/** Remove a row by token; absent tokens are a no-op. */
|
|
51
|
+
remove(token: any): boolean;
|
|
52
|
+
};
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
# The Jaren jobs format — the durable queue
|
|
2
|
+
|
|
3
|
+
This document is normative. The key words MUST, MUST NOT, SHOULD and
|
|
4
|
+
MAY are to be interpreted as described in RFC 2119. Error codes join
|
|
5
|
+
the package's single runtime table (MODEL-FORMAT §7); this order adds
|
|
6
|
+
none — see §9.
|
|
7
|
+
|
|
8
|
+
## 1. Scope
|
|
9
|
+
|
|
10
|
+
A single-database job queue: enqueue, lease, execute, retry with
|
|
11
|
+
backoff, dead-letter, and recover after a crash — the persistence,
|
|
12
|
+
polling, locking, retries and recovery an adopter otherwise builds as
|
|
13
|
+
infrastructure. It is enabled per store and lives in two internal
|
|
14
|
+
tables beside the model's own:
|
|
15
|
+
|
|
16
|
+
```js
|
|
17
|
+
const store = await openStore(model, { driver: nodeDriver(), jobs: true });
|
|
18
|
+
const id = await store.jobs.enqueue('welcome-mail', { userId: 'u1' });
|
|
19
|
+
const worker = store.jobs.createWorker({
|
|
20
|
+
handlers: { 'welcome-mail': async (payload, context) => { /* … */ } },
|
|
21
|
+
concurrency: 2,
|
|
22
|
+
});
|
|
23
|
+
worker.start();
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The queue composes with `@jarenjs/flow` checkpointed runs through §7
|
|
27
|
+
— that composition, not the queue alone, is the reason this exists:
|
|
28
|
+
a run that crashes resumes from its recorded nodes instead of
|
|
29
|
+
restarting.
|
|
30
|
+
|
|
31
|
+
## 2. The job record
|
|
32
|
+
|
|
33
|
+
`_jaren_jobs`, one row per job, surfaced camel-cased and frozen:
|
|
34
|
+
|
|
35
|
+
| column | meaning |
|
|
36
|
+
|---|---|
|
|
37
|
+
| `id` | TEXT primary key; caller-supplied or a UUID. **A caller-supplied id makes `enqueue` idempotent**: re-enqueueing an existing id changes nothing and answers the id — this is the idempotency key of §7 |
|
|
38
|
+
| `kind` | the registered handler name (the suite's named-registry discipline) |
|
|
39
|
+
| `payload` | the enqueue payload, JSON text; `null` stays null |
|
|
40
|
+
| `state` | `pending` → `leased` → `done`, or `failed` (awaiting retry) → `dead` (attempts exhausted) |
|
|
41
|
+
| `run_at` | epoch ms eligibility: scheduling and retry backoff are the same mechanism |
|
|
42
|
+
| `attempts` | claims so far; incremented AT claim, so a crashed attempt counts |
|
|
43
|
+
| `max_attempts` | per job, default 5 |
|
|
44
|
+
| `lease_until`, `lease_owner` | the lease (§3); null unless leased |
|
|
45
|
+
| `last_error` | the last failure's `message`, retained through retries and into `dead` |
|
|
46
|
+
| `result` | the completion value, JSON text (§7 records the DAG output here) |
|
|
47
|
+
| `created_at`, `updated_at` | epoch ms |
|
|
48
|
+
|
|
49
|
+
`_jaren_job_checkpoints` holds `(run_id, node_id, value)` rows — the
|
|
50
|
+
flow checkpoint store of §7, keyed by the JOB id (the run id IS the
|
|
51
|
+
job id). Both tables are created on open when `jobs` is requested;
|
|
52
|
+
neither appears in the model.
|
|
53
|
+
|
|
54
|
+
## 3. Leasing and exactly-once execution
|
|
55
|
+
|
|
56
|
+
Claiming is ONE guarded statement — one statement is one transaction,
|
|
57
|
+
so two workers cannot claim the same job, without any distributed
|
|
58
|
+
lock:
|
|
59
|
+
|
|
60
|
+
```sql
|
|
61
|
+
UPDATE "_jaren_jobs" SET state='leased', lease_owner=?, lease_until=?,
|
|
62
|
+
attempts=attempts+1, updated_at=?
|
|
63
|
+
WHERE id = (SELECT id FROM "_jaren_jobs"
|
|
64
|
+
WHERE (state='pending' OR state='failed'
|
|
65
|
+
OR (state='leased' AND lease_until < ?))
|
|
66
|
+
AND run_at <= ? AND kind IN (…the worker's registered kinds…)
|
|
67
|
+
ORDER BY run_at, created_at, id LIMIT 1)
|
|
68
|
+
RETURNING *
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
- The `kind IN` clause is the deploy-ordering protection: **a job
|
|
72
|
+
whose kind has no registered handler on this worker is simply never
|
|
73
|
+
claimed by it** — it stays `pending` and shows in `counts()`, it is
|
|
74
|
+
NOT dead-lettered by an accident of rollout order.
|
|
75
|
+
- Every later transition is guarded by the lease:
|
|
76
|
+
`… WHERE id=? AND state='leased' AND lease_owner=?`. A worker whose
|
|
77
|
+
lease expired mid-run (a stall, a long GC, a laptop lid) finds its
|
|
78
|
+
completion matching ZERO rows and its result discarded — the job
|
|
79
|
+
belongs to whoever re-claimed it. Execution is therefore
|
|
80
|
+
AT-LEAST-ONCE; **completion is exactly-once**. Idempotency of side
|
|
81
|
+
effects is the handler's responsibility (§7).
|
|
82
|
+
- Eligibility order is `run_at, created_at, id` — oldest first,
|
|
83
|
+
deterministic. There are no priority classes (§8).
|
|
84
|
+
|
|
85
|
+
## 4. Retry and dead-lettering
|
|
86
|
+
|
|
87
|
+
A failed attempt (the handler threw or rejected) records
|
|
88
|
+
`last_error` and either:
|
|
89
|
+
|
|
90
|
+
- `attempts < max_attempts` → `state='failed'`,
|
|
91
|
+
`run_at = now + backoff(attempts)` — the retry IS a scheduled job;
|
|
92
|
+
- else → `state='dead'`, terminal. Dead jobs keep their `last_error`
|
|
93
|
+
and their checkpoint rows (a post-mortem can read exactly how far
|
|
94
|
+
the run got).
|
|
95
|
+
|
|
96
|
+
Backoff is exponential with jitter:
|
|
97
|
+
`min(cap, base × 2^(attempts−1)) × (0.5 + random()/2)`, defaults
|
|
98
|
+
`base` 1 000 ms, `cap` 60 000 ms. `Math.random` is the runtime
|
|
99
|
+
default; the `random` option injects a deterministic source and every
|
|
100
|
+
test in this repository does.
|
|
101
|
+
|
|
102
|
+
## 5. Recovery
|
|
103
|
+
|
|
104
|
+
A worker that dies mid-job leaves a `leased` row whose `lease_until`
|
|
105
|
+
passes; the claim statement (§3) treats an expired lease exactly like
|
|
106
|
+
`pending`, so **recovery is not a separate sweeper — it is the next
|
|
107
|
+
claim**. The reclaimed attempt re-runs the handler; a §7 DAG job
|
|
108
|
+
resumes from its checkpoint rows rather than restarting. The lease
|
|
109
|
+
duration (`leaseMs`, default 30 000 ms) is therefore the recovery
|
|
110
|
+
latency ceiling: a handler that legitimately outlives its lease gets
|
|
111
|
+
reclaimed — size `leaseMs` to the slowest honest handler.
|
|
112
|
+
|
|
113
|
+
## 6. Workers and concurrency
|
|
114
|
+
|
|
115
|
+
`createWorker({ handlers, concurrency, pollInterval, leaseMs, owner,
|
|
116
|
+
maxAttempts, backoff, random })` returns `{ start(), stop(),
|
|
117
|
+
stats() }`:
|
|
118
|
+
|
|
119
|
+
- `concurrency` (default 1) independent claim-execute loops share one
|
|
120
|
+
worker registration;
|
|
121
|
+
- an idle loop sleeps `pollInterval` (default 500 ms — at most
|
|
122
|
+
2 claims/s of idle cost per loop, stated). An `enqueue` on the SAME
|
|
123
|
+
store wakes every idle local loop immediately, so same-process
|
|
124
|
+
latency is not poll-bound; **cross-process wake-up is polling**,
|
|
125
|
+
plainly (§8);
|
|
126
|
+
- `stop({ graceMs })` aborts in-flight handlers and resolves once they
|
|
127
|
+
settle **or** the grace period expires (default 5 s), answering
|
|
128
|
+
`{ drained, inFlight }` — see §6.1;
|
|
129
|
+
- `stats()` reports claims, completions, failures, wakes, polls and the
|
|
130
|
+
in-flight handler count.
|
|
131
|
+
|
|
132
|
+
### 6.1 A handler cannot break the loop, and cannot hold shutdown
|
|
133
|
+
|
|
134
|
+
Two invariants a long-running process depends on.
|
|
135
|
+
|
|
136
|
+
**No handler outcome rejects the claim-execute loop.** A handler is host
|
|
137
|
+
code: it may resolve with something JSON cannot express (a `BigInt`, a
|
|
138
|
+
cycle, a throwing `toJSON`) or reject with a value whose own `message`
|
|
139
|
+
throws when read. Both are normalized totally and become an ordinary
|
|
140
|
+
failed attempt — retried with backoff, dead-lettered at `maxAttempts`,
|
|
141
|
+
recorded in `last_error`. The reason this matters more than it looks: a
|
|
142
|
+
rejected loop stops claiming, and a queue that has silently stopped
|
|
143
|
+
draining looks exactly like a queue with nothing to do.
|
|
144
|
+
|
|
145
|
+
**Shutdown is bounded.** Handlers receive an `AbortSignal` alongside the
|
|
146
|
+
job:
|
|
147
|
+
|
|
148
|
+
```js
|
|
149
|
+
handlers: {
|
|
150
|
+
sync: async (payload, { job, signal, checkpointsFor }) => {
|
|
151
|
+
const res = await fetch(url, { signal }); // cancelled on stop()
|
|
152
|
+
…
|
|
153
|
+
},
|
|
154
|
+
}
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
`stop({ graceMs })` aborts the signal, waits up to `graceMs`, and then
|
|
158
|
+
returns `{ drained, inFlight }` regardless. `store.close({ graceMs })`
|
|
159
|
+
does the same and **closes the connection either way**, then reports
|
|
160
|
+
`JD2062` when handlers were left running — a report, not a refusal: the
|
|
161
|
+
handle really is released. A handler that ignores its signal therefore
|
|
162
|
+
cannot hold the database file open for the life of the process, and the
|
|
163
|
+
lease expiry (§5) lets another worker re-claim its job.
|
|
164
|
+
|
|
165
|
+
## 7. The DAG composition
|
|
166
|
+
|
|
167
|
+
```js
|
|
168
|
+
import { compileDag } from '@jarenjs/flow';
|
|
169
|
+
const runner = createDagJobRunner(store, {
|
|
170
|
+
compileDag, // INJECTED — db never imports flow
|
|
171
|
+
documents: { 'sync-report': dagDocument },
|
|
172
|
+
tasks: { fetch: async () => { /* … */ } },
|
|
173
|
+
});
|
|
174
|
+
runner.start();
|
|
175
|
+
await store.jobs.enqueue('sync-report', { input: { day: '2026-08-05' } });
|
|
176
|
+
```
|
|
177
|
+
|
|
178
|
+
- The engine arrives as a capability (the D10 shape applied to flow);
|
|
179
|
+
`@jarenjs/db`'s manifest and import graph name `@jarenjs/flow`
|
|
180
|
+
nowhere, asserted by test.
|
|
181
|
+
- The job id is the run id. Node values save into
|
|
182
|
+
`_jaren_job_checkpoints` as the run progresses; `complete` records
|
|
183
|
+
the DAG result AND marks the job `done` **in one guarded
|
|
184
|
+
statement** — one transaction, so a failure leaves neither, and the
|
|
185
|
+
checkpoint rows of a finished job are pruned in the same breath.
|
|
186
|
+
- A reclaimed DAG job resumes: recorded nodes seed (FLOW-FORMAT
|
|
187
|
+
§7.6), the rest re-run. A task node with side effects MUST thread
|
|
188
|
+
an idempotency key (the job id is the natural one) into whatever it
|
|
189
|
+
touches; the queue cannot make a non-idempotent effect safe, and
|
|
190
|
+
does not pretend to.
|
|
191
|
+
- The document belongs WITH the job kind at worker construction —
|
|
192
|
+
resuming under a different document is undefined (FLOW-FORMAT
|
|
193
|
+
§7.6), so deploys that change a dag document SHOULD drain old jobs
|
|
194
|
+
first or version the kind name.
|
|
195
|
+
|
|
196
|
+
## 8. Non-goals
|
|
197
|
+
|
|
198
|
+
- **One database, one machine.** Leasing coordinates workers on one
|
|
199
|
+
SQLite file. Over a network filesystem (NFS, SMB, many container
|
|
200
|
+
volume mounts) SQLite's locking is **not reliable — this queue is
|
|
201
|
+
NOT a safe cross-machine coordination substrate there**. Same-host
|
|
202
|
+
processes over WAL are the supported topology.
|
|
203
|
+
- No priority classes in 0.1 (`run_at` ordering only), no cron or
|
|
204
|
+
recurring schedules (re-enqueue from a completed handler if
|
|
205
|
+
needed), no workflow-level compensation or sagas, no per-job abort
|
|
206
|
+
signal (the lease is the timeout story), no cross-process push —
|
|
207
|
+
wake-on-write is same-process; everything else polls.
|
|
208
|
+
- Throughput is SQLite's single-writer throughput; the measured
|
|
209
|
+
numbers live in the execution notes, not in marketing.
|
|
210
|
+
|
|
211
|
+
## 9. Errors
|
|
212
|
+
|
|
213
|
+
This format adds NO codes. API misuse (a malformed handler map, a
|
|
214
|
+
non-string kind, a worker started twice) is a `TypeError` at the
|
|
215
|
+
call, matching the capture and live precedents; storage failures ride
|
|
216
|
+
the existing `JD2005` wrap; a job's own failure is DATA — recorded in
|
|
217
|
+
`last_error` and the state machine of §4 — because a queue that
|
|
218
|
+
throws away its failure story has failed twice.
|