@jarenjs/linq 0.73.0 → 0.83.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +8 -10
- package/README.md +15 -1
- package/docs/AI-PEN.md +1 -1
- package/docs/APP-PEN.md +3 -3
- package/docs/CHARTS-PEN.md +1 -1
- package/docs/CONTRACT-PEN.md +4 -4
- package/docs/DB-CLIENT.md +200 -8
- package/docs/FLOW-PEN.md +3 -3
- package/docs/FORMS-PEN.md +2 -2
- package/docs/FORMULA-PEN.md +41 -0
- package/docs/JSLT-PEN.md +4 -4
- package/docs/JTLT-PEN.md +1 -1
- package/docs/LINQ-FORMAT.md +60 -43
- package/docs/MESSAGES-PEN.md +1 -1
- package/docs/MIGRATION-PEN.md +3 -3
- package/docs/MODEL-PEN.md +27 -3
- package/docs/PROJECT-PEN.md +10 -4
- package/docs/QUERY-PEN.md +78 -51
- package/docs/SCHEMA-PEN.md +2 -2
- package/package.json +10 -6
- package/src/db/effects.js +148 -0
- package/src/db/handle.js +2 -0
- package/src/db/index.js +6 -0
- package/src/db/ingest.js +153 -0
- package/src/db/open.js +3 -0
- package/src/db/range.js +261 -0
- package/src/db/receipts.js +151 -0
- package/src/db/records.js +61 -0
- package/src/db/runs.js +164 -0
- package/src/db/search.js +86 -0
- package/src/errors.js +3 -1
- package/src/expression.js +5 -2
- package/src/federate.js +157 -139
- package/src/formula/index.js +15 -0
- package/src/messages/vocabulary.js +2 -1
- package/src/model/define.js +2 -0
- package/src/model/entity.js +15 -0
- package/src/project/index.js +6 -5
- package/types/db.d.ts +101 -0
- package/types/formula.d.ts +9 -0
- package/types/index.d.ts +14 -2
- package/types/message-vocabulary.d.ts +2 -1
- package/types/model.d.ts +18 -0
- package/types/project.d.ts +9 -3
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** Durable external intent/evidence in mapped tables, fenced by the existing job engine. */
|
|
3
|
+
import { canonicalizeJson } from '@jarenjs/json/canonical';
|
|
4
|
+
import { mappedRecords, recordTransaction, copyRecord, refuseRecord } from './records.js';
|
|
5
|
+
|
|
6
|
+
const zero = Object.freeze({ changes: 0, writes: 0, revisions: 0 });
|
|
7
|
+
const terminal = (state) => state === 'confirmed' || state === 'rejected';
|
|
8
|
+
/**
|
|
9
|
+
* Preparation/outbox and each settlement commit locally. Remote delivery never
|
|
10
|
+
* runs inside this adapter. Intent is permanent until evidence resolves it.
|
|
11
|
+
* @param {any} client @param {{ operations: any, maxLegs?: number, maxBytes?: number }} options
|
|
12
|
+
*/
|
|
13
|
+
export function createDbEffectStore(client, options) {
|
|
14
|
+
const records = mappedRecords(client, options?.operations);
|
|
15
|
+
const { maxLegs = 64, maxBytes = 262144 } = options;
|
|
16
|
+
for (const limit of [maxLegs, maxBytes]) if (!Number.isSafeInteger(limit) || limit < 1) throw new TypeError('effect limits must be finite positive integers');
|
|
17
|
+
const inside = (fn) => recordTransaction(client, fn);
|
|
18
|
+
const requireRecord = async (tx, id) => {
|
|
19
|
+
const record = await records.get(tx, id);
|
|
20
|
+
if (!record) throw refuseRecord('unknown external operation');
|
|
21
|
+
return record;
|
|
22
|
+
};
|
|
23
|
+
const guard = async (tx, record, revision, lease) => {
|
|
24
|
+
if (record.revision !== revision || lease?.jobId !== record.plan.jobId) throw refuseRecord('stale operation revision or foreign job');
|
|
25
|
+
await tx.jobs.assertLease(lease);
|
|
26
|
+
};
|
|
27
|
+
const legOf = (record, id) => {
|
|
28
|
+
const leg = record.legs.find((value) => value.id === id);
|
|
29
|
+
if (!leg) throw refuseRecord('unknown operation leg');
|
|
30
|
+
return leg;
|
|
31
|
+
};
|
|
32
|
+
const write = async (tx, record) => {
|
|
33
|
+
record.revision++;
|
|
34
|
+
await records.put(tx, record);
|
|
35
|
+
return { record, changes: 1, writes: 1, revisions: 1 };
|
|
36
|
+
};
|
|
37
|
+
return Object.freeze({
|
|
38
|
+
/** Caller authorization must establish review/compensation authority before preparation.
|
|
39
|
+
* @param {any} plan @param {(tx: any) => any} [prepare] */
|
|
40
|
+
prepare(plan, prepare) {
|
|
41
|
+
const frozen = copyRecord(plan);
|
|
42
|
+
if (['id', 'jobId', 'kind', 'actor', 'reason', 'hashVersion'].some((name) => typeof frozen[name] !== 'string' || !frozen[name])
|
|
43
|
+
|| !Array.isArray(frozen.legs) || !frozen.legs.length || frozen.legs.length > maxLegs
|
|
44
|
+
|| new Set(frozen.legs.map((leg) => leg.id)).size !== frozen.legs.length
|
|
45
|
+
|| frozen.legs.some((leg) => typeof leg.id !== 'string' || !leg.id
|
|
46
|
+
|| !['single-send', 'provider-idempotent'].includes(leg.request?.safety)
|
|
47
|
+
|| !Number.isSafeInteger(leg.maxAttempts) || leg.maxAttempts < 1 || leg.maxAttempts > 100
|
|
48
|
+
|| (leg.request.safety === 'single-send' && leg.maxAttempts !== 1)
|
|
49
|
+
|| (leg.request.safety === 'provider-idempotent' && !leg.request.idempotencyKey))
|
|
50
|
+
|| (frozen.compensationOf !== undefined && frozen.compensationAuthorized !== true)
|
|
51
|
+
|| new TextEncoder().encode(canonicalizeJson(frozen)).byteLength > maxBytes)
|
|
52
|
+
throw new TypeError('effect preparation needs bounded reviewed legs, replay safety, attempt budgets and explicit compensation authority');
|
|
53
|
+
const fingerprint = canonicalizeJson(frozen);
|
|
54
|
+
return inside(async (tx) => {
|
|
55
|
+
const prior = await records.get(tx, frozen.id);
|
|
56
|
+
if (prior) {
|
|
57
|
+
if (prior.fingerprint !== fingerprint) throw refuseRecord('reviewed operation payload changed');
|
|
58
|
+
return { record: prior, ...zero };
|
|
59
|
+
}
|
|
60
|
+
if (await tx.jobs.get(frozen.jobId) !== undefined) throw refuseRecord('operation job identity already belongs to another enqueue');
|
|
61
|
+
if (prepare) await prepare(tx);
|
|
62
|
+
const record = { id: frozen.id, plan: frozen, fingerprint, revision: 1,
|
|
63
|
+
legs: frozen.legs.map((leg) => ({ id: leg.id, state: 'prepared', attempts: 0, evidence: null, intent: null })), decisions: [] };
|
|
64
|
+
await records.put(tx, record, true);
|
|
65
|
+
await tx.jobs.enqueue(frozen.kind, { operationId: frozen.id }, { id: frozen.jobId });
|
|
66
|
+
return { record, changes: 1, writes: 1, revisions: 1 };
|
|
67
|
+
});
|
|
68
|
+
},
|
|
69
|
+
/** @param {string} id */
|
|
70
|
+
get: (id) => inside((tx) => requireRecord(tx, id)),
|
|
71
|
+
/** Persist sending before dispatch. A stale worker cannot start another send.
|
|
72
|
+
* @param {string} id @param {string} legId @param {number} revision @param {any} lease */
|
|
73
|
+
begin(id, legId, revision, lease) {
|
|
74
|
+
return inside(async (tx) => {
|
|
75
|
+
const record = await requireRecord(tx, id);
|
|
76
|
+
await guard(tx, record, revision, lease);
|
|
77
|
+
const leg = legOf(record, legId), plan = record.plan.legs.find((value) => value.id === legId);
|
|
78
|
+
if (leg.state !== 'prepared' && leg.state !== 'retry-approved') return { state: leg.state, record, ...zero };
|
|
79
|
+
if (leg.attempts >= plan.maxAttempts) return { state: 'exhausted', record, ...zero };
|
|
80
|
+
leg.state = 'sending';
|
|
81
|
+
leg.attempts++;
|
|
82
|
+
leg.intent = { generation: lease.generation, attempt: leg.attempts };
|
|
83
|
+
return { state: 'sending', ...(await write(tx, record)) };
|
|
84
|
+
});
|
|
85
|
+
},
|
|
86
|
+
/** Only the active attempt may persist transport evidence.
|
|
87
|
+
* @param {string} id @param {string} legId @param {number} revision @param {any} lease @param {any} outcome */
|
|
88
|
+
settle(id, legId, revision, lease, outcome) {
|
|
89
|
+
const observation = copyRecord(outcome);
|
|
90
|
+
if (!['confirmed', 'rejected', 'unresolved'].includes(observation.state)
|
|
91
|
+
|| !Object.hasOwn(observation, 'evidence') || new TextEncoder().encode(canonicalizeJson(observation)).byteLength > maxBytes)
|
|
92
|
+
throw refuseRecord('external settlement needs bounded confirmed/rejected/unresolved evidence');
|
|
93
|
+
return inside(async (tx) => {
|
|
94
|
+
const record = await requireRecord(tx, id);
|
|
95
|
+
await guard(tx, record, revision, lease);
|
|
96
|
+
const leg = legOf(record, legId);
|
|
97
|
+
if (leg.state !== 'sending' || leg.intent.generation !== lease.generation) throw refuseRecord('no sending intent for this attempt');
|
|
98
|
+
leg.state = observation.state;
|
|
99
|
+
leg.evidence = observation.evidence;
|
|
100
|
+
return write(tx, record);
|
|
101
|
+
});
|
|
102
|
+
},
|
|
103
|
+
/** Lost workers leave uncertainty, never proof of non-application.
|
|
104
|
+
* @param {string} id @param {number} revision @param {any} lease */
|
|
105
|
+
recover(id, revision, lease) {
|
|
106
|
+
return inside(async (tx) => {
|
|
107
|
+
const record = await requireRecord(tx, id);
|
|
108
|
+
await guard(tx, record, revision, lease);
|
|
109
|
+
let changed = false;
|
|
110
|
+
for (const leg of record.legs) if (leg.state === 'sending') { leg.state = 'unresolved'; changed = true; }
|
|
111
|
+
return changed ? write(tx, record) : { record, ...zero };
|
|
112
|
+
});
|
|
113
|
+
},
|
|
114
|
+
/** Explicit read-back or operator decisions; absence is evidence only under
|
|
115
|
+
* the declared authoritative non-application guarantee. Idempotent retries
|
|
116
|
+
* retain the reviewed request/key and the original durable attempt budget.
|
|
117
|
+
* @param {string} id @param {string} legId @param {number} revision @param {any} lease @param {any} decision */
|
|
118
|
+
reconcile(id, legId, revision, lease, decision) {
|
|
119
|
+
const proof = copyRecord(decision);
|
|
120
|
+
if (['id', 'actor', 'reason'].some((key) => typeof proof[key] !== 'string' || !proof[key])
|
|
121
|
+
|| !['confirm', 'reject', 'retry'].includes(proof.action) || !Object.hasOwn(proof, 'evidence')) throw refuseRecord('reconciliation needs actor/reason and evidence');
|
|
122
|
+
return inside(async (tx) => {
|
|
123
|
+
const record = await requireRecord(tx, id);
|
|
124
|
+
if (lease?.jobId !== record.plan.jobId) throw refuseRecord('foreign reconciliation job');
|
|
125
|
+
await tx.jobs.assertLease(lease);
|
|
126
|
+
const entry = { ...proof, legId };
|
|
127
|
+
const prior = record.decisions.find((value) => value.id === proof.id);
|
|
128
|
+
if (prior) {
|
|
129
|
+
if (canonicalizeJson(prior) !== canonicalizeJson(entry)) throw refuseRecord('reconciliation decision collision');
|
|
130
|
+
return { record, ...zero };
|
|
131
|
+
}
|
|
132
|
+
await guard(tx, record, revision, lease);
|
|
133
|
+
const leg = legOf(record, legId), plan = record.plan.legs.find((value) => value.id === legId);
|
|
134
|
+
if (terminal(leg.state) || !['sending', 'unresolved'].includes(leg.state)) throw refuseRecord('only unresolved intent can be reconciled');
|
|
135
|
+
if (proof.action === 'retry' && (plan.request.safety !== 'provider-idempotent' && proof.guarantee !== 'authoritative-non-application'))
|
|
136
|
+
throw refuseRecord('absence is not proof of non-application');
|
|
137
|
+
// Single-send remains single-send: an authoritative negative result can
|
|
138
|
+
// be recorded as rejected, followed by a separately reviewed operation.
|
|
139
|
+
if (proof.action === 'retry' && leg.attempts >= plan.maxAttempts) throw refuseRecord('durable attempt budget exhausted');
|
|
140
|
+
if (record.decisions.length >= 128 || new TextEncoder().encode(canonicalizeJson(entry)).byteLength > maxBytes) throw refuseRecord('reconciliation evidence limit');
|
|
141
|
+
leg.state = proof.action === 'confirm' ? 'confirmed' : proof.action === 'reject' ? 'rejected' : 'retry-approved';
|
|
142
|
+
leg.evidence = proof.evidence;
|
|
143
|
+
record.decisions.push(entry);
|
|
144
|
+
return write(tx, record);
|
|
145
|
+
});
|
|
146
|
+
},
|
|
147
|
+
});
|
|
148
|
+
}
|
package/src/db/handle.js
CHANGED
|
@@ -18,6 +18,7 @@ import { fromAsync, AsyncSequence } from '../async.js';
|
|
|
18
18
|
import { Graph } from './include.js';
|
|
19
19
|
import { requireMembership } from './membership.js';
|
|
20
20
|
import { registerLive } from './live.js';
|
|
21
|
+
import { createDbRangeProvider } from './range.js';
|
|
21
22
|
|
|
22
23
|
/** The chain surface, read once from the class: every public operator
|
|
23
24
|
* and terminal, `explain` set aside for its overload. */
|
|
@@ -59,6 +60,7 @@ export function createEntityHandle(store, name) {
|
|
|
59
60
|
// the graph with nothing included: the root clauses, the keyset and
|
|
60
61
|
// the page over the rows alone
|
|
61
62
|
members.graph = () => new Graph(set, name);
|
|
63
|
+
members.range = (spec, options) => createDbRangeProvider(store, name, spec, options);
|
|
62
64
|
members.link = (own, member, target) => {
|
|
63
65
|
requireMembership(set.relations, name, member, 'link');
|
|
64
66
|
set.link(own, member, target);
|
package/src/db/index.js
CHANGED
|
@@ -22,4 +22,10 @@
|
|
|
22
22
|
|
|
23
23
|
export { open, defaultValidator } from './open.js';
|
|
24
24
|
export { createDbLedger } from './ledger.js';
|
|
25
|
+
export { createDbIngestionStore } from './ingest.js';
|
|
26
|
+
export { createDbRangeProvider } from './range.js';
|
|
27
|
+
export { createLexicalRangeProvider } from './search.js';
|
|
25
28
|
export { defineReplication } from './replication.js';
|
|
29
|
+
export { createDbReceipts } from './receipts.js';
|
|
30
|
+
export { createDbEffectStore } from './effects.js';
|
|
31
|
+
export { createDbRunStore } from './runs.js';
|
package/src/db/ingest.js
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** Atomic page/checkpoint staging and complete-generation publication. */
|
|
3
|
+
import { canonicalizeJson } from '@jarenjs/json/canonical';
|
|
4
|
+
import { isThenable } from '@jarenjs/core/function';
|
|
5
|
+
|
|
6
|
+
const copy = (value) => JSON.parse(canonicalizeJson(value));
|
|
7
|
+
const key = (...parts) => canonicalizeJson(parts);
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Adapt application-declared collections under the client's transaction owner.
|
|
11
|
+
* Reconcile is a synchronous pure policy; network I/O belongs outside this
|
|
12
|
+
* adapter. No table, schema, credentials or provider field policy is inferred.
|
|
13
|
+
* @param {any} client
|
|
14
|
+
* @param {{ staging: string, checkpoints: string, publications: string, facts?: string,
|
|
15
|
+
* reconcile?: (existing: any, incoming: any, evidence: any) => any }} options
|
|
16
|
+
*/
|
|
17
|
+
export function createDbIngestionStore(client, options) {
|
|
18
|
+
if (!client || typeof client.transaction !== 'function' || !client.collections || !options)
|
|
19
|
+
throw new TypeError('ingestion store needs a client and declared collection names');
|
|
20
|
+
const names = ['staging', 'checkpoints', 'publications', ...(options.facts ? ['facts'] : [])];
|
|
21
|
+
if (new Set(names.map((name) => options[name])).size !== names.length
|
|
22
|
+
|| names.some((name) => typeof options[name] !== 'string' || !Object.hasOwn(client.collections, options[name])))
|
|
23
|
+
throw new TypeError('ingestion collections must be distinct declared collections');
|
|
24
|
+
if (options.reconcile !== undefined && typeof options.reconcile !== 'function') throw new TypeError('reconcile must be a pure function');
|
|
25
|
+
const reconcile = options.reconcile ?? ((_existing, incoming) => incoming);
|
|
26
|
+
const inside = (fn) => typeof client.close === 'function' ? client.transaction(fn, { mode: 'immediate' }) : client.transaction(fn);
|
|
27
|
+
const fingerprint = (plan) => key(plan.source, plan.version, plan.partitions, plan.input, plan.policyRevision, plan.consistency);
|
|
28
|
+
const checkpointId = (plan) => key(plan.source, plan.generation);
|
|
29
|
+
const publications = (tx) => tx.collections[options.publications];
|
|
30
|
+
const checkpoints = (tx) => tx.collections[options.checkpoints];
|
|
31
|
+
const zero = { changes: 0, writes: 0, revisions: 0 };
|
|
32
|
+
const matching = async (tx, plan) => {
|
|
33
|
+
const checkpoint = await checkpoints(tx).get(checkpointId(plan));
|
|
34
|
+
if (!checkpoint || checkpoint.fingerprint !== fingerprint(plan)) throw new TypeError('ingestion checkpoint does not match the plan');
|
|
35
|
+
return checkpoint;
|
|
36
|
+
};
|
|
37
|
+
return Object.freeze({
|
|
38
|
+
/** @param {any} plan */
|
|
39
|
+
begin(plan) {
|
|
40
|
+
return inside(async (tx) => {
|
|
41
|
+
const published = await publications(tx).get(plan.source);
|
|
42
|
+
if (published?.fingerprint === fingerprint(plan)) return { state: 'unchanged', ...zero, manifest: published };
|
|
43
|
+
const id = checkpointId(plan);
|
|
44
|
+
const existing = await checkpoints(tx).get(id);
|
|
45
|
+
if (existing) {
|
|
46
|
+
if (existing.fingerprint !== fingerprint(plan)) return { state: 'refused', reason: 'generation-mismatch' };
|
|
47
|
+
if (existing.status !== 'staging') return { state: 'refused', reason: existing.status };
|
|
48
|
+
return { state: 'staging', checkpoint: existing, ...zero };
|
|
49
|
+
}
|
|
50
|
+
const checkpoint = { id, fingerprint: fingerprint(plan), source: plan.source, generation: plan.generation,
|
|
51
|
+
version: plan.version, expectedPublication: published?.generation ?? null, status: 'staging',
|
|
52
|
+
partitions: plan.partitions.map((partition) => ({ id: partition, cursor: null, complete: false, pages: [] })),
|
|
53
|
+
pages: 0, rows: 0, bytes: 0 };
|
|
54
|
+
await checkpoints(tx).insert(checkpoint);
|
|
55
|
+
return { state: 'staging', checkpoint, ...zero };
|
|
56
|
+
});
|
|
57
|
+
},
|
|
58
|
+
/** Page bytes and its continuation/completion evidence commit together.
|
|
59
|
+
* @param {any} plan @param {string} partition @param {any} page */
|
|
60
|
+
stage(plan, partition, page) {
|
|
61
|
+
return inside(async (tx) => {
|
|
62
|
+
const checkpoint = await matching(tx, plan);
|
|
63
|
+
if (checkpoint.status !== 'staging') return { state: 'refused', reason: checkpoint.status };
|
|
64
|
+
const part = checkpoint.partitions.find((p) => p.id === partition);
|
|
65
|
+
if (!part) return { state: 'refused', reason: 'unknown-partition' };
|
|
66
|
+
const id = key(plan.source, plan.generation, partition, page.cursor);
|
|
67
|
+
const staging = tx.collections[options.staging];
|
|
68
|
+
const existing = await staging.get(id);
|
|
69
|
+
const record = copy({ id, partition, generation: plan.generation, source: plan.source, page });
|
|
70
|
+
if (existing) {
|
|
71
|
+
if (canonicalizeJson(existing) !== canonicalizeJson(record)) return { state: 'refused', reason: 'page-conflict' };
|
|
72
|
+
return { state: 'staged', checkpoint, ...zero };
|
|
73
|
+
}
|
|
74
|
+
if (part.complete || canonicalizeJson(part.cursor) !== canonicalizeJson(page.cursor)) return { state: 'refused', reason: 'stale-cursor' };
|
|
75
|
+
await staging.insert(record);
|
|
76
|
+
part.pages.push(id);
|
|
77
|
+
part.cursor = page.continuation;
|
|
78
|
+
part.complete = page.complete === true && page.reason === null && page.version === plan.version;
|
|
79
|
+
checkpoint.pages++;
|
|
80
|
+
checkpoint.rows += page.rows.length;
|
|
81
|
+
checkpoint.bytes += page.bytes;
|
|
82
|
+
if (page.reason !== null) checkpoint.status = 'incomplete';
|
|
83
|
+
if (page.version !== plan.version) checkpoint.status = 'source-changed';
|
|
84
|
+
await checkpoints(tx).put(checkpoint, checkpoint.id);
|
|
85
|
+
return { state: 'staged', checkpoint, ...zero };
|
|
86
|
+
});
|
|
87
|
+
},
|
|
88
|
+
/** Preserve partial observations but fence the generation from publication.
|
|
89
|
+
* @param {any} plan @param {string} reason */
|
|
90
|
+
invalidate(plan, reason) {
|
|
91
|
+
return inside(async (tx) => {
|
|
92
|
+
const checkpoint = await matching(tx, plan);
|
|
93
|
+
if (checkpoint.status === 'published' || checkpoint.status === reason) return;
|
|
94
|
+
await checkpoints(tx).put({ ...checkpoint, status: reason }, checkpoint.id);
|
|
95
|
+
});
|
|
96
|
+
},
|
|
97
|
+
/** Completion evidence, effective facts and the pointer share one commit.
|
|
98
|
+
* @param {any} plan @param {any} evidence @param {{ signal?: AbortSignal }} [context] */
|
|
99
|
+
publish(plan, evidence, context = {}) {
|
|
100
|
+
return inside(async (tx) => {
|
|
101
|
+
if (context.signal?.aborted) return { state: 'refused', reason: 'cancelled' };
|
|
102
|
+
const checkpoint = await matching(tx, plan);
|
|
103
|
+
const published = await publications(tx).get(plan.source);
|
|
104
|
+
if (published?.fingerprint === fingerprint(plan)) return { state: 'unchanged', ...zero, manifest: published };
|
|
105
|
+
if (checkpoint.status !== 'staging' || checkpoint.partitions.some((part) => !part.complete)) return { state: 'refused', reason: 'incomplete' };
|
|
106
|
+
if ((published?.generation ?? null) !== checkpoint.expectedPublication) return { state: 'refused', reason: 'publication-conflict' };
|
|
107
|
+
if (!evidence || evidence.version !== plan.version || evidence.consistency !== plan.consistency)
|
|
108
|
+
return { state: 'refused', reason: 'source-changed' };
|
|
109
|
+
let changes = 0;
|
|
110
|
+
const seen = new Set();
|
|
111
|
+
for (const part of checkpoint.partitions) for (const id of part.pages) {
|
|
112
|
+
const record = await tx.collections[options.staging].get(id);
|
|
113
|
+
if (!record || record.page.version !== plan.version || record.page.reason !== null) throw new TypeError('publication lacks page evidence');
|
|
114
|
+
for (let i = 0; i < record.page.rows.length; i++) {
|
|
115
|
+
const providerId = record.page.ids[i];
|
|
116
|
+
const factId = key(plan.source, part.id, providerId);
|
|
117
|
+
if (seen.has(factId)) throw new TypeError('publication has duplicate source identities');
|
|
118
|
+
seen.add(factId);
|
|
119
|
+
if (!options.facts) continue;
|
|
120
|
+
const facts = tx.collections[options.facts];
|
|
121
|
+
const prior = await facts.get(factId);
|
|
122
|
+
const value = reconcile(prior?.value ?? null, copy(record.page.rows[i]), { source: plan.source, partition: part.id, providerId, version: plan.version });
|
|
123
|
+
if (isThenable(value)) throw new TypeError('reconcile must be synchronous; network I/O cannot hold a page transaction');
|
|
124
|
+
if (context.signal?.aborted) throw new TypeError('publication cancelled');
|
|
125
|
+
if (prior && canonicalizeJson(prior.value) === canonicalizeJson(value)) continue;
|
|
126
|
+
await facts.put({ id: factId, source: plan.source, partition: part.id, providerId,
|
|
127
|
+
value: copy(value), revision: (prior?.revision ?? 0) + 1 }, factId);
|
|
128
|
+
changes++;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
const manifest = { id: plan.source, source: plan.source, generation: plan.generation, version: plan.version,
|
|
132
|
+
fingerprint: fingerprint(plan), partitions: copy(checkpoint.partitions), evidence: copy(evidence),
|
|
133
|
+
pages: checkpoint.pages, rows: checkpoint.rows, bytes: checkpoint.bytes };
|
|
134
|
+
if (context.signal?.aborted) throw new TypeError('publication cancelled');
|
|
135
|
+
await publications(tx).put(manifest, plan.source);
|
|
136
|
+
await checkpoints(tx).put({ ...checkpoint, status: 'published' }, checkpoint.id);
|
|
137
|
+
if (context.signal?.aborted) throw new TypeError('publication cancelled');
|
|
138
|
+
return { state: 'published', changes, writes: changes, revisions: changes, manifest };
|
|
139
|
+
});
|
|
140
|
+
},
|
|
141
|
+
/** @param {string} source */
|
|
142
|
+
current: (source) => client.collections[options.publications].get(source),
|
|
143
|
+
/** @param {any} plan */
|
|
144
|
+
inspect(plan) {
|
|
145
|
+
return inside(async (tx) => {
|
|
146
|
+
const checkpoint = await matching(tx, plan);
|
|
147
|
+
const observations = [];
|
|
148
|
+
for (const part of checkpoint.partitions) for (const id of part.pages) observations.push(await tx.collections[options.staging].get(id));
|
|
149
|
+
return { checkpoint, observations };
|
|
150
|
+
});
|
|
151
|
+
},
|
|
152
|
+
});
|
|
153
|
+
}
|
package/src/db/open.js
CHANGED
|
@@ -91,6 +91,9 @@ export async function open(model, options) {
|
|
|
91
91
|
const inner = {
|
|
92
92
|
store: tx,
|
|
93
93
|
capabilities: tx.capabilities,
|
|
94
|
+
sql: tx.sql,
|
|
95
|
+
jobs: tx.jobs,
|
|
96
|
+
sync: tx.sync,
|
|
94
97
|
...handlesOf(tx),
|
|
95
98
|
transaction: (fn) => tx.transaction((nested) => fn(transactionClient(nested))),
|
|
96
99
|
// the named-savepoint group (MODEL-FORMAT §5.2), forwarded as it
|
package/src/db/range.js
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
//@ts-check
|
|
2
|
+
/** Structural ranges over the store's cursors, keyset pages and committed capture. */
|
|
3
|
+
import { canonicalizeJson } from '@jarenjs/json/canonical';
|
|
4
|
+
import { resolveRuntime } from '@jarenjs/core/runtime';
|
|
5
|
+
import { deepFreeze } from '@jarenjs/core/object';
|
|
6
|
+
|
|
7
|
+
/** Open a bounded source. Resident mode obtains a complete, bounded source
|
|
8
|
+
* snapshot before offering index seeks or an exact total. Sequential mode
|
|
9
|
+
* admits only first pages and continuations through the store's keyset pager.
|
|
10
|
+
* @param {any} store @param {string} entity @param {any} [spec]
|
|
11
|
+
* @param {any} [options] @returns {Promise<any>} */
|
|
12
|
+
export async function createDbRangeProvider(store, entity, spec = {}, options = {}) {
|
|
13
|
+
spec = structuredClone(spec);
|
|
14
|
+
const keys = options.keys?.slice();
|
|
15
|
+
if (!Array.isArray(keys) || !keys.length || keys.some((key) => typeof key !== 'string') || new Set(keys).size !== keys.length)
|
|
16
|
+
throw new TypeError('range provider needs distinct logical key members');
|
|
17
|
+
if (spec.take !== undefined || spec.skip !== undefined || spec.after !== undefined || spec.include !== undefined)
|
|
18
|
+
throw new TypeError('range provider owns the window and serves root rows');
|
|
19
|
+
const bounds = { rows: options.maxRows ?? 256, bytes: options.maxBytes ?? 262144,
|
|
20
|
+
pages: options.maxPages ?? 4, inFlight: options.maxInFlight ?? 2, subscriptions: options.maxSubscriptions ?? 8 };
|
|
21
|
+
if (!Object.values(bounds).every((n) => Number.isSafeInteger(n) && n > 0 && n < Number.MAX_SAFE_INTEGER))
|
|
22
|
+
throw new TypeError('range provider bounds must be finite positive safe integers');
|
|
23
|
+
const resident = options.resident === true;
|
|
24
|
+
const profile = options.profile === undefined ? undefined : structuredClone(options.profile);
|
|
25
|
+
const host = resolveRuntime(options.runtime);
|
|
26
|
+
const source = options.source ?? host.uuid();
|
|
27
|
+
const query = options.query ?? canonicalizeJson({ source, entity, spec, keys,
|
|
28
|
+
schema: options.schemaVersion ?? null, profile: profile ?? null });
|
|
29
|
+
if (typeof source !== 'string' || !source || typeof query !== 'string' || !query)
|
|
30
|
+
throw new TypeError('range source and query identities must be nonempty strings');
|
|
31
|
+
const encoder = new TextEncoder();
|
|
32
|
+
const bytesOf = (value) => encoder.encode(JSON.stringify(value)).byteLength;
|
|
33
|
+
const keyOf = (row) => {
|
|
34
|
+
const values = keys.map((key) => row[key]);
|
|
35
|
+
if (values.some((value) => typeof value !== 'string' && !(typeof value === 'number' && Number.isFinite(value))))
|
|
36
|
+
throw new TypeError('a range row must carry every stable key member');
|
|
37
|
+
return keys.length === 1 ? String(values[0]) : canonicalizeJson(values);
|
|
38
|
+
};
|
|
39
|
+
let revision = 1;
|
|
40
|
+
let dataVersion;
|
|
41
|
+
let disposed = false;
|
|
42
|
+
let generation = -1;
|
|
43
|
+
let ticket = 0;
|
|
44
|
+
let rows = null;
|
|
45
|
+
let residentBytes = 0;
|
|
46
|
+
const continuations = new Map();
|
|
47
|
+
const pending = new Map();
|
|
48
|
+
const exports = new Set();
|
|
49
|
+
const observers = new Set();
|
|
50
|
+
const stats = { sourceReads: 0, sourceRows: 0, sourceBytes: 0 };
|
|
51
|
+
const snapshot = () => `${source}-v${revision}`;
|
|
52
|
+
const invalidate = (reason) => {
|
|
53
|
+
revision++;
|
|
54
|
+
rows = null; residentBytes = 0; continuations.clear();
|
|
55
|
+
const event = Object.freeze({ type: 'reset', reason, query, snapshot: snapshot(), revision,
|
|
56
|
+
capture: 'committed-store-records; external changes detected on request' });
|
|
57
|
+
for (const observer of observers) {
|
|
58
|
+
try { observer(event); }
|
|
59
|
+
catch { /* A subscriber cannot suppress a sibling's source reset. */ }
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
const unsubscribe = store.observe((record) => {
|
|
63
|
+
if (!disposed && record.collections.includes(entity)) invalidate('source-changed');
|
|
64
|
+
});
|
|
65
|
+
const checkVersion = async (over) => {
|
|
66
|
+
const current = await over.dataVersion();
|
|
67
|
+
if (dataVersion !== undefined && current !== dataVersion) invalidate('external-source-changed');
|
|
68
|
+
dataVersion = current;
|
|
69
|
+
};
|
|
70
|
+
const readPage = async (over, pageOptions, used) => {
|
|
71
|
+
stats.sourceReads++;
|
|
72
|
+
const record = (work) => {
|
|
73
|
+
if (work === undefined) return;
|
|
74
|
+
stats.sourceRows += work.rows; stats.sourceBytes += work.bytes;
|
|
75
|
+
if (used !== undefined) used.work += work.rows;
|
|
76
|
+
};
|
|
77
|
+
try {
|
|
78
|
+
const page = await over.entity(entity).page(spec, {
|
|
79
|
+
...pageOptions, lookahead: false, profile,
|
|
80
|
+
});
|
|
81
|
+
record(page.work);
|
|
82
|
+
return page;
|
|
83
|
+
}
|
|
84
|
+
catch (error) { record(error?.work); throw error; }
|
|
85
|
+
};
|
|
86
|
+
const loadResident = async (over, signal, used) => {
|
|
87
|
+
const page = await readPage(over, { limit: bounds.rows + 1, maxBytes: bounds.bytes, signal }, used);
|
|
88
|
+
const loaded = page.items;
|
|
89
|
+
const bytes = bytesOf(loaded);
|
|
90
|
+
if (page.hasMore !== false || loaded.length > bounds.rows || bytes > bounds.bytes) {
|
|
91
|
+
throw new RangeError('range source exceeds resident credits');
|
|
92
|
+
}
|
|
93
|
+
const identities = loaded.map(keyOf);
|
|
94
|
+
if (new Set(identities).size !== identities.length) throw new TypeError('range keys must be unique');
|
|
95
|
+
loaded.forEach(deepFreeze);
|
|
96
|
+
rows = loaded; residentBytes = bytes;
|
|
97
|
+
};
|
|
98
|
+
try {
|
|
99
|
+
await store.transaction(async (tx) => {
|
|
100
|
+
await checkVersion(tx);
|
|
101
|
+
if (resident) await loadResident(tx);
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
catch (error) { unsubscribe(); throw error; }
|
|
105
|
+
|
|
106
|
+
const capabilities = Object.freeze({ seekIndex: resident && options.seekIndex !== false, seekKey: false,
|
|
107
|
+
continuation: true, live: true, exactTotal: resident && options.exactTotal !== false, completeExport: resident });
|
|
108
|
+
const provider = {
|
|
109
|
+
capabilities,
|
|
110
|
+
get query() { return query; },
|
|
111
|
+
get snapshot() { return snapshot(); },
|
|
112
|
+
stats: () => ({ ...stats, pending: pending.size, pages: continuations.size,
|
|
113
|
+
rows: rows?.length ?? 0, bytes: residentBytes, subscriptions: observers.size, exports: exports.size, disposed }),
|
|
114
|
+
subscribe(observer) {
|
|
115
|
+
if (disposed) throw new Error('range provider is disposed');
|
|
116
|
+
if (typeof observer !== 'function') throw new TypeError('range subscriber must be callable');
|
|
117
|
+
if (observers.size >= bounds.subscriptions) throw new RangeError('range subscription bound reached');
|
|
118
|
+
observers.add(observer);
|
|
119
|
+
return () => observers.delete(observer);
|
|
120
|
+
},
|
|
121
|
+
request(request, signal) {
|
|
122
|
+
request = structuredClone(request);
|
|
123
|
+
const identity = Object.fromEntries(['generation', 'requestId', 'query', 'snapshot'].map((key) => [key, request?.[key]]));
|
|
124
|
+
const used = { pages: 0, rows: 0, bytes: 0, work: 0 };
|
|
125
|
+
const refuse = (state, reason) => ({ ...identity, state, reason, used: { ...used } });
|
|
126
|
+
if (disposed) return Promise.resolve(refuse('error', 'disposed'));
|
|
127
|
+
if (signal?.aborted) return Promise.resolve(refuse('error', 'cancelled'));
|
|
128
|
+
if (!Number.isSafeInteger(request?.generation) || request.generation < 0 || typeof request.requestId !== 'string')
|
|
129
|
+
return Promise.resolve(refuse('error', 'invalid-identity'));
|
|
130
|
+
if (request.query !== query || request.generation < generation) return Promise.resolve(refuse('invalidated', 'query-changed'));
|
|
131
|
+
if (pending.size >= bounds.inFlight) return Promise.resolve(refuse('budget-exhausted', 'in-flight'));
|
|
132
|
+
generation = request.generation;
|
|
133
|
+
const currentTicket = ++ticket;
|
|
134
|
+
const controller = new AbortController();
|
|
135
|
+
const abort = () => controller.abort(signal.reason);
|
|
136
|
+
signal?.addEventListener('abort', abort, { once: true });
|
|
137
|
+
const run = async () => {
|
|
138
|
+
await Promise.resolve();
|
|
139
|
+
if (disposed) return refuse('error', 'disposed');
|
|
140
|
+
if (controller.signal.aborted) return refuse('error', 'cancelled');
|
|
141
|
+
const credits = request.credits;
|
|
142
|
+
if (!['pages', 'rows', 'bytes', 'work'].every((key) => Number.isSafeInteger(credits?.[key]) && credits[key] >= 0))
|
|
143
|
+
return refuse('error', 'invalid-credits');
|
|
144
|
+
let start;
|
|
145
|
+
let limit;
|
|
146
|
+
let after;
|
|
147
|
+
if (request.continuation != null) {
|
|
148
|
+
const cursor = continuations.get(request.continuation);
|
|
149
|
+
if (!cursor || cursor.snapshot !== request.snapshot || cursor.query !== request.query)
|
|
150
|
+
return refuse('invalidated', 'continuation-changed');
|
|
151
|
+
start = cursor.end; limit = cursor.limit; after = cursor.after;
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
const range = request.range;
|
|
155
|
+
if (!range || !Number.isSafeInteger(range.start) || !Number.isSafeInteger(range.end) || range.start < 0 || range.end < range.start)
|
|
156
|
+
return refuse('error', 'invalid-range');
|
|
157
|
+
start = range.start; limit = range.end - range.start;
|
|
158
|
+
if (start > 0 && !capabilities.seekIndex) return refuse('error', 'unsupported-seek');
|
|
159
|
+
}
|
|
160
|
+
if (credits.pages < 1 || credits.rows < limit || credits.work < limit || credits.bytes < 2
|
|
161
|
+
|| limit > bounds.rows || credits.bytes > Number.MAX_SAFE_INTEGER)
|
|
162
|
+
return refuse('budget-exhausted', 'credits');
|
|
163
|
+
try {
|
|
164
|
+
let page;
|
|
165
|
+
const wantedRevision = revision;
|
|
166
|
+
const result = await store.transaction(async (tx) => {
|
|
167
|
+
await checkVersion(tx);
|
|
168
|
+
if (request.snapshot !== snapshot()) return null;
|
|
169
|
+
if (resident && rows === null) {
|
|
170
|
+
// A reset rebuilds the whole bounded source. Admission reserves
|
|
171
|
+
// its maximum root pulls separately from the requested slice.
|
|
172
|
+
if (credits.work - limit < bounds.rows + 1) throw new RangeError('resident refresh exceeds work credits');
|
|
173
|
+
await loadResident(tx, controller.signal, used);
|
|
174
|
+
}
|
|
175
|
+
if (resident) {
|
|
176
|
+
const result = rows.slice(start, start + limit);
|
|
177
|
+
used.work += result.length;
|
|
178
|
+
return result;
|
|
179
|
+
}
|
|
180
|
+
if (limit === 0) return [];
|
|
181
|
+
// The store counts row payloads; a range also owes array brackets
|
|
182
|
+
// and commas. Reserve their maximum before any row is pulled.
|
|
183
|
+
const maxBytes = Math.min(credits.bytes, bounds.bytes) - limit - 1;
|
|
184
|
+
if (maxBytes < 1) throw new RangeError('range array exceeds byte credits');
|
|
185
|
+
page = await readPage(tx, { limit, after, maxBytes, signal: controller.signal }, used);
|
|
186
|
+
return page.items;
|
|
187
|
+
}, { signal: controller.signal });
|
|
188
|
+
if (disposed) return refuse('error', 'disposed');
|
|
189
|
+
if (controller.signal.aborted) return refuse('error', 'cancelled');
|
|
190
|
+
if (currentTicket !== ticket || request.generation !== generation) return refuse('invalidated', 'superseded');
|
|
191
|
+
if (result === null || wantedRevision !== revision || request.snapshot !== snapshot()) return refuse('invalidated', 'snapshot-changed');
|
|
192
|
+
used.pages = 1;
|
|
193
|
+
const bytes = bytesOf(result);
|
|
194
|
+
if (bytes > Math.min(credits.bytes, bounds.bytes)) return refuse('budget-exhausted', 'bytes');
|
|
195
|
+
result.forEach(deepFreeze);
|
|
196
|
+
const resultKeys = result.map(keyOf);
|
|
197
|
+
if (new Set(resultKeys).size !== resultKeys.length) return refuse('error', 'duplicate-key');
|
|
198
|
+
used.rows = result.length; used.bytes = bytes;
|
|
199
|
+
let continuation = null;
|
|
200
|
+
const more = resident ? start + result.length < rows.length : page?.hasMore !== false && result.length > 0;
|
|
201
|
+
if (more) {
|
|
202
|
+
continuation = host.uuid();
|
|
203
|
+
if (continuations.size >= bounds.pages) continuations.delete(continuations.keys().next().value);
|
|
204
|
+
continuations.set(continuation, { query, snapshot: snapshot(), end: start + result.length, limit, after: page?.continuation });
|
|
205
|
+
}
|
|
206
|
+
return { ...identity, state: 'ready', rows: result, keys: resultKeys, used,
|
|
207
|
+
continuation, total: capabilities.exactTotal ? { kind: 'known', value: rows.length } : { kind: 'unknown' } };
|
|
208
|
+
}
|
|
209
|
+
catch (error) {
|
|
210
|
+
return refuse(disposed ? 'error' : controller.signal.aborted ? 'error'
|
|
211
|
+
: error instanceof RangeError || ['JD2007', 'JD2073', 'JD2074', 'JD2076'].includes(error?.code) ? 'budget-exhausted' : 'error',
|
|
212
|
+
disposed ? 'disposed' : controller.signal.aborted ? 'cancelled' : error?.code ?? error.message);
|
|
213
|
+
}
|
|
214
|
+
};
|
|
215
|
+
const promise = run().finally(() => { pending.delete(controller); signal?.removeEventListener('abort', abort); });
|
|
216
|
+
pending.set(controller, promise);
|
|
217
|
+
return promise;
|
|
218
|
+
},
|
|
219
|
+
export(request, signal) {
|
|
220
|
+
if (disposed) throw new Error('disposed');
|
|
221
|
+
if (exports.size >= bounds.inFlight) throw new RangeError('Export credits');
|
|
222
|
+
const pageRows = request.pageRows ?? 64, pageBytes = request.pageBytes ?? bounds.bytes;
|
|
223
|
+
if (!resident) throw new Error('unsupported-export');
|
|
224
|
+
if (!Number.isSafeInteger(pageRows) || pageRows <= 0 || pageRows > bounds.rows
|
|
225
|
+
|| !Number.isSafeInteger(pageBytes) || pageBytes < 2) throw new RangeError('Invalid export credits');
|
|
226
|
+
const iterator = (async function* () {
|
|
227
|
+
try {
|
|
228
|
+
// Verify external changes before certifying the bounded resident source epoch.
|
|
229
|
+
await store.transaction(checkVersion);
|
|
230
|
+
if (!rows) throw new Error('Snapshot must be reloaded before export');
|
|
231
|
+
const total = rows.length;
|
|
232
|
+
for (let start = 0; start < total; start += pageRows) {
|
|
233
|
+
if (disposed || signal?.aborted || request.query !== query || request.snapshot !== snapshot() || !rows)
|
|
234
|
+
throw new Error('Incomplete snapshot export');
|
|
235
|
+
const items = rows.slice(start, start + pageRows);
|
|
236
|
+
if (bytesOf(items) > Math.min(pageBytes, bounds.bytes)) throw new RangeError('Export byte credits');
|
|
237
|
+
yield { state: 'ready', rows: items, keys: items.map(keyOf), query, snapshot: request.snapshot };
|
|
238
|
+
}
|
|
239
|
+
await store.transaction(checkVersion);
|
|
240
|
+
if (disposed || signal?.aborted || request.query !== query || request.snapshot !== snapshot() || !rows)
|
|
241
|
+
throw new Error('Incomplete snapshot export');
|
|
242
|
+
yield { state: 'complete', total, query, snapshot: request.snapshot };
|
|
243
|
+
}
|
|
244
|
+
finally { exports.delete(iterator); }
|
|
245
|
+
})();
|
|
246
|
+
exports.add(iterator);
|
|
247
|
+
return iterator;
|
|
248
|
+
},
|
|
249
|
+
async dispose() {
|
|
250
|
+
if (!disposed) {
|
|
251
|
+
disposed = true; unsubscribe(); observers.clear();
|
|
252
|
+
for (const controller of pending.keys()) controller.abort();
|
|
253
|
+
}
|
|
254
|
+
await Promise.allSettled([...pending.values()]);
|
|
255
|
+
await Promise.allSettled([...exports].map((iterator) => iterator.return()));
|
|
256
|
+
exports.clear();
|
|
257
|
+
continuations.clear(); rows = null; residentBytes = 0;
|
|
258
|
+
},
|
|
259
|
+
};
|
|
260
|
+
return Object.freeze(provider);
|
|
261
|
+
}
|