@spooky-sync/core 0.0.1-canary.13 → 0.0.1-canary.131
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +56 -0
- package/dist/index.d.ts +1171 -372
- package/dist/index.js +5726 -1277
- package/dist/otel/index.d.ts +21 -0
- package/dist/otel/index.js +86 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +107 -0
- package/dist/types.d.ts +746 -0
- package/package.json +39 -8
- package/skills/sp00ky-core/SKILL.md +258 -0
- package/skills/sp00ky-core/references/auth.md +98 -0
- package/skills/sp00ky-core/references/config.md +76 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +9 -2
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +77 -32
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +288 -0
- package/src/modules/crdt/crdt-hydration.test.ts +206 -0
- package/src/modules/crdt/index.ts +357 -0
- package/src/modules/data/data.hydration.test.ts +150 -0
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.recurring.test.ts +137 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/index.ts +1234 -127
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +154 -0
- package/src/modules/devtools/index.ts +191 -30
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +81 -0
- package/src/modules/feature-flag/index.test.ts +120 -0
- package/src/modules/feature-flag/index.ts +209 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.ts +12 -5
- package/src/modules/sync/queue/queue-up.ts +29 -14
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.ts +73 -7
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.ts +1017 -62
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +182 -17
- package/src/otel/index.ts +127 -0
- package/src/services/database/cache-engine.ts +143 -0
- package/src/services/database/database.ts +11 -11
- package/src/services/database/engine-factory.ts +32 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +6 -0
- package/src/services/database/local-migrator.ts +28 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +452 -66
- package/src/services/database/plan-render.test.ts +133 -0
- package/src/services/database/plan-render.ts +100 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +13 -13
- package/src/services/database/sqlite-cache-engine.test.ts +85 -0
- package/src/services/database/sqlite-cache-engine.ts +699 -0
- package/src/services/database/sqlite-worker.ts +116 -0
- package/src/services/database/surql-translate.ts +291 -0
- package/src/services/database/surreal-cache-engine.ts +139 -0
- package/src/services/logger/index.ts +6 -101
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +41 -0
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +295 -38
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +104 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +18 -2
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +185 -0
- package/src/sp00ky.ts +1016 -0
- package/src/types.ts +258 -15
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +35 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/surql.ts +24 -15
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +77 -1
- package/src/spooky.ts +0 -392
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type { QueryPlan, WhereNode, WhereComparison } from '@spooky-sync/query-builder';
|
|
2
|
+
import type { OrderBy, RelationFetch } from './cache-engine';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Render helpers that turn an engine-neutral {@link QueryPlan} into a concrete
|
|
6
|
+
* dialect. Two consumers: `SurrealCacheEngine` (SurrealQL) and, indirectly, the
|
|
7
|
+
* SQLite worker (which uses the SQL variants). Relations are NOT rendered here —
|
|
8
|
+
* they are resolved by `resolveRelations` via the {@link RelationFetch}
|
|
9
|
+
* primitive, so both engines share the exact same decomposition.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export interface RenderedQuery {
|
|
13
|
+
sql: string;
|
|
14
|
+
vars: Record<string, unknown>;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
interface RenderCtx {
|
|
18
|
+
vars: Record<string, unknown>;
|
|
19
|
+
n: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function bind(ctx: RenderCtx, value: unknown): string {
|
|
23
|
+
const name = `__p${ctx.n++}`;
|
|
24
|
+
ctx.vars[name] = value;
|
|
25
|
+
return `$${name}`;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function renderComparisonSurql(c: WhereComparison, ctx: RenderCtx): string {
|
|
29
|
+
const rawRight = c.paramRef ? `$${c.paramRef}` : bind(ctx, c.value);
|
|
30
|
+
// `id` is a RecordId, but correlation/filter values arrive as record-id
|
|
31
|
+
// STRINGS (e.g. "thread:abc"). On SurrealDB `id = "thread:abc"` never matches
|
|
32
|
+
// (string ≠ record), so a base select filtered by id (e.g. the ThreadDetail
|
|
33
|
+
// query `… FROM thread WHERE id = $id`) resolves empty — and its whole
|
|
34
|
+
// `.related()` subtree (author, comments) then loads nothing. Coerce with
|
|
35
|
+
// `type::record(<string> …)` (idempotent if already a RecordId), mirroring
|
|
36
|
+
// renderRelationFetchSurql's matchField coercion.
|
|
37
|
+
const right =
|
|
38
|
+
c.field === 'id' ? `type::record(<string> ${rawRight})` : rawRight;
|
|
39
|
+
return c.swap ? `${right} ${c.op} ${c.field}` : `${c.field} ${c.op} ${right}`;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Render a WHERE conjunction (AND of comparisons / OR-groups) to SurrealQL. */
|
|
43
|
+
export function renderWhereSurql(nodes: WhereNode[], ctx: RenderCtx): string {
|
|
44
|
+
return nodes
|
|
45
|
+
.map((node) => {
|
|
46
|
+
if ('or' in node) {
|
|
47
|
+
return `(${node.or.map((c) => renderComparisonSurql(c, ctx)).join(' OR ')})`;
|
|
48
|
+
}
|
|
49
|
+
return renderComparisonSurql(node, ctx);
|
|
50
|
+
})
|
|
51
|
+
.join(' AND ');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function renderOrderBy(orderBy: OrderBy): string {
|
|
55
|
+
return ` ORDER BY ${orderBy.map(([f, d]) => `${f} ${d}`).join(', ')}`;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Render the BASE of a SELECT (no relations) to SurrealQL. `params` supplies
|
|
60
|
+
* pre-existing bound params (e.g. `$__win` for windowing, or `paramRef` values)
|
|
61
|
+
* and is merged into the returned vars.
|
|
62
|
+
*/
|
|
63
|
+
export function renderBaseSelectSurql(
|
|
64
|
+
plan: QueryPlan,
|
|
65
|
+
params: Record<string, unknown> = {}
|
|
66
|
+
): RenderedQuery {
|
|
67
|
+
const ctx: RenderCtx = { vars: { ...params }, n: 0 };
|
|
68
|
+
const projection = plan.select && plan.select.length > 0 ? plan.select.join(', ') : '*';
|
|
69
|
+
let sql = `SELECT ${projection} FROM ${plan.table}`;
|
|
70
|
+
if (plan.where && plan.where.length > 0) {
|
|
71
|
+
sql += ` WHERE ${renderWhereSurql(plan.where, ctx)}`;
|
|
72
|
+
}
|
|
73
|
+
if (plan.orderBy && plan.orderBy.length > 0) sql += renderOrderBy(plan.orderBy);
|
|
74
|
+
if (plan.limit !== undefined) sql += ` LIMIT ${plan.limit}`;
|
|
75
|
+
if (plan.offset !== undefined) sql += ` START ${plan.offset}`;
|
|
76
|
+
return { sql: `${sql};`, vars: ctx.vars };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Render a batched relation fetch to SurrealQL:
|
|
81
|
+
* `SELECT <select> FROM <table> WHERE <matchField> IN $__keys [AND <where>] [ORDER BY]`.
|
|
82
|
+
* The resolver re-applies per-parent ORDER/LIMIT after grouping, so LIMIT is
|
|
83
|
+
* intentionally omitted here.
|
|
84
|
+
*/
|
|
85
|
+
export function renderRelationFetchSurql(req: RelationFetch): RenderedQuery {
|
|
86
|
+
const ctx: RenderCtx = { vars: { __keys: req.keys }, n: 0 };
|
|
87
|
+
const projection =
|
|
88
|
+
req.select && req.select.length > 0 ? ['id', ...req.select].join(', ') : '*';
|
|
89
|
+
// The correlation keys arrive as record-id STRINGS (`"user:abc"`), but the
|
|
90
|
+
// matched column (`id`, or a `record<…>` foreign key) is a RecordId. In
|
|
91
|
+
// SurrealDB `id IN ["user:abc"]` never matches (string ≠ record), so every
|
|
92
|
+
// `.related()` field would resolve empty. Coerce each key to a record id with
|
|
93
|
+
// `type::record(<string> …)` (idempotent if a key is already a RecordId).
|
|
94
|
+
let sql = `SELECT ${projection} FROM ${req.table} WHERE ${req.matchField} IN $__keys.map(|$__k| type::record(<string> $__k))`;
|
|
95
|
+
if (req.where && req.where.length > 0) {
|
|
96
|
+
sql += ` AND ${renderWhereSurql(req.where, ctx)}`;
|
|
97
|
+
}
|
|
98
|
+
if (req.orderBy && req.orderBy.length > 0) sql += renderOrderBy(req.orderBy);
|
|
99
|
+
return { sql: `${sql};`, vars: ctx.vars };
|
|
100
|
+
}
|
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import type { RelationPlan, WhereNode } from '@spooky-sync/query-builder';
|
|
3
|
+
import { resolveRelations, sortRows, stableKey } from './relation-resolver';
|
|
4
|
+
import {
|
|
5
|
+
RelationCycleError,
|
|
6
|
+
type RelationFetch,
|
|
7
|
+
type Row,
|
|
8
|
+
type RowFetcher,
|
|
9
|
+
} from './cache-engine';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Deterministic in-memory store used as the reference for decomposition. It
|
|
13
|
+
* implements the SAME fetch primitive the real engines expose
|
|
14
|
+
* (`WHERE matchField IN keys AND <where>`, ORDER BY, projection) so a passing
|
|
15
|
+
* resolver here is correct independent of SurrealDB/SQLite specifics. Rows are
|
|
16
|
+
* deep-cloned on read (as a real engine returns fresh objects), so nested
|
|
17
|
+
* attachment never mutates the store.
|
|
18
|
+
*/
|
|
19
|
+
class MemStore implements RowFetcher {
|
|
20
|
+
constructor(private tables: Record<string, Row[]>) {}
|
|
21
|
+
|
|
22
|
+
async fetchRelation(req: RelationFetch): Promise<Row[]> {
|
|
23
|
+
const keySet = new Set(req.keys.map(stableKey));
|
|
24
|
+
let rows = (this.tables[req.table] ?? []).filter((r) =>
|
|
25
|
+
keySet.has(stableKey(r[req.matchField]))
|
|
26
|
+
);
|
|
27
|
+
if (req.where) rows = rows.filter((r) => matchesWhere(r, req.where!));
|
|
28
|
+
rows = rows.map((r) => structuredClone(r));
|
|
29
|
+
if (req.orderBy) rows = sortRows(rows, req.orderBy);
|
|
30
|
+
if (req.select) {
|
|
31
|
+
rows = rows.map((r) => {
|
|
32
|
+
const out: Row = {};
|
|
33
|
+
for (const f of ['id', ...req.select!]) if (f in r) out[f] = r[f];
|
|
34
|
+
return out;
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
return rows;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function matchesWhere(row: Row, where: WhereNode[]): boolean {
|
|
42
|
+
const cmp = (c: { field: string; op: string; value: unknown }): boolean => {
|
|
43
|
+
const v = row[c.field];
|
|
44
|
+
switch (c.op) {
|
|
45
|
+
case '=':
|
|
46
|
+
return stableKey(v) === stableKey(c.value);
|
|
47
|
+
case '!=':
|
|
48
|
+
return stableKey(v) !== stableKey(c.value);
|
|
49
|
+
case '>':
|
|
50
|
+
return (v as number) > (c.value as number);
|
|
51
|
+
case '>=':
|
|
52
|
+
return (v as number) >= (c.value as number);
|
|
53
|
+
case '<':
|
|
54
|
+
return (v as number) < (c.value as number);
|
|
55
|
+
case '<=':
|
|
56
|
+
return (v as number) <= (c.value as number);
|
|
57
|
+
default:
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
};
|
|
61
|
+
return where.every((node) => {
|
|
62
|
+
if ('or' in node) return node.or.some(cmp);
|
|
63
|
+
return cmp(node);
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Convenience builder for a RelationPlan.
|
|
68
|
+
function rel(p: Partial<RelationPlan> & Pick<RelationPlan, 'alias' | 'table' | 'cardinality' | 'foreignKeyField'>): RelationPlan {
|
|
69
|
+
return p as RelationPlan;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
describe('resolveRelations — decomposition', () => {
|
|
73
|
+
it('1. flat, no relations: leaves parents untouched', async () => {
|
|
74
|
+
const store = new MemStore({});
|
|
75
|
+
const parents: Row[] = [{ id: 'post:1', title: 'a' }];
|
|
76
|
+
await resolveRelations(parents, undefined, store);
|
|
77
|
+
expect(parents).toEqual([{ id: 'post:1', title: 'a' }]);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('2a. one-to-one match: attaches the single related row', async () => {
|
|
81
|
+
const store = new MemStore({ user: [{ id: 'user:1', name: 'ana' }] });
|
|
82
|
+
const parents: Row[] = [{ id: 'post:1', author: 'user:1' }];
|
|
83
|
+
await resolveRelations(
|
|
84
|
+
parents,
|
|
85
|
+
[rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 })],
|
|
86
|
+
store
|
|
87
|
+
);
|
|
88
|
+
expect(parents[0].author).toEqual({ id: 'user:1', name: 'ana' });
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('2b. one-to-one no match: attaches null', async () => {
|
|
92
|
+
const store = new MemStore({ user: [] });
|
|
93
|
+
const parents: Row[] = [{ id: 'post:1', author: 'user:404' }];
|
|
94
|
+
await resolveRelations(
|
|
95
|
+
parents,
|
|
96
|
+
[rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 })],
|
|
97
|
+
store
|
|
98
|
+
);
|
|
99
|
+
expect(parents[0].author).toBeNull();
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it('3. one-to-many: empty vs N children grouped per parent', async () => {
|
|
103
|
+
const store = new MemStore({
|
|
104
|
+
comment: [
|
|
105
|
+
{ id: 'comment:1', post: 'post:1', body: 'x' },
|
|
106
|
+
{ id: 'comment:2', post: 'post:1', body: 'y' },
|
|
107
|
+
{ id: 'comment:3', post: 'post:2', body: 'z' },
|
|
108
|
+
],
|
|
109
|
+
});
|
|
110
|
+
const parents: Row[] = [{ id: 'post:1' }, { id: 'post:2' }, { id: 'post:3' }];
|
|
111
|
+
await resolveRelations(
|
|
112
|
+
parents,
|
|
113
|
+
[rel({ alias: 'comments', table: 'comment', cardinality: 'many', foreignKeyField: 'post' })],
|
|
114
|
+
store
|
|
115
|
+
);
|
|
116
|
+
expect((parents[0].comments as Row[]).map((c) => c.id)).toEqual(['comment:1', 'comment:2']);
|
|
117
|
+
expect((parents[1].comments as Row[]).map((c) => c.id)).toEqual(['comment:3']);
|
|
118
|
+
expect(parents[2].comments).toEqual([]);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it('4. per-parent LIMIT + ORDER applied within each group, not globally', async () => {
|
|
122
|
+
const store = new MemStore({
|
|
123
|
+
comment: [
|
|
124
|
+
{ id: 'comment:1', post: 'post:1', rank: 3 },
|
|
125
|
+
{ id: 'comment:2', post: 'post:1', rank: 1 },
|
|
126
|
+
{ id: 'comment:3', post: 'post:1', rank: 2 },
|
|
127
|
+
{ id: 'comment:4', post: 'post:2', rank: 5 },
|
|
128
|
+
{ id: 'comment:5', post: 'post:2', rank: 4 },
|
|
129
|
+
],
|
|
130
|
+
});
|
|
131
|
+
const parents: Row[] = [{ id: 'post:1' }, { id: 'post:2' }];
|
|
132
|
+
await resolveRelations(
|
|
133
|
+
parents,
|
|
134
|
+
[
|
|
135
|
+
rel({
|
|
136
|
+
alias: 'comments',
|
|
137
|
+
table: 'comment',
|
|
138
|
+
cardinality: 'many',
|
|
139
|
+
foreignKeyField: 'post',
|
|
140
|
+
orderBy: [['rank', 'asc']],
|
|
141
|
+
limit: 2,
|
|
142
|
+
}),
|
|
143
|
+
],
|
|
144
|
+
store
|
|
145
|
+
);
|
|
146
|
+
// post:1 keeps its OWN top-2 by rank (1,2) — not a global top-2 (which would
|
|
147
|
+
// have starved post:2).
|
|
148
|
+
expect((parents[0].comments as Row[]).map((c) => c.rank)).toEqual([1, 2]);
|
|
149
|
+
expect((parents[1].comments as Row[]).map((c) => c.rank)).toEqual([4, 5]);
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it('5. sub-where filters the relation batch', async () => {
|
|
153
|
+
const store = new MemStore({
|
|
154
|
+
comment: [
|
|
155
|
+
{ id: 'comment:1', post: 'post:1', hidden: false },
|
|
156
|
+
{ id: 'comment:2', post: 'post:1', hidden: true },
|
|
157
|
+
],
|
|
158
|
+
});
|
|
159
|
+
const parents: Row[] = [{ id: 'post:1' }];
|
|
160
|
+
await resolveRelations(
|
|
161
|
+
parents,
|
|
162
|
+
[
|
|
163
|
+
rel({
|
|
164
|
+
alias: 'comments',
|
|
165
|
+
table: 'comment',
|
|
166
|
+
cardinality: 'many',
|
|
167
|
+
foreignKeyField: 'post',
|
|
168
|
+
where: [{ field: 'hidden', op: '=', value: false }],
|
|
169
|
+
}),
|
|
170
|
+
],
|
|
171
|
+
store
|
|
172
|
+
);
|
|
173
|
+
expect((parents[0].comments as Row[]).map((c) => c.id)).toEqual(['comment:1']);
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
it('6. 2-level nested with cross-level $parent dependency (many -> one)', async () => {
|
|
177
|
+
const store = new MemStore({
|
|
178
|
+
comment: [
|
|
179
|
+
{ id: 'comment:1', post: 'post:1', author: 'user:1' },
|
|
180
|
+
{ id: 'comment:2', post: 'post:1', author: 'user:2' },
|
|
181
|
+
],
|
|
182
|
+
user: [
|
|
183
|
+
{ id: 'user:1', name: 'ana' },
|
|
184
|
+
{ id: 'user:2', name: 'bob' },
|
|
185
|
+
],
|
|
186
|
+
});
|
|
187
|
+
const parents: Row[] = [{ id: 'post:1' }];
|
|
188
|
+
await resolveRelations(
|
|
189
|
+
parents,
|
|
190
|
+
[
|
|
191
|
+
rel({
|
|
192
|
+
alias: 'comments',
|
|
193
|
+
table: 'comment',
|
|
194
|
+
cardinality: 'many',
|
|
195
|
+
foreignKeyField: 'post',
|
|
196
|
+
relations: [
|
|
197
|
+
rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 }),
|
|
198
|
+
],
|
|
199
|
+
}),
|
|
200
|
+
],
|
|
201
|
+
store
|
|
202
|
+
);
|
|
203
|
+
const comments = parents[0].comments as Row[];
|
|
204
|
+
expect(comments.map((c) => (c.author as Row).name)).toEqual(['ana', 'bob']);
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
it('7. 3-level deep, mixed cardinality, O(depth) batch count', async () => {
|
|
208
|
+
let batches = 0;
|
|
209
|
+
const base = new MemStore({
|
|
210
|
+
comment: [{ id: 'comment:1', post: 'post:1', author: 'user:1' }],
|
|
211
|
+
user: [{ id: 'user:1', org: 'org:1' }],
|
|
212
|
+
org: [{ id: 'org:1', name: 'acme' }],
|
|
213
|
+
});
|
|
214
|
+
const counting: RowFetcher = {
|
|
215
|
+
fetchRelation: (req) => {
|
|
216
|
+
batches++;
|
|
217
|
+
return base.fetchRelation(req);
|
|
218
|
+
},
|
|
219
|
+
};
|
|
220
|
+
const parents: Row[] = [{ id: 'post:1' }];
|
|
221
|
+
await resolveRelations(
|
|
222
|
+
parents,
|
|
223
|
+
[
|
|
224
|
+
rel({
|
|
225
|
+
alias: 'comments',
|
|
226
|
+
table: 'comment',
|
|
227
|
+
cardinality: 'many',
|
|
228
|
+
foreignKeyField: 'post',
|
|
229
|
+
relations: [
|
|
230
|
+
rel({
|
|
231
|
+
alias: 'author',
|
|
232
|
+
table: 'user',
|
|
233
|
+
cardinality: 'one',
|
|
234
|
+
foreignKeyField: 'author',
|
|
235
|
+
limit: 1,
|
|
236
|
+
relations: [
|
|
237
|
+
rel({ alias: 'org', table: 'org', cardinality: 'one', foreignKeyField: 'org', limit: 1 }),
|
|
238
|
+
],
|
|
239
|
+
}),
|
|
240
|
+
],
|
|
241
|
+
}),
|
|
242
|
+
],
|
|
243
|
+
counting
|
|
244
|
+
);
|
|
245
|
+
const org = ((parents[0].comments as Row[])[0].author as Row).org as Row;
|
|
246
|
+
expect(org.name).toBe('acme');
|
|
247
|
+
// One batch per level (3 levels), NOT per row.
|
|
248
|
+
expect(batches).toBe(3);
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
it('8. null/absent foreign keys mid-tree yield empty, no spurious fetch', async () => {
|
|
252
|
+
let fetchedKeys: unknown[] = [];
|
|
253
|
+
const base = new MemStore({ user: [{ id: 'user:1', name: 'ana' }] });
|
|
254
|
+
const spy: RowFetcher = {
|
|
255
|
+
fetchRelation: (req) => {
|
|
256
|
+
fetchedKeys = req.keys;
|
|
257
|
+
return base.fetchRelation(req);
|
|
258
|
+
},
|
|
259
|
+
};
|
|
260
|
+
const parents: Row[] = [
|
|
261
|
+
{ id: 'post:1', author: 'user:1' },
|
|
262
|
+
{ id: 'post:2', author: null },
|
|
263
|
+
{ id: 'post:3' }, // absent
|
|
264
|
+
];
|
|
265
|
+
await resolveRelations(
|
|
266
|
+
parents,
|
|
267
|
+
[rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 })],
|
|
268
|
+
spy
|
|
269
|
+
);
|
|
270
|
+
expect(fetchedKeys).toEqual(['user:1']); // null/absent excluded
|
|
271
|
+
expect(parents[0].author).toEqual({ id: 'user:1', name: 'ana' });
|
|
272
|
+
expect(parents[1].author).toBeNull();
|
|
273
|
+
expect(parents[2].author).toBeNull();
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
it('9. nesting past MAX_RELATION_DEPTH throws RelationCycleError', async () => {
|
|
277
|
+
// Build a plan nested deeper than the guard.
|
|
278
|
+
let leaf: RelationPlan = rel({ alias: 'r', table: 't', cardinality: 'one', foreignKeyField: 'r' });
|
|
279
|
+
for (let i = 0; i < 20; i++) {
|
|
280
|
+
leaf = rel({ alias: 'r', table: 't', cardinality: 'one', foreignKeyField: 'r', relations: [leaf] });
|
|
281
|
+
}
|
|
282
|
+
const store = new MemStore({ t: [{ id: 't:1', r: 't:1' }] });
|
|
283
|
+
await expect(resolveRelations([{ id: 't:1', r: 't:1' }], [leaf], store)).rejects.toBeInstanceOf(
|
|
284
|
+
RelationCycleError
|
|
285
|
+
);
|
|
286
|
+
});
|
|
287
|
+
|
|
288
|
+
it('10. RecordId-shaped keys group identically to their string form', async () => {
|
|
289
|
+
// Parent FK is a RecordId-like object; child id is the string form.
|
|
290
|
+
const rid = { tb: 'user', id: '1', toString: () => 'user:1' };
|
|
291
|
+
const store = new MemStore({ user: [{ id: 'user:1', name: 'ana' }] });
|
|
292
|
+
const parents: Row[] = [{ id: 'post:1', author: rid }];
|
|
293
|
+
await resolveRelations(
|
|
294
|
+
parents,
|
|
295
|
+
[rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 })],
|
|
296
|
+
store
|
|
297
|
+
);
|
|
298
|
+
expect((parents[0].author as Row).name).toBe('ana');
|
|
299
|
+
// alias appended LAST (key order parity with `SELECT *, <sub> AS alias`).
|
|
300
|
+
expect(Object.keys(parents[0])).toEqual(['id', 'author']);
|
|
301
|
+
});
|
|
302
|
+
});
|
|
303
|
+
|
|
304
|
+
describe('stableKey', () => {
|
|
305
|
+
it('collapses RecordId object and string form', () => {
|
|
306
|
+
expect(stableKey({ tb: 'user', id: '1' })).toBe('user:1');
|
|
307
|
+
expect(stableKey('user:1')).toBe('user:1');
|
|
308
|
+
});
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Independent, obviously-correct reference resolver: for each parent, fetch its
|
|
313
|
+
* relations directly (no batching, no dedup) and recurse. The batched
|
|
314
|
+
* `resolveRelations` must produce byte-identical output for random trees.
|
|
315
|
+
*/
|
|
316
|
+
async function naiveResolve(parents: Row[], relations: RelationPlan[] | undefined, store: MemStore): Promise<void> {
|
|
317
|
+
if (!relations) return;
|
|
318
|
+
for (const parent of parents) {
|
|
319
|
+
for (const r of relations) {
|
|
320
|
+
const isOne = r.cardinality === 'one';
|
|
321
|
+
const key = isOne ? parent[r.foreignKeyField] : parent['id'];
|
|
322
|
+
let bucket: Row[] = [];
|
|
323
|
+
if (key != null) {
|
|
324
|
+
bucket = await store.fetchRelation({
|
|
325
|
+
table: r.table,
|
|
326
|
+
matchField: isOne ? 'id' : r.foreignKeyField,
|
|
327
|
+
keys: [key],
|
|
328
|
+
where: r.where,
|
|
329
|
+
orderBy: r.orderBy,
|
|
330
|
+
select: r.select,
|
|
331
|
+
});
|
|
332
|
+
await naiveResolve(bucket, r.relations, store);
|
|
333
|
+
}
|
|
334
|
+
if (r.orderBy) bucket = sortRows(bucket, r.orderBy);
|
|
335
|
+
if (r.limit !== undefined) bucket = bucket.slice(0, r.limit);
|
|
336
|
+
parent[r.alias] = isOne ? bucket[0] ?? null : bucket;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// Tiny seeded PRNG so failures reproduce (Math.random is nondeterministic).
|
|
342
|
+
function mulberry32(seed: number): () => number {
|
|
343
|
+
return () => {
|
|
344
|
+
seed |= 0;
|
|
345
|
+
seed = (seed + 0x6d2b79f5) | 0;
|
|
346
|
+
let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
|
|
347
|
+
t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
|
|
348
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
describe('resolveRelations — property: batched == naive over random trees', () => {
|
|
353
|
+
const TABLES = ['a', 'b', 'c', 'd'];
|
|
354
|
+
|
|
355
|
+
it('matches the naive reference for 200 random datasets/plans', async () => {
|
|
356
|
+
for (let seed = 0; seed < 200; seed++) {
|
|
357
|
+
const rng = mulberry32(seed + 1);
|
|
358
|
+
const pick = <T,>(arr: T[]): T => arr[Math.floor(rng() * arr.length)];
|
|
359
|
+
|
|
360
|
+
// Random dataset: each table gets a few rows with random FK fields.
|
|
361
|
+
const tables: Record<string, Row[]> = {};
|
|
362
|
+
for (const t of TABLES) {
|
|
363
|
+
const n = Math.floor(rng() * 5);
|
|
364
|
+
tables[t] = [];
|
|
365
|
+
for (let i = 0; i < n; i++) {
|
|
366
|
+
tables[t].push({
|
|
367
|
+
id: `${t}:${i}`,
|
|
368
|
+
rank: Math.floor(rng() * 5),
|
|
369
|
+
// FK columns pointing at every table (some valid, some dangling).
|
|
370
|
+
...Object.fromEntries(
|
|
371
|
+
TABLES.map((ft) => [`${ft}_fk`, rng() < 0.7 ? `${ft}:${Math.floor(rng() * 5)}` : null])
|
|
372
|
+
),
|
|
373
|
+
});
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// Random relation tree, depth <= 4.
|
|
378
|
+
const buildPlan = (depth: number): RelationPlan[] => {
|
|
379
|
+
if (depth > 4 || rng() < 0.35) return [];
|
|
380
|
+
const count = Math.floor(rng() * 2) + 1;
|
|
381
|
+
const out: RelationPlan[] = [];
|
|
382
|
+
for (let i = 0; i < count; i++) {
|
|
383
|
+
const table = pick(TABLES);
|
|
384
|
+
const cardinality = rng() < 0.5 ? 'one' : 'many';
|
|
385
|
+
out.push(
|
|
386
|
+
rel({
|
|
387
|
+
alias: `rel_${depth}_${i}`,
|
|
388
|
+
table,
|
|
389
|
+
cardinality,
|
|
390
|
+
foreignKeyField: cardinality === 'one' ? `${table}_fk` : `${pick(TABLES)}_fk`,
|
|
391
|
+
orderBy: rng() < 0.5 ? [['rank', pick(['asc', 'desc'] as const)]] : undefined,
|
|
392
|
+
limit: rng() < 0.5 ? Math.floor(rng() * 3) + 1 : undefined,
|
|
393
|
+
relations: buildPlan(depth + 1),
|
|
394
|
+
})
|
|
395
|
+
);
|
|
396
|
+
}
|
|
397
|
+
return out;
|
|
398
|
+
};
|
|
399
|
+
const plan = buildPlan(0);
|
|
400
|
+
|
|
401
|
+
const rootTable = pick(TABLES);
|
|
402
|
+
const roots = (tables[rootTable] ?? []).map((r) => structuredClone(r));
|
|
403
|
+
if (roots.length === 0) continue;
|
|
404
|
+
|
|
405
|
+
const batched = structuredClone(roots);
|
|
406
|
+
const naive = structuredClone(roots);
|
|
407
|
+
await resolveRelations(batched, plan, new MemStore(structuredClone(tables)));
|
|
408
|
+
await naiveResolve(naive, plan, new MemStore(structuredClone(tables)));
|
|
409
|
+
|
|
410
|
+
expect(batched, `seed ${seed}`).toEqual(naive);
|
|
411
|
+
}
|
|
412
|
+
});
|
|
413
|
+
});
|
|
Binary file
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
Diagnostic} from 'surrealdb';
|
|
1
3
|
import {
|
|
2
4
|
applyDiagnostics,
|
|
3
5
|
createRemoteEngines,
|
|
4
|
-
Diagnostic,
|
|
5
6
|
Surreal,
|
|
6
|
-
SurrealTransaction,
|
|
7
7
|
} from 'surrealdb';
|
|
8
|
-
import {
|
|
9
|
-
import { Logger } from '../logger/index';
|
|
8
|
+
import type { Sp00kyConfig } from '../../types';
|
|
9
|
+
import type { Logger } from '../logger/index';
|
|
10
10
|
import { AbstractDatabaseService } from './database';
|
|
11
11
|
import { createDatabaseEventSystem, DatabaseEventTypes } from './events/index';
|
|
12
12
|
|
|
13
13
|
export class RemoteDatabaseService extends AbstractDatabaseService {
|
|
14
|
-
private config:
|
|
14
|
+
private config: Sp00kyConfig<any>['database'];
|
|
15
15
|
protected eventType = DatabaseEventTypes.RemoteQuery;
|
|
16
16
|
|
|
17
|
-
constructor(config:
|
|
17
|
+
constructor(config: Sp00kyConfig<any>['database'], logger: Logger) {
|
|
18
18
|
const events = createDatabaseEventSystem();
|
|
19
19
|
super(
|
|
20
20
|
new Surreal({
|
|
@@ -29,7 +29,7 @@ export class RemoteDatabaseService extends AbstractDatabaseService {
|
|
|
29
29
|
type,
|
|
30
30
|
phase,
|
|
31
31
|
service: 'surrealdb:remote',
|
|
32
|
-
Category: '
|
|
32
|
+
Category: 'sp00ky-client::RemoteDatabaseService::diagnostics',
|
|
33
33
|
},
|
|
34
34
|
`Remote SurrealDB diagnostics captured ${type}:${phase}`
|
|
35
35
|
);
|
|
@@ -43,7 +43,7 @@ export class RemoteDatabaseService extends AbstractDatabaseService {
|
|
|
43
43
|
this.config = config;
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
-
getConfig():
|
|
46
|
+
getConfig(): Sp00kyConfig<any>['database'] {
|
|
47
47
|
return this.config;
|
|
48
48
|
}
|
|
49
49
|
|
|
@@ -55,7 +55,7 @@ export class RemoteDatabaseService extends AbstractDatabaseService {
|
|
|
55
55
|
endpoint,
|
|
56
56
|
namespace,
|
|
57
57
|
database,
|
|
58
|
-
Category: '
|
|
58
|
+
Category: 'sp00ky-client::RemoteDatabaseService::connect',
|
|
59
59
|
},
|
|
60
60
|
'Connecting to remote database'
|
|
61
61
|
);
|
|
@@ -68,25 +68,25 @@ export class RemoteDatabaseService extends AbstractDatabaseService {
|
|
|
68
68
|
|
|
69
69
|
if (token) {
|
|
70
70
|
this.logger.debug(
|
|
71
|
-
{ Category: '
|
|
71
|
+
{ Category: 'sp00ky-client::RemoteDatabaseService::connect' },
|
|
72
72
|
'Authenticating with token'
|
|
73
73
|
);
|
|
74
74
|
await this.client.authenticate(token);
|
|
75
75
|
}
|
|
76
76
|
this.logger.info(
|
|
77
|
-
{ Category: '
|
|
77
|
+
{ Category: 'sp00ky-client::RemoteDatabaseService::connect' },
|
|
78
78
|
'Connected to remote database'
|
|
79
79
|
);
|
|
80
80
|
} catch (err) {
|
|
81
81
|
this.logger.error(
|
|
82
|
-
{ err, Category: '
|
|
82
|
+
{ err, Category: 'sp00ky-client::RemoteDatabaseService::connect' },
|
|
83
83
|
'Failed to connect to remote database'
|
|
84
84
|
);
|
|
85
85
|
throw err;
|
|
86
86
|
}
|
|
87
87
|
} else {
|
|
88
88
|
this.logger.warn(
|
|
89
|
-
{ Category: '
|
|
89
|
+
{ Category: 'sp00ky-client::RemoteDatabaseService::connect' },
|
|
90
90
|
'No endpoint configured for remote database'
|
|
91
91
|
);
|
|
92
92
|
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import { pureWriteOpResult } from './sqlite-cache-engine';
|
|
4
|
+
import { translateSurql } from './surql-translate';
|
|
5
|
+
import type { SqlOp } from './surql-translate';
|
|
6
|
+
import { surql } from '../../utils/surql';
|
|
7
|
+
|
|
8
|
+
// `pureWriteOpResult` is the single source of truth for what a pure-write op
|
|
9
|
+
// contributes to a query's per-statement results. The batched fast path in
|
|
10
|
+
// `query()` and the per-op `execOp` path BOTH route through it, so a caller that
|
|
11
|
+
// reads a statement's output (e.g. `create()` reads `resultIndex:0` for the new
|
|
12
|
+
// row + its id) sees the same shape either way.
|
|
13
|
+
describe('pureWriteOpResult', () => {
|
|
14
|
+
it('echoes the written row (with id) for an upsert — no read-back', () => {
|
|
15
|
+
const op: SqlOp = {
|
|
16
|
+
kind: 'upsert',
|
|
17
|
+
id: 'connection:CONN_abc',
|
|
18
|
+
data: { provider: 'chesscom', username: 'hikaru' },
|
|
19
|
+
mode: 'replace',
|
|
20
|
+
};
|
|
21
|
+
expect(pureWriteOpResult(op)).toEqual({
|
|
22
|
+
provider: 'chesscom',
|
|
23
|
+
username: 'hikaru',
|
|
24
|
+
id: 'connection:CONN_abc',
|
|
25
|
+
});
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('stringifies a RecordId id via stableKey', () => {
|
|
29
|
+
const op: SqlOp = {
|
|
30
|
+
kind: 'upsert',
|
|
31
|
+
id: new RecordId('connection', 'CONN_abc'),
|
|
32
|
+
data: { provider: 'lichess' },
|
|
33
|
+
mode: 'replace',
|
|
34
|
+
};
|
|
35
|
+
expect(pureWriteOpResult(op)).toEqual({ provider: 'lichess', id: 'connection:CONN_abc' });
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it('yields [] for delete / deleteAll and null for noop', () => {
|
|
39
|
+
expect(pureWriteOpResult({ kind: 'delete', id: 'game:1' })).toEqual([]);
|
|
40
|
+
expect(pureWriteOpResult({ kind: 'deleteAll', table: 'game' })).toEqual([]);
|
|
41
|
+
expect(pureWriteOpResult({ kind: 'noop' })).toBeNull();
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
// Regression: a single `create()` compiles to an all-upsert transaction
|
|
46
|
+
// (createSet for the row + createMutation for the pending-mutation log) and
|
|
47
|
+
// extracts `resultIndex:0` for the created row. The SQLite fast path must return
|
|
48
|
+
// that row (with its id) at that index — returning `[]` there dropped the id and
|
|
49
|
+
// crashed the reconcile in `encodeRecordId` ("reading 'table'").
|
|
50
|
+
describe('create() tx result shaping (fast path parity)', () => {
|
|
51
|
+
it('resultIndex:0 carries the created row with its id', () => {
|
|
52
|
+
const rid = new RecordId('connection', 'CONN_abc');
|
|
53
|
+
const mid = new RecordId('_00_pending_mutations', '1');
|
|
54
|
+
const vars = {
|
|
55
|
+
id: rid,
|
|
56
|
+
mid,
|
|
57
|
+
data_provider: 'chesscom',
|
|
58
|
+
data_username: 'hikaru',
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
// Same statement pair DataModule.create emits.
|
|
62
|
+
const sealed = surql.seal(
|
|
63
|
+
surql.tx([
|
|
64
|
+
surql.createSet('id', [
|
|
65
|
+
{ key: 'provider', variable: 'data_provider' },
|
|
66
|
+
{ key: 'username', variable: 'data_username' },
|
|
67
|
+
]),
|
|
68
|
+
surql.createMutation('create', 'mid', 'id', 'data'),
|
|
69
|
+
]),
|
|
70
|
+
{ resultIndex: 0 }
|
|
71
|
+
);
|
|
72
|
+
|
|
73
|
+
const { transaction, ops } = translateSurql(sealed.sql, vars);
|
|
74
|
+
expect(transaction).toBe(true);
|
|
75
|
+
// Both statements are upserts → the engine takes the all-write fast path.
|
|
76
|
+
expect(ops.every((o) => o.kind === 'upsert')).toBe(true);
|
|
77
|
+
|
|
78
|
+
// Fast-path shaping: [null (BEGIN), ...one result per statement].
|
|
79
|
+
const shaped = [null, ...ops.map(pureWriteOpResult)];
|
|
80
|
+
const created = sealed.extract(shaped) as unknown as { id: unknown; provider: string };
|
|
81
|
+
|
|
82
|
+
expect(created.id).toBe('connection:CONN_abc');
|
|
83
|
+
expect(created.provider).toBe('chesscom');
|
|
84
|
+
});
|
|
85
|
+
});
|