@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.211

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/AGENTS.md +57 -0
  2. package/dist/index.d.ts +2514 -58
  3. package/dist/index.js +12561 -2449
  4. package/dist/otel/index.d.ts +2 -2
  5. package/dist/otel/index.js +6 -6
  6. package/dist/sqlite-open.js +303 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +439 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +472 -0
  11. package/dist/types.d.ts +751 -11
  12. package/package.json +11 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
  15. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  16. package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
  17. package/src/bucket-blurhash.test.ts +148 -0
  18. package/src/build-globals.d.ts +12 -0
  19. package/src/events/events.test.ts +2 -1
  20. package/src/events/index.ts +3 -0
  21. package/src/index.ts +36 -2
  22. package/src/modules/app-release/index.test.ts +125 -0
  23. package/src/modules/app-release/index.ts +201 -0
  24. package/src/modules/auth/auth.local-first.test.ts +101 -0
  25. package/src/modules/auth/events/index.ts +2 -1
  26. package/src/modules/auth/index.ts +127 -24
  27. package/src/modules/cache/cache.relay.test.ts +95 -0
  28. package/src/modules/cache/index.ts +163 -43
  29. package/src/modules/cache/types.ts +2 -2
  30. package/src/modules/crdt/crdt-field.ts +294 -0
  31. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  32. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  33. package/src/modules/crdt/index.ts +463 -0
  34. package/src/modules/crdt/loro-loader.ts +25 -0
  35. package/src/modules/data/data.hydration.test.ts +142 -0
  36. package/src/modules/data/data.membership.test.ts +523 -0
  37. package/src/modules/data/data.notify-table.test.ts +41 -0
  38. package/src/modules/data/data.pending-ids.test.ts +199 -0
  39. package/src/modules/data/data.rebind.test.ts +170 -0
  40. package/src/modules/data/data.rematerialize.test.ts +114 -0
  41. package/src/modules/data/data.run.test.ts +113 -0
  42. package/src/modules/data/data.settled-writes.test.ts +206 -0
  43. package/src/modules/data/data.status.test.ts +249 -0
  44. package/src/modules/data/id-set-plan.test.ts +122 -0
  45. package/src/modules/data/index.ts +1815 -151
  46. package/src/modules/data/mutation-id.test.ts +25 -0
  47. package/src/modules/data/mutation-id.ts +35 -0
  48. package/src/modules/data/window-query.test.ts +52 -0
  49. package/src/modules/data/window-query.ts +194 -0
  50. package/src/modules/devtools/flags.ts +349 -0
  51. package/src/modules/devtools/index.ts +450 -46
  52. package/src/modules/devtools/notify-throttle.test.ts +154 -0
  53. package/src/modules/devtools/state-shape.test.ts +146 -0
  54. package/src/modules/devtools/storage-info.test.ts +79 -0
  55. package/src/modules/devtools/storage-info.ts +168 -0
  56. package/src/modules/devtools/versions.test.ts +74 -0
  57. package/src/modules/devtools/versions.ts +110 -0
  58. package/src/modules/feature-flag/index.test.ts +251 -0
  59. package/src/modules/feature-flag/index.ts +308 -0
  60. package/src/modules/ref-tables.test.ts +91 -0
  61. package/src/modules/ref-tables.ts +88 -0
  62. package/src/modules/sync/engine.ts +164 -82
  63. package/src/modules/sync/events/index.ts +9 -2
  64. package/src/modules/sync/queue/queue-down.test.ts +180 -0
  65. package/src/modules/sync/queue/queue-down.ts +80 -13
  66. package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
  67. package/src/modules/sync/queue/queue-up.ts +241 -57
  68. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  69. package/src/modules/sync/scheduler.retry.test.ts +237 -0
  70. package/src/modules/sync/scheduler.ts +215 -13
  71. package/src/modules/sync/sync.cleanup.test.ts +116 -0
  72. package/src/modules/sync/sync.health.test.ts +149 -0
  73. package/src/modules/sync/sync.heartbeat.test.ts +80 -0
  74. package/src/modules/sync/sync.live-removal.test.ts +175 -0
  75. package/src/modules/sync/sync.reconnect.test.ts +145 -0
  76. package/src/modules/sync/sync.subquery.test.ts +82 -0
  77. package/src/modules/sync/sync.tabs.test.ts +249 -0
  78. package/src/modules/sync/sync.ts +1726 -99
  79. package/src/modules/sync/utils.test.ts +269 -2
  80. package/src/modules/sync/utils.ts +201 -17
  81. package/src/otel/index.ts +13 -10
  82. package/src/services/blobs/blob-cache.test.ts +359 -0
  83. package/src/services/blobs/blob-cache.ts +603 -0
  84. package/src/services/blobs/blob-manifest.ts +227 -0
  85. package/src/services/blobs/blob-store.test.ts +77 -0
  86. package/src/services/blobs/blob-store.ts +359 -0
  87. package/src/services/blobs/blob.fixture.ts +90 -0
  88. package/src/services/blobs/index.ts +70 -0
  89. package/src/services/database/cache-engine.ts +193 -0
  90. package/src/services/database/connection-supervisor.test.ts +289 -0
  91. package/src/services/database/connection-supervisor.ts +415 -0
  92. package/src/services/database/database.query-timeout.test.ts +83 -0
  93. package/src/services/database/database.ts +41 -12
  94. package/src/services/database/engine-factory.ts +33 -0
  95. package/src/services/database/errors.ts +34 -0
  96. package/src/services/database/events/index.ts +2 -1
  97. package/src/services/database/index.ts +7 -0
  98. package/src/services/database/local-migrator.ts +30 -27
  99. package/src/services/database/local.test.ts +64 -0
  100. package/src/services/database/local.ts +484 -67
  101. package/src/services/database/plan-render.test.ts +159 -0
  102. package/src/services/database/plan-render.ts +108 -0
  103. package/src/services/database/relation-resolver.test.ts +413 -0
  104. package/src/services/database/relation-resolver.ts +0 -0
  105. package/src/services/database/remote.ts +110 -14
  106. package/src/services/database/sqlite-cache-engine.test.ts +616 -0
  107. package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
  108. package/src/services/database/sqlite-cache-engine.ts +1358 -0
  109. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  110. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  111. package/src/services/database/sqlite-lock-verify.test.ts +33 -0
  112. package/src/services/database/sqlite-lock-verify.ts +45 -0
  113. package/src/services/database/sqlite-open.test.ts +150 -0
  114. package/src/services/database/sqlite-open.ts +164 -0
  115. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  116. package/src/services/database/sqlite-plan-sql.ts +138 -0
  117. package/src/services/database/sqlite-projection.test.ts +99 -0
  118. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  119. package/src/services/database/sqlite-select.test.ts +246 -0
  120. package/src/services/database/sqlite-select.ts +131 -0
  121. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  122. package/src/services/database/sqlite-transport.ts +224 -0
  123. package/src/services/database/sqlite-worker.ts +437 -0
  124. package/src/services/database/surql-translate.ts +416 -0
  125. package/src/services/database/surreal-cache-engine.ts +161 -0
  126. package/src/services/logger/index.ts +3 -2
  127. package/src/services/persistence/localstorage.ts +2 -2
  128. package/src/services/persistence/resilient.ts +11 -4
  129. package/src/services/persistence/surrealdb.ts +10 -10
  130. package/src/services/stream-processor/index.ts +796 -84
  131. package/src/services/stream-processor/permissions.test.ts +47 -0
  132. package/src/services/stream-processor/permissions.ts +53 -0
  133. package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
  134. package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
  135. package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
  136. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  137. package/src/services/stream-processor/wasm-types.ts +59 -3
  138. package/src/services/tabs/broker-client.ts +283 -0
  139. package/src/services/tabs/broker.test.ts +327 -0
  140. package/src/services/tabs/coordinator.test.ts +365 -0
  141. package/src/services/tabs/coordinator.ts +633 -0
  142. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  143. package/src/services/tabs/leader-locks.ts +75 -0
  144. package/src/services/tabs/protocol.ts +258 -0
  145. package/src/services/tabs/support.ts +36 -0
  146. package/src/services/tabs/tabs-broker-worker.ts +640 -0
  147. package/src/sp00ky.auth-order.test.ts +92 -0
  148. package/src/sp00ky.init-query.test.ts +183 -0
  149. package/src/sp00ky.local-first.test.ts +60 -0
  150. package/src/sp00ky.ts +1693 -0
  151. package/src/types.ts +528 -13
  152. package/src/utils/blurhash.ts +90 -0
  153. package/src/utils/error-classification.test.ts +44 -0
  154. package/src/utils/error-classification.ts +7 -0
  155. package/src/utils/index.ts +79 -13
  156. package/src/utils/parser.test.ts +49 -120
  157. package/src/utils/parser.ts +32 -2
  158. package/src/utils/semver.test.ts +32 -0
  159. package/src/utils/semver.ts +30 -0
  160. package/src/utils/surql.ts +30 -18
  161. package/src/utils/withRetry.test.ts +1 -1
  162. package/tsdown.config.ts +86 -1
  163. package/src/spooky.ts +0 -395
@@ -0,0 +1,159 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import type { QueryPlan } from '@spooky-sync/query-builder';
3
+ import { renderBaseSelectSurql, renderRelationFetchSurql } from './plan-render';
4
+ import { buildWindowMaterializationPlan } from '../../modules/data/window-query';
5
+
6
+ describe('renderBaseSelectSurql', () => {
7
+ it('renders projection, where, order, limit, offset', () => {
8
+ const plan: QueryPlan = {
9
+ table: 'post',
10
+ select: ['title', 'body'],
11
+ where: [{ field: 'published', op: '=', value: true }],
12
+ orderBy: [['created', 'desc']],
13
+ limit: 10,
14
+ offset: 20,
15
+ };
16
+ const { sql, vars } = renderBaseSelectSurql(plan);
17
+ expect(sql).toBe('SELECT title, body FROM post WHERE published = $__p0 ORDER BY created desc LIMIT 10 START 20;');
18
+ expect(vars).toEqual({ __p0: true });
19
+ });
20
+
21
+ it('renders OR groups and comparison ops', () => {
22
+ const plan: QueryPlan = {
23
+ table: 'game',
24
+ where: [
25
+ { or: [{ field: 'white', op: '=', value: 'u:1' }, { field: 'black', op: '=', value: 'u:1' }] },
26
+ { field: 'moves', op: '>=', value: 5 },
27
+ ],
28
+ };
29
+ const { sql, vars } = renderBaseSelectSurql(plan);
30
+ expect(sql).toBe('SELECT * FROM game WHERE (white = $__p0 OR black = $__p1) AND moves >= $__p2;');
31
+ expect(vars).toEqual({ __p0: 'u:1', __p1: 'u:1', __p2: 5 });
32
+ });
33
+
34
+ it('honors paramRef and swap without binding a new var', () => {
35
+ const plan: QueryPlan = {
36
+ table: 't',
37
+ where: [{ field: 'tags', op: 'CONTAINS', value: undefined, paramRef: 'tag', swap: true }],
38
+ };
39
+ const { sql, vars } = renderBaseSelectSurql(plan, { tag: 'x' });
40
+ expect(sql).toBe('SELECT * FROM t WHERE $tag CONTAINS tags;');
41
+ expect(vars).toEqual({ tag: 'x' });
42
+ });
43
+ });
44
+
45
+ // Regression guard for the "authors + comments don't load" class: any filter on
46
+ // a RecordId column (`id`) must coerce its string value with
47
+ // `type::record(<string> …)`. On SurrealDB `id = "thread:x"` (string) never
48
+ // matches a RecordId, so a base select filtered by id resolves EMPTY and its
49
+ // whole `.related()` subtree (author, comments) loads nothing. The string-based
50
+ // MemStore in relation-resolver.test.ts can't catch this (it compares keys as
51
+ // strings), so these assert the rendered SurrealQL directly — both the parent
52
+ // (base select) and child (relation fetch) sides.
53
+ describe('record-id coercion (authors/comments loading regression)', () => {
54
+ it('coerces a base-select `id = <value>` filter to a record id', () => {
55
+ const plan: QueryPlan = { table: 'thread', where: [{ field: 'id', op: '=', value: 'thread:abc' }] };
56
+ const { sql, vars } = renderBaseSelectSurql(plan);
57
+ expect(sql).toBe('SELECT * FROM thread WHERE id = type::record(<string> $__p0);');
58
+ expect(vars).toEqual({ __p0: 'thread:abc' });
59
+ });
60
+
61
+ it('coerces a base-select `id = $paramRef` filter (the ThreadDetail path)', () => {
62
+ const plan: QueryPlan = { table: 'thread', where: [{ field: 'id', op: '=', value: undefined, paramRef: 'id' }] };
63
+ const { sql } = renderBaseSelectSurql(plan, { id: 'thread:abc' });
64
+ expect(sql).toBe('SELECT * FROM thread WHERE id = type::record(<string> $id);');
65
+ });
66
+
67
+ it('does NOT coerce non-id fields (plain string/bool columns stay literal)', () => {
68
+ const plan: QueryPlan = {
69
+ table: 'thread',
70
+ where: [{ field: 'title', op: '=', value: 'hi' }, { field: 'published', op: '=', value: true }],
71
+ };
72
+ const { sql } = renderBaseSelectSurql(plan);
73
+ expect(sql).toBe('SELECT * FROM thread WHERE title = $__p0 AND published = $__p1;');
74
+ });
75
+
76
+ // Regression guard for the thread-detail "crossed results → 404" bug: a
77
+ // slave-mode node carries BOTH a baked `value` and a `paramRef` (= the field
78
+ // name). Materialization MUST bind `params[paramRef]` (the query's identity),
79
+ // NOT the baked value — otherwise a stale/other plan whose baked id ≠ the
80
+ // query's params would surface a different record's row. So rendering a plan
81
+ // baked to id-A with params for id-B must filter by B.
82
+ it('slave-mode node binds params over the baked value (records follow identity)', () => {
83
+ const planBakedToA: QueryPlan = {
84
+ table: 'thread',
85
+ where: [{ field: 'id', op: '=', value: 'thread:A', paramRef: 'id' }],
86
+ };
87
+ const { sql, vars } = renderBaseSelectSurql(planBakedToA, { id: 'thread:B' });
88
+ expect(sql).toBe('SELECT * FROM thread WHERE id = type::record(<string> $id);');
89
+ expect(vars).toEqual({ id: 'thread:B' }); // B wins — the baked A is ignored
90
+ });
91
+
92
+ it('slave-mode node falls back to the baked value when the param is absent', () => {
93
+ const plan: QueryPlan = {
94
+ table: 'thread',
95
+ where: [{ field: 'id', op: '=', value: 'thread:A', paramRef: 'id' }],
96
+ };
97
+ const { sql, vars } = renderBaseSelectSurql(plan, {}); // no params.id
98
+ expect(sql).toBe('SELECT * FROM thread WHERE id = type::record(<string> $__p0);');
99
+ expect(vars).toEqual({ __p0: 'thread:A' });
100
+ });
101
+
102
+ it('coerces `id` inside an OR group too', () => {
103
+ const plan: QueryPlan = {
104
+ table: 'thread',
105
+ where: [{ or: [{ field: 'id', op: '=', value: 'thread:a' }, { field: 'id', op: '=', value: 'thread:b' }] }],
106
+ };
107
+ const { sql } = renderBaseSelectSurql(plan);
108
+ expect(sql).toBe(
109
+ 'SELECT * FROM thread WHERE (id = type::record(<string> $__p0) OR id = type::record(<string> $__p1));'
110
+ );
111
+ });
112
+
113
+ it('relation fetch coerces its matchField keys (the .118 fix — kept locked)', () => {
114
+ const { sql } = renderRelationFetchSurql({ table: 'user', matchField: 'id', keys: ['user:1'] });
115
+ expect(sql).toContain('id IN $__keys.map(|$__k| type::record(<string> $__k))');
116
+ });
117
+ });
118
+
119
+ describe('renderRelationFetchSurql', () => {
120
+ it('builds a WHERE ... IN $__keys batch fetch, omitting LIMIT', () => {
121
+ const { sql, vars } = renderRelationFetchSurql({
122
+ table: 'comment',
123
+ matchField: 'post',
124
+ keys: ['post:1', 'post:2'],
125
+ where: [{ field: 'hidden', op: '=', value: false }],
126
+ orderBy: [['rank', 'asc']],
127
+ });
128
+ expect(sql).toBe('SELECT * FROM comment WHERE post IN $__keys.map(|$__k| type::record(<string> $__k)) AND hidden = $__p0 ORDER BY rank asc;');
129
+ expect(vars).toEqual({ __keys: ['post:1', 'post:2'], __p0: false });
130
+ });
131
+ });
132
+
133
+ describe('buildWindowMaterializationPlan', () => {
134
+ it('restricts to the id-set for offset queries, dropping where/limit/offset', () => {
135
+ const plan: QueryPlan = {
136
+ table: 'post',
137
+ where: [{ field: 'x', op: '=', value: 1 }],
138
+ orderBy: [['created', 'desc']],
139
+ limit: 10,
140
+ offset: 20,
141
+ relations: [{ alias: 'a', table: 'u', cardinality: 'one', foreignKeyField: 'a' }],
142
+ };
143
+ const win = buildWindowMaterializationPlan(plan, ['post:5', 'post:6']);
144
+ expect(win).toEqual({
145
+ table: 'post',
146
+ orderBy: [['created', 'desc']],
147
+ relations: [{ alias: 'a', table: 'u', cardinality: 'one', foreignKeyField: 'a' }],
148
+ ids: ['post:5', 'post:6'],
149
+ where: undefined,
150
+ limit: undefined,
151
+ offset: undefined,
152
+ });
153
+ });
154
+
155
+ it('returns null for non-offset queries (keep normal path)', () => {
156
+ expect(buildWindowMaterializationPlan({ table: 'post', limit: 10 }, ['post:1'])).toBeNull();
157
+ expect(buildWindowMaterializationPlan({ table: 'post', offset: 0 }, ['post:1'])).toBeNull();
158
+ });
159
+ });
@@ -0,0 +1,108 @@
1
+ import type { QueryPlan, WhereNode, WhereComparison } from '@spooky-sync/query-builder';
2
+ import type { OrderBy, RelationFetch } from './cache-engine';
3
+
4
+ /**
5
+ * Render helpers that turn an engine-neutral {@link QueryPlan} into a concrete
6
+ * dialect. Two consumers: `SurrealCacheEngine` (SurrealQL) and, indirectly, the
7
+ * SQLite worker (which uses the SQL variants). Relations are NOT rendered here —
8
+ * they are resolved by `resolveRelations` via the {@link RelationFetch}
9
+ * primitive, so both engines share the exact same decomposition.
10
+ */
11
+
12
+ export interface RenderedQuery {
13
+ sql: string;
14
+ vars: Record<string, unknown>;
15
+ }
16
+
17
+ interface RenderCtx {
18
+ vars: Record<string, unknown>;
19
+ n: number;
20
+ }
21
+
22
+ function bind(ctx: RenderCtx, value: unknown): string {
23
+ const name = `__p${ctx.n++}`;
24
+ ctx.vars[name] = value;
25
+ return `$${name}`;
26
+ }
27
+
28
+ function renderComparisonSurql(c: WhereComparison, ctx: RenderCtx): string {
29
+ // Prefer the query's own param (`paramRef`) so a filter materializes from
30
+ // `params` (the query's identity), not a baked literal — this is what slaves
31
+ // a query's rows to its id. A pure `$`-ref node has no `value`, so it always
32
+ // uses the param. A slave-mode node also carries `value` as a fallback for
33
+ // when the param is absent (e.g. a non-column field stripped by parseParams).
34
+ const useParam =
35
+ c.paramRef !== undefined &&
36
+ (c.value === undefined || Object.prototype.hasOwnProperty.call(ctx.vars, c.paramRef));
37
+ const rawRight = useParam ? `$${c.paramRef}` : bind(ctx, c.value);
38
+ // `id` is a RecordId, but correlation/filter values arrive as record-id
39
+ // STRINGS (e.g. "thread:abc"). On SurrealDB `id = "thread:abc"` never matches
40
+ // (string ≠ record), so a base select filtered by id (e.g. the ThreadDetail
41
+ // query `… FROM thread WHERE id = $id`) resolves empty — and its whole
42
+ // `.related()` subtree (author, comments) then loads nothing. Coerce with
43
+ // `type::record(<string> …)` (idempotent if already a RecordId), mirroring
44
+ // renderRelationFetchSurql's matchField coercion.
45
+ const right =
46
+ c.field === 'id' ? `type::record(<string> ${rawRight})` : rawRight;
47
+ return c.swap ? `${right} ${c.op} ${c.field}` : `${c.field} ${c.op} ${right}`;
48
+ }
49
+
50
+ /** Render a WHERE conjunction (AND of comparisons / OR-groups) to SurrealQL. */
51
+ export function renderWhereSurql(nodes: WhereNode[], ctx: RenderCtx): string {
52
+ return nodes
53
+ .map((node) => {
54
+ if ('or' in node) {
55
+ return `(${node.or.map((c) => renderComparisonSurql(c, ctx)).join(' OR ')})`;
56
+ }
57
+ return renderComparisonSurql(node, ctx);
58
+ })
59
+ .join(' AND ');
60
+ }
61
+
62
+ function renderOrderBy(orderBy: OrderBy): string {
63
+ return ` ORDER BY ${orderBy.map(([f, d]) => `${f} ${d}`).join(', ')}`;
64
+ }
65
+
66
+ /**
67
+ * Render the BASE of a SELECT (no relations) to SurrealQL. `params` supplies
68
+ * pre-existing bound params (e.g. `$__win` for windowing, or `paramRef` values)
69
+ * and is merged into the returned vars.
70
+ */
71
+ export function renderBaseSelectSurql(
72
+ plan: QueryPlan,
73
+ params: Record<string, unknown> = {}
74
+ ): RenderedQuery {
75
+ const ctx: RenderCtx = { vars: { ...params }, n: 0 };
76
+ const projection = plan.select && plan.select.length > 0 ? plan.select.join(', ') : '*';
77
+ let sql = `SELECT ${projection} FROM ${plan.table}`;
78
+ if (plan.where && plan.where.length > 0) {
79
+ sql += ` WHERE ${renderWhereSurql(plan.where, ctx)}`;
80
+ }
81
+ if (plan.orderBy && plan.orderBy.length > 0) sql += renderOrderBy(plan.orderBy);
82
+ if (plan.limit !== undefined) sql += ` LIMIT ${plan.limit}`;
83
+ if (plan.offset !== undefined) sql += ` START ${plan.offset}`;
84
+ return { sql: `${sql};`, vars: ctx.vars };
85
+ }
86
+
87
+ /**
88
+ * Render a batched relation fetch to SurrealQL:
89
+ * `SELECT <select> FROM <table> WHERE <matchField> IN $__keys [AND <where>] [ORDER BY]`.
90
+ * The resolver re-applies per-parent ORDER/LIMIT after grouping, so LIMIT is
91
+ * intentionally omitted here.
92
+ */
93
+ export function renderRelationFetchSurql(req: RelationFetch): RenderedQuery {
94
+ const ctx: RenderCtx = { vars: { __keys: req.keys }, n: 0 };
95
+ const projection =
96
+ req.select && req.select.length > 0 ? ['id', ...req.select].join(', ') : '*';
97
+ // The correlation keys arrive as record-id STRINGS (`"user:abc"`), but the
98
+ // matched column (`id`, or a `record<…>` foreign key) is a RecordId. In
99
+ // SurrealDB `id IN ["user:abc"]` never matches (string ≠ record), so every
100
+ // `.related()` field would resolve empty. Coerce each key to a record id with
101
+ // `type::record(<string> …)` (idempotent if a key is already a RecordId).
102
+ let sql = `SELECT ${projection} FROM ${req.table} WHERE ${req.matchField} IN $__keys.map(|$__k| type::record(<string> $__k))`;
103
+ if (req.where && req.where.length > 0) {
104
+ sql += ` AND ${renderWhereSurql(req.where, ctx)}`;
105
+ }
106
+ if (req.orderBy && req.orderBy.length > 0) sql += renderOrderBy(req.orderBy);
107
+ return { sql: `${sql};`, vars: ctx.vars };
108
+ }
@@ -0,0 +1,413 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import type { RelationPlan, WhereNode } from '@spooky-sync/query-builder';
3
+ import { resolveRelations, sortRows, stableKey } from './relation-resolver';
4
+ import {
5
+ RelationCycleError,
6
+ type RelationFetch,
7
+ type Row,
8
+ type RowFetcher,
9
+ } from './cache-engine';
10
+
11
+ /**
12
+ * Deterministic in-memory store used as the reference for decomposition. It
13
+ * implements the SAME fetch primitive the real engines expose
14
+ * (`WHERE matchField IN keys AND <where>`, ORDER BY, projection) so a passing
15
+ * resolver here is correct independent of SurrealDB/SQLite specifics. Rows are
16
+ * deep-cloned on read (as a real engine returns fresh objects), so nested
17
+ * attachment never mutates the store.
18
+ */
19
+ class MemStore implements RowFetcher {
20
+ constructor(private tables: Record<string, Row[]>) {}
21
+
22
+ async fetchRelation(req: RelationFetch): Promise<Row[]> {
23
+ const keySet = new Set(req.keys.map(stableKey));
24
+ let rows = (this.tables[req.table] ?? []).filter((r) =>
25
+ keySet.has(stableKey(r[req.matchField]))
26
+ );
27
+ if (req.where) rows = rows.filter((r) => matchesWhere(r, req.where!));
28
+ rows = rows.map((r) => structuredClone(r));
29
+ if (req.orderBy) rows = sortRows(rows, req.orderBy);
30
+ if (req.select) {
31
+ rows = rows.map((r) => {
32
+ const out: Row = {};
33
+ for (const f of ['id', ...req.select!]) if (f in r) out[f] = r[f];
34
+ return out;
35
+ });
36
+ }
37
+ return rows;
38
+ }
39
+ }
40
+
41
+ function matchesWhere(row: Row, where: WhereNode[]): boolean {
42
+ const cmp = (c: { field: string; op: string; value: unknown }): boolean => {
43
+ const v = row[c.field];
44
+ switch (c.op) {
45
+ case '=':
46
+ return stableKey(v) === stableKey(c.value);
47
+ case '!=':
48
+ return stableKey(v) !== stableKey(c.value);
49
+ case '>':
50
+ return (v as number) > (c.value as number);
51
+ case '>=':
52
+ return (v as number) >= (c.value as number);
53
+ case '<':
54
+ return (v as number) < (c.value as number);
55
+ case '<=':
56
+ return (v as number) <= (c.value as number);
57
+ default:
58
+ return false;
59
+ }
60
+ };
61
+ return where.every((node) => {
62
+ if ('or' in node) return node.or.some(cmp);
63
+ return cmp(node);
64
+ });
65
+ }
66
+
67
+ // Convenience builder for a RelationPlan.
68
+ function rel(p: Partial<RelationPlan> & Pick<RelationPlan, 'alias' | 'table' | 'cardinality' | 'foreignKeyField'>): RelationPlan {
69
+ return p as RelationPlan;
70
+ }
71
+
72
+ describe('resolveRelations — decomposition', () => {
73
+ it('1. flat, no relations: leaves parents untouched', async () => {
74
+ const store = new MemStore({});
75
+ const parents: Row[] = [{ id: 'post:1', title: 'a' }];
76
+ await resolveRelations(parents, undefined, store);
77
+ expect(parents).toEqual([{ id: 'post:1', title: 'a' }]);
78
+ });
79
+
80
+ it('2a. one-to-one match: attaches the single related row', async () => {
81
+ const store = new MemStore({ user: [{ id: 'user:1', name: 'ana' }] });
82
+ const parents: Row[] = [{ id: 'post:1', author: 'user:1' }];
83
+ await resolveRelations(
84
+ parents,
85
+ [rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 })],
86
+ store
87
+ );
88
+ expect(parents[0].author).toEqual({ id: 'user:1', name: 'ana' });
89
+ });
90
+
91
+ it('2b. one-to-one no match: attaches null', async () => {
92
+ const store = new MemStore({ user: [] });
93
+ const parents: Row[] = [{ id: 'post:1', author: 'user:404' }];
94
+ await resolveRelations(
95
+ parents,
96
+ [rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 })],
97
+ store
98
+ );
99
+ expect(parents[0].author).toBeNull();
100
+ });
101
+
102
+ it('3. one-to-many: empty vs N children grouped per parent', async () => {
103
+ const store = new MemStore({
104
+ comment: [
105
+ { id: 'comment:1', post: 'post:1', body: 'x' },
106
+ { id: 'comment:2', post: 'post:1', body: 'y' },
107
+ { id: 'comment:3', post: 'post:2', body: 'z' },
108
+ ],
109
+ });
110
+ const parents: Row[] = [{ id: 'post:1' }, { id: 'post:2' }, { id: 'post:3' }];
111
+ await resolveRelations(
112
+ parents,
113
+ [rel({ alias: 'comments', table: 'comment', cardinality: 'many', foreignKeyField: 'post' })],
114
+ store
115
+ );
116
+ expect((parents[0].comments as Row[]).map((c) => c.id)).toEqual(['comment:1', 'comment:2']);
117
+ expect((parents[1].comments as Row[]).map((c) => c.id)).toEqual(['comment:3']);
118
+ expect(parents[2].comments).toEqual([]);
119
+ });
120
+
121
+ it('4. per-parent LIMIT + ORDER applied within each group, not globally', async () => {
122
+ const store = new MemStore({
123
+ comment: [
124
+ { id: 'comment:1', post: 'post:1', rank: 3 },
125
+ { id: 'comment:2', post: 'post:1', rank: 1 },
126
+ { id: 'comment:3', post: 'post:1', rank: 2 },
127
+ { id: 'comment:4', post: 'post:2', rank: 5 },
128
+ { id: 'comment:5', post: 'post:2', rank: 4 },
129
+ ],
130
+ });
131
+ const parents: Row[] = [{ id: 'post:1' }, { id: 'post:2' }];
132
+ await resolveRelations(
133
+ parents,
134
+ [
135
+ rel({
136
+ alias: 'comments',
137
+ table: 'comment',
138
+ cardinality: 'many',
139
+ foreignKeyField: 'post',
140
+ orderBy: [['rank', 'asc']],
141
+ limit: 2,
142
+ }),
143
+ ],
144
+ store
145
+ );
146
+ // post:1 keeps its OWN top-2 by rank (1,2) — not a global top-2 (which would
147
+ // have starved post:2).
148
+ expect((parents[0].comments as Row[]).map((c) => c.rank)).toEqual([1, 2]);
149
+ expect((parents[1].comments as Row[]).map((c) => c.rank)).toEqual([4, 5]);
150
+ });
151
+
152
+ it('5. sub-where filters the relation batch', async () => {
153
+ const store = new MemStore({
154
+ comment: [
155
+ { id: 'comment:1', post: 'post:1', hidden: false },
156
+ { id: 'comment:2', post: 'post:1', hidden: true },
157
+ ],
158
+ });
159
+ const parents: Row[] = [{ id: 'post:1' }];
160
+ await resolveRelations(
161
+ parents,
162
+ [
163
+ rel({
164
+ alias: 'comments',
165
+ table: 'comment',
166
+ cardinality: 'many',
167
+ foreignKeyField: 'post',
168
+ where: [{ field: 'hidden', op: '=', value: false }],
169
+ }),
170
+ ],
171
+ store
172
+ );
173
+ expect((parents[0].comments as Row[]).map((c) => c.id)).toEqual(['comment:1']);
174
+ });
175
+
176
+ it('6. 2-level nested with cross-level $parent dependency (many -> one)', async () => {
177
+ const store = new MemStore({
178
+ comment: [
179
+ { id: 'comment:1', post: 'post:1', author: 'user:1' },
180
+ { id: 'comment:2', post: 'post:1', author: 'user:2' },
181
+ ],
182
+ user: [
183
+ { id: 'user:1', name: 'ana' },
184
+ { id: 'user:2', name: 'bob' },
185
+ ],
186
+ });
187
+ const parents: Row[] = [{ id: 'post:1' }];
188
+ await resolveRelations(
189
+ parents,
190
+ [
191
+ rel({
192
+ alias: 'comments',
193
+ table: 'comment',
194
+ cardinality: 'many',
195
+ foreignKeyField: 'post',
196
+ relations: [
197
+ rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 }),
198
+ ],
199
+ }),
200
+ ],
201
+ store
202
+ );
203
+ const comments = parents[0].comments as Row[];
204
+ expect(comments.map((c) => (c.author as Row).name)).toEqual(['ana', 'bob']);
205
+ });
206
+
207
+ it('7. 3-level deep, mixed cardinality, O(depth) batch count', async () => {
208
+ let batches = 0;
209
+ const base = new MemStore({
210
+ comment: [{ id: 'comment:1', post: 'post:1', author: 'user:1' }],
211
+ user: [{ id: 'user:1', org: 'org:1' }],
212
+ org: [{ id: 'org:1', name: 'acme' }],
213
+ });
214
+ const counting: RowFetcher = {
215
+ fetchRelation: (req) => {
216
+ batches++;
217
+ return base.fetchRelation(req);
218
+ },
219
+ };
220
+ const parents: Row[] = [{ id: 'post:1' }];
221
+ await resolveRelations(
222
+ parents,
223
+ [
224
+ rel({
225
+ alias: 'comments',
226
+ table: 'comment',
227
+ cardinality: 'many',
228
+ foreignKeyField: 'post',
229
+ relations: [
230
+ rel({
231
+ alias: 'author',
232
+ table: 'user',
233
+ cardinality: 'one',
234
+ foreignKeyField: 'author',
235
+ limit: 1,
236
+ relations: [
237
+ rel({ alias: 'org', table: 'org', cardinality: 'one', foreignKeyField: 'org', limit: 1 }),
238
+ ],
239
+ }),
240
+ ],
241
+ }),
242
+ ],
243
+ counting
244
+ );
245
+ const org = ((parents[0].comments as Row[])[0].author as Row).org as Row;
246
+ expect(org.name).toBe('acme');
247
+ // One batch per level (3 levels), NOT per row.
248
+ expect(batches).toBe(3);
249
+ });
250
+
251
+ it('8. null/absent foreign keys mid-tree yield empty, no spurious fetch', async () => {
252
+ let fetchedKeys: unknown[] = [];
253
+ const base = new MemStore({ user: [{ id: 'user:1', name: 'ana' }] });
254
+ const spy: RowFetcher = {
255
+ fetchRelation: (req) => {
256
+ fetchedKeys = req.keys;
257
+ return base.fetchRelation(req);
258
+ },
259
+ };
260
+ const parents: Row[] = [
261
+ { id: 'post:1', author: 'user:1' },
262
+ { id: 'post:2', author: null },
263
+ { id: 'post:3' }, // absent
264
+ ];
265
+ await resolveRelations(
266
+ parents,
267
+ [rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 })],
268
+ spy
269
+ );
270
+ expect(fetchedKeys).toEqual(['user:1']); // null/absent excluded
271
+ expect(parents[0].author).toEqual({ id: 'user:1', name: 'ana' });
272
+ expect(parents[1].author).toBeNull();
273
+ expect(parents[2].author).toBeNull();
274
+ });
275
+
276
+ it('9. nesting past MAX_RELATION_DEPTH throws RelationCycleError', async () => {
277
+ // Build a plan nested deeper than the guard.
278
+ let leaf: RelationPlan = rel({ alias: 'r', table: 't', cardinality: 'one', foreignKeyField: 'r' });
279
+ for (let i = 0; i < 20; i++) {
280
+ leaf = rel({ alias: 'r', table: 't', cardinality: 'one', foreignKeyField: 'r', relations: [leaf] });
281
+ }
282
+ const store = new MemStore({ t: [{ id: 't:1', r: 't:1' }] });
283
+ await expect(resolveRelations([{ id: 't:1', r: 't:1' }], [leaf], store)).rejects.toBeInstanceOf(
284
+ RelationCycleError
285
+ );
286
+ });
287
+
288
+ it('10. RecordId-shaped keys group identically to their string form', async () => {
289
+ // Parent FK is a RecordId-like object; child id is the string form.
290
+ const rid = { tb: 'user', id: '1', toString: () => 'user:1' };
291
+ const store = new MemStore({ user: [{ id: 'user:1', name: 'ana' }] });
292
+ const parents: Row[] = [{ id: 'post:1', author: rid }];
293
+ await resolveRelations(
294
+ parents,
295
+ [rel({ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author', limit: 1 })],
296
+ store
297
+ );
298
+ expect((parents[0].author as Row).name).toBe('ana');
299
+ // alias appended LAST (key order parity with `SELECT *, <sub> AS alias`).
300
+ expect(Object.keys(parents[0])).toEqual(['id', 'author']);
301
+ });
302
+ });
303
+
304
+ describe('stableKey', () => {
305
+ it('collapses RecordId object and string form', () => {
306
+ expect(stableKey({ tb: 'user', id: '1' })).toBe('user:1');
307
+ expect(stableKey('user:1')).toBe('user:1');
308
+ });
309
+ });
310
+
311
+ /**
312
+ * Independent, obviously-correct reference resolver: for each parent, fetch its
313
+ * relations directly (no batching, no dedup) and recurse. The batched
314
+ * `resolveRelations` must produce byte-identical output for random trees.
315
+ */
316
+ async function naiveResolve(parents: Row[], relations: RelationPlan[] | undefined, store: MemStore): Promise<void> {
317
+ if (!relations) return;
318
+ for (const parent of parents) {
319
+ for (const r of relations) {
320
+ const isOne = r.cardinality === 'one';
321
+ const key = isOne ? parent[r.foreignKeyField] : parent['id'];
322
+ let bucket: Row[] = [];
323
+ if (key != null) {
324
+ bucket = await store.fetchRelation({
325
+ table: r.table,
326
+ matchField: isOne ? 'id' : r.foreignKeyField,
327
+ keys: [key],
328
+ where: r.where,
329
+ orderBy: r.orderBy,
330
+ select: r.select,
331
+ });
332
+ await naiveResolve(bucket, r.relations, store);
333
+ }
334
+ if (r.orderBy) bucket = sortRows(bucket, r.orderBy);
335
+ if (r.limit !== undefined) bucket = bucket.slice(0, r.limit);
336
+ parent[r.alias] = isOne ? bucket[0] ?? null : bucket;
337
+ }
338
+ }
339
+ }
340
+
341
+ // Tiny seeded PRNG so failures reproduce (Math.random is nondeterministic).
342
+ function mulberry32(seed: number): () => number {
343
+ return () => {
344
+ seed |= 0;
345
+ seed = (seed + 0x6d2b79f5) | 0;
346
+ let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
347
+ t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
348
+ return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
349
+ };
350
+ }
351
+
352
+ describe('resolveRelations — property: batched == naive over random trees', () => {
353
+ const TABLES = ['a', 'b', 'c', 'd'];
354
+
355
+ it('matches the naive reference for 200 random datasets/plans', async () => {
356
+ for (let seed = 0; seed < 200; seed++) {
357
+ const rng = mulberry32(seed + 1);
358
+ const pick = <T,>(arr: T[]): T => arr[Math.floor(rng() * arr.length)];
359
+
360
+ // Random dataset: each table gets a few rows with random FK fields.
361
+ const tables: Record<string, Row[]> = {};
362
+ for (const t of TABLES) {
363
+ const n = Math.floor(rng() * 5);
364
+ tables[t] = [];
365
+ for (let i = 0; i < n; i++) {
366
+ tables[t].push({
367
+ id: `${t}:${i}`,
368
+ rank: Math.floor(rng() * 5),
369
+ // FK columns pointing at every table (some valid, some dangling).
370
+ ...Object.fromEntries(
371
+ TABLES.map((ft) => [`${ft}_fk`, rng() < 0.7 ? `${ft}:${Math.floor(rng() * 5)}` : null])
372
+ ),
373
+ });
374
+ }
375
+ }
376
+
377
+ // Random relation tree, depth <= 4.
378
+ const buildPlan = (depth: number): RelationPlan[] => {
379
+ if (depth > 4 || rng() < 0.35) return [];
380
+ const count = Math.floor(rng() * 2) + 1;
381
+ const out: RelationPlan[] = [];
382
+ for (let i = 0; i < count; i++) {
383
+ const table = pick(TABLES);
384
+ const cardinality = rng() < 0.5 ? 'one' : 'many';
385
+ out.push(
386
+ rel({
387
+ alias: `rel_${depth}_${i}`,
388
+ table,
389
+ cardinality,
390
+ foreignKeyField: cardinality === 'one' ? `${table}_fk` : `${pick(TABLES)}_fk`,
391
+ orderBy: rng() < 0.5 ? [['rank', pick(['asc', 'desc'] as const)]] : undefined,
392
+ limit: rng() < 0.5 ? Math.floor(rng() * 3) + 1 : undefined,
393
+ relations: buildPlan(depth + 1),
394
+ })
395
+ );
396
+ }
397
+ return out;
398
+ };
399
+ const plan = buildPlan(0);
400
+
401
+ const rootTable = pick(TABLES);
402
+ const roots = (tables[rootTable] ?? []).map((r) => structuredClone(r));
403
+ if (roots.length === 0) continue;
404
+
405
+ const batched = structuredClone(roots);
406
+ const naive = structuredClone(roots);
407
+ await resolveRelations(batched, plan, new MemStore(structuredClone(tables)));
408
+ await naiveResolve(naive, plan, new MemStore(structuredClone(tables)));
409
+
410
+ expect(batched, `seed ${seed}`).toEqual(naive);
411
+ }
412
+ });
413
+ });