@syncular/typegen 0.28.0 → 0.29.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -565,7 +565,11 @@ message. The prepared statement then yields the result column names +
565
565
  `declaredTypes` (SQLite's `sqlite3_column_decltype`).
566
566
 
567
567
  For FTS-backed named queries the synthesized type-check database creates the
568
- same declared columns plus `_syncular_source_id UNINDEXED`. `MATCH :query`
568
+ same declared columns plus `_syncular_source_id UNINDEXED`, and the client's
569
+ source-id mapping table. The emitted SQL reads every `_syncular_source_id`
570
+ reference it can bind through that mapping table on the projection rowid, so
571
+ FTS5 does not fetch a content row per match (SYQL §15); the authored SQL keeps
572
+ its types and identity. `MATCH :query`
569
573
  infers a string parameter, and the FTS table reference is folded into its
570
574
  owning synced table for dependency and scope-coverage metadata. `bm25`,
571
575
  `highlight`, and `snippet` are accepted deterministic FTS5 auxiliary
package/dist/query.d.ts CHANGED
@@ -264,8 +264,10 @@ export interface SqlRelations {
264
264
  * top-level CTE body, `nested` inside any other subquery. */
265
265
  readonly scopeAt: (offset: number) => string;
266
266
  }
267
- export declare function scanTableRefs(sql: string, ir: IrDocument): TableRef[];
268
- export declare function scanRelations(sql: string, ir: IrDocument): SqlRelations;
267
+ export declare function scanTableRefs(sql: string, ir: IrDocument, lowered?: boolean): TableRef[];
268
+ /** `lowered` scans emitted SQL, whose FTS mapping reads
269
+ * ({@link lowerFtsSourceIds}) are internal and not relations of the query. */
270
+ export declare function scanRelations(sql: string, ir: IrDocument, lowered?: boolean): SqlRelations;
269
271
  export interface SelectItem {
270
272
  /** The raw expression text (before any `AS alias`). */
271
273
  readonly expr: string;
@@ -296,6 +298,21 @@ export interface QueryDb {
296
298
  * type + NOT NULL, so `prepare()` validates references and decltype resolves.
297
299
  * This is exported so the generator can build one in-memory DB per run. */
298
300
  export declare function synthesizeDdl(ir: IrDocument): string;
301
+ /** SPEC client-local FTS5 projections: the source-id mapping table whose `id`
302
+ * equals the projection rowid. */
303
+ export declare function ftsMappingTable(index: string): string;
304
+ /**
305
+ * SYQL §15 FTS source-id lowering. Reading an FTS projection's
306
+ * `_syncular_source_id` makes FTS5 fetch the content row of every hit. Each
307
+ * FTS relation introduced by `FROM` or an inner `JOIN` gains a join to its
308
+ * mapping table on the projection rowid, and every reference that SQLite binds
309
+ * to that relation reads the mapping's `source_id` instead. The mapping
310
+ * columns carry `__syql_fts_` names, so no authored unqualified name changes
311
+ * its binding. A bare projected reference keeps its result name through an
312
+ * explicit alias. References this pass cannot bind with certainty keep
313
+ * reading the projection; both read the same value.
314
+ */
315
+ export declare function lowerFtsSourceIds(sql: string, ir: IrDocument): string;
299
316
  /**
300
317
  * Analyze ONE already-split statement against the IR + a prepared-DDL DB. The
301
318
  * name is pre-resolved (path-derived or a `-- name:` override) by the caller;
package/dist/query.js CHANGED
@@ -476,15 +476,20 @@ function sqlIdentifier(token) {
476
476
  const value = token.text.slice(1, -1);
477
477
  return quote === '[' ? value : value.split(`${quote}${quote}`).join(quote);
478
478
  }
479
- export function scanTableRefs(sql, ir) {
480
- return scanRelations(sql, ir).tables;
479
+ export function scanTableRefs(sql, ir, lowered = false) {
480
+ return scanRelations(sql, ir, lowered).tables;
481
481
  }
482
- export function scanRelations(sql, ir) {
482
+ /** `lowered` scans emitted SQL, whose FTS mapping reads
483
+ * ({@link lowerFtsSourceIds}) are internal and not relations of the query. */
484
+ export function scanRelations(sql, ir, lowered = false) {
483
485
  const tokens = lexSyqlSqlSource('query SQL', sql).filter((token) => !isSyqlTrivia(token) && token.kind !== 'eof');
484
486
  const known = new Map(ir.tables.flatMap((table) => [
485
487
  [table.name.toLowerCase(), table.name],
486
488
  ...table.ftsIndexes.map((index) => [index.name.toLowerCase(), index.name]),
487
489
  ]));
490
+ const mappings = new Set(lowered
491
+ ? ir.tables.flatMap((table) => table.ftsIndexes.map((index) => ftsMappingTable(index.name).toLowerCase()))
492
+ : []);
488
493
  const closes = new Map();
489
494
  const parents = [];
490
495
  const depths = [];
@@ -665,6 +670,8 @@ export function scanRelations(sql, ir) {
665
670
  });
666
671
  continue;
667
672
  }
673
+ if (mappings.has(rawTable.toLowerCase()))
674
+ continue;
668
675
  const table = known.get(rawTable.toLowerCase());
669
676
  if (table === undefined)
670
677
  throw new TypegenError('query SQL', `unresolved table relation ${JSON.stringify(rawTable)}`);
@@ -1112,11 +1119,205 @@ export function synthesizeDdl(ir) {
1112
1119
  }
1113
1120
  for (const index of table.ftsIndexes) {
1114
1121
  const tokenize = index.tokenize.replaceAll("'", "''");
1115
- lines.push(`CREATE VIRTUAL TABLE ${index.name} USING fts5(${FTS_SOURCE_ID_COLUMN} UNINDEXED, ${index.columns.join(', ')}, tokenize='${tokenize}');`);
1122
+ lines.push(`CREATE VIRTUAL TABLE ${index.name} USING fts5(${FTS_SOURCE_ID_COLUMN} UNINDEXED, ${index.columns.join(', ')}, tokenize='${tokenize}');`, `CREATE TABLE ${ftsMappingTable(index.name)} (id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE);`);
1116
1123
  }
1117
1124
  }
1118
1125
  return lines.join('\n');
1119
1126
  }
1127
+ /** SPEC client-local FTS5 projections: the source-id mapping table whose `id`
1128
+ * equals the projection rowid. */
1129
+ export function ftsMappingTable(index) {
1130
+ return `_syncular_fts_${index}`;
1131
+ }
1132
+ const FTS_LOWERING_STOPS = new Set([
1133
+ 'join',
1134
+ 'inner',
1135
+ 'left',
1136
+ 'right',
1137
+ 'full',
1138
+ 'cross',
1139
+ 'natural',
1140
+ 'where',
1141
+ 'group',
1142
+ 'having',
1143
+ 'order',
1144
+ 'limit',
1145
+ 'window',
1146
+ 'union',
1147
+ 'except',
1148
+ 'intersect',
1149
+ ]);
1150
+ const SELECT_LIST_BOUNDARIES = new Set([
1151
+ 'select',
1152
+ 'from',
1153
+ 'where',
1154
+ 'group',
1155
+ 'having',
1156
+ 'order',
1157
+ 'limit',
1158
+ 'on',
1159
+ 'join',
1160
+ 'window',
1161
+ 'values',
1162
+ ]);
1163
+ /**
1164
+ * SYQL §15 FTS source-id lowering. Reading an FTS projection's
1165
+ * `_syncular_source_id` makes FTS5 fetch the content row of every hit. Each
1166
+ * FTS relation introduced by `FROM` or an inner `JOIN` gains a join to its
1167
+ * mapping table on the projection rowid, and every reference that SQLite binds
1168
+ * to that relation reads the mapping's `source_id` instead. The mapping
1169
+ * columns carry `__syql_fts_` names, so no authored unqualified name changes
1170
+ * its binding. A bare projected reference keeps its result name through an
1171
+ * explicit alias. References this pass cannot bind with certainty keep
1172
+ * reading the projection; both read the same value.
1173
+ */
1174
+ export function lowerFtsSourceIds(sql, ir) {
1175
+ const ftsNames = new Set(ir.tables.flatMap((table) => table.ftsIndexes.map((index) => index.name)));
1176
+ const relations = scanRelations(sql, ir);
1177
+ if (!relations.tables.some((ref) => ftsNames.has(ref.table)))
1178
+ return sql;
1179
+ const tokens = lexSyqlSqlSource('query SQL', sql).filter((token) => !isSyqlTrivia(token) && token.kind !== 'eof');
1180
+ const depths = [];
1181
+ const closes = new Map();
1182
+ const stack = [];
1183
+ tokens.forEach((token, index) => {
1184
+ if (token.text === ')') {
1185
+ const open = stack.pop();
1186
+ if (open !== undefined)
1187
+ closes.set(open, index);
1188
+ }
1189
+ depths[index] = stack.length;
1190
+ if (token.text === '(')
1191
+ stack.push(index);
1192
+ });
1193
+ const lower = (index) => {
1194
+ const token = tokens[index];
1195
+ return token?.kind === 'identifier' ? token.text.toLowerCase() : undefined;
1196
+ };
1197
+ // Every SELECT scope as an offset range; the outer statement spans all.
1198
+ const scopes = [
1199
+ { start: -1, end: sql.length + 1 },
1200
+ ...tokens.flatMap((token, index) => {
1201
+ const close = tokens[closes.get(index) ?? -1];
1202
+ const next = lower(index + 1);
1203
+ return token.text === '(' &&
1204
+ close !== undefined &&
1205
+ (next === 'select' || next === 'with' || next === 'values')
1206
+ ? [{ start: token.span.start.offset, end: close.span.start.offset }]
1207
+ : [];
1208
+ }),
1209
+ ];
1210
+ const scopeAt = (offset) => scopes
1211
+ .filter((scope) => scope.start < offset && offset < scope.end)
1212
+ .reduce((inner, scope) => scope.end - scope.start < inner.end - inner.start ? scope : inner);
1213
+ const tokenAt = (offset) => tokens.findIndex((token) => token.span.start.offset === offset);
1214
+ const bindings = [
1215
+ ...relations.tables.map((ref) => ({ alias: ref.alias, start: ref.start })),
1216
+ ...relations.cteRefs.map((ref) => ({ alias: ref.alias, start: ref.start })),
1217
+ ].map((relation) => ({ ...relation, scope: scopeAt(relation.start) }));
1218
+ const edits = [];
1219
+ const lowered = new Map();
1220
+ for (const ref of relations.tables) {
1221
+ if (!ftsNames.has(ref.table) || ref.nullable)
1222
+ continue;
1223
+ const at = tokenAt(ref.start);
1224
+ const operator = lower(at - 1);
1225
+ if (operator !== 'from' && operator !== 'join')
1226
+ continue;
1227
+ let after = at + 1;
1228
+ if (lower(after) === 'as')
1229
+ after += 1;
1230
+ if (ref.explicitAlias !== undefined)
1231
+ after += 1;
1232
+ const depth = depths[at] ?? 0;
1233
+ let insertAt = tokens[after - 1].span.end.offset;
1234
+ if (lower(after) === 'on') {
1235
+ let cursor = after + 1;
1236
+ while (cursor < tokens.length &&
1237
+ (depths[cursor] ?? 0) >= depth &&
1238
+ !(depths[cursor] === depth &&
1239
+ (tokens[cursor]?.text === ',' ||
1240
+ FTS_LOWERING_STOPS.has(lower(cursor) ?? ''))))
1241
+ cursor += 1;
1242
+ insertAt = tokens[cursor - 1].span.end.offset;
1243
+ }
1244
+ else if (lower(after) === 'using') {
1245
+ const close = tokens[closes.get(after + 1) ?? -1];
1246
+ if (close === undefined)
1247
+ continue;
1248
+ insertAt = close.span.end.offset;
1249
+ }
1250
+ const n = lowered.size + 1;
1251
+ const mapping = `__syql_fts_${n}`;
1252
+ lowered.set(ref.start, `${mapping}.__syql_fts_source_${n}`);
1253
+ edits.push({
1254
+ start: insertAt,
1255
+ end: insertAt,
1256
+ text: ` JOIN (SELECT id AS __syql_fts_rowid_${n}, source_id AS __syql_fts_source_${n} FROM ${ftsMappingTable(ref.table)}) AS ${mapping} ON ${mapping}.__syql_fts_rowid_${n} = ${tokens[ref.explicitAlias === undefined ? at : after - 1].text}.rowid`,
1257
+ });
1258
+ }
1259
+ if (lowered.size === 0)
1260
+ return sql;
1261
+ // An authored `AS _syncular_source_id` alias could capture an unqualified
1262
+ // ORDER BY reference; leave unqualified references alone then.
1263
+ const aliased = tokens.some((token, index) => lower(index - 1) === 'as' &&
1264
+ sqlIdentifier(token)?.toLowerCase() === FTS_SOURCE_ID_COLUMN);
1265
+ for (let index = 0; index < tokens.length; index += 1) {
1266
+ const token = tokens[index];
1267
+ if (sqlIdentifier(token)?.toLowerCase() !== FTS_SOURCE_ID_COLUMN ||
1268
+ lower(index - 1) === 'as' ||
1269
+ tokens[index + 1]?.text === '.' ||
1270
+ tokens[index + 1]?.text === '(')
1271
+ continue;
1272
+ const qualified = tokens[index - 1]?.text === '.';
1273
+ const first = qualified ? index - 2 : index;
1274
+ const firstToken = tokens[first];
1275
+ const scope = scopeAt(token.span.start.offset);
1276
+ let target;
1277
+ if (qualified) {
1278
+ const qualifier = sqlIdentifier(firstToken)?.toLowerCase();
1279
+ const bound = bindings
1280
+ .filter((relation) => relation.alias.toLowerCase() === qualifier &&
1281
+ relation.scope.start < token.span.start.offset &&
1282
+ token.span.start.offset < relation.scope.end)
1283
+ .reduce((inner, relation) => inner === undefined ||
1284
+ relation.scope.end - relation.scope.start <
1285
+ inner.scope.end - inner.scope.start
1286
+ ? relation
1287
+ : inner, undefined);
1288
+ target = bound === undefined ? undefined : lowered.get(bound.start);
1289
+ }
1290
+ else if (!aliased) {
1291
+ const own = bindings.filter((relation) => relation.scope === scope);
1292
+ target = own.length === 1 ? lowered.get(own[0]?.start ?? -1) : undefined;
1293
+ }
1294
+ if (target === undefined)
1295
+ continue;
1296
+ // A bare select-list item keeps its result name.
1297
+ let boundary = first - 1;
1298
+ while (boundary >= 0 &&
1299
+ !(depths[boundary] === depths[first] &&
1300
+ SELECT_LIST_BOUNDARIES.has(lower(boundary) ?? '')) &&
1301
+ (depths[boundary] ?? 0) >= (depths[first] ?? 0))
1302
+ boundary -= 1;
1303
+ const previous = tokens[first - 1]?.text.toLowerCase();
1304
+ const next = tokens[index + 1]?.text.toLowerCase();
1305
+ const bare = lower(boundary) === 'select' &&
1306
+ (previous === ',' ||
1307
+ previous === 'select' ||
1308
+ previous === 'distinct' ||
1309
+ previous === 'all') &&
1310
+ (next === ',' || next === 'from');
1311
+ edits.push({
1312
+ start: firstToken.span.start.offset,
1313
+ end: token.span.end.offset,
1314
+ text: bare ? `${target} AS ${token.text}` : target,
1315
+ });
1316
+ }
1317
+ return edits
1318
+ .sort((a, b) => b.start - a.start || b.end - a.end)
1319
+ .reduce((text, edit) => text.slice(0, edit.start) + edit.text + text.slice(edit.end), sql);
1320
+ }
1120
1321
  /**
1121
1322
  * Analyze ONE already-split statement against the IR + a prepared-DDL DB. The
1122
1323
  * name is pre-resolved (path-derived or a `-- name:` override) by the caller;
@@ -1287,17 +1488,28 @@ export function analyzeStatement(name, location, statementText, ir, db, naming =
1287
1488
  }
1288
1489
  }
1289
1490
  const reactive = inferReactiveMetadata(sql, refs, ir, params, columns);
1491
+ const physical = lowerFtsSourceIds(sql, ir);
1492
+ if (physical !== sql) {
1493
+ const lowered = analyze(physical);
1494
+ if (lowered.paramsCount !== described.paramsCount ||
1495
+ lowered.columnNames.length !== described.columnNames.length ||
1496
+ lowered.columnNames.some((n, i) => n !== described.columnNames[i])) {
1497
+ throw new TypegenError(file, `internal: FTS source-id lowering changed the result columns [${described.columnNames.join(', ')}] to [${lowered.columnNames.join(', ')}]`);
1498
+ }
1499
+ }
1290
1500
  return {
1291
1501
  name,
1292
1502
  file,
1293
1503
  sourceSql,
1294
- sql,
1295
- positionalSql: toPositionalSql(sql),
1296
- relations: scanTableRefs(toPositionalSql(sql), ir).map((ref) => ({
1504
+ sql: physical,
1505
+ positionalSql: toPositionalSql(physical),
1506
+ relations: scanTableRefs(toPositionalSql(physical), ir, true).map((ref) => ({
1297
1507
  table: ref.table,
1298
1508
  start: ref.start,
1299
1509
  end: ref.end,
1300
- ...(ref.explicitAlias === undefined ? {} : { alias: ref.explicitAlias }),
1510
+ ...(ref.explicitAlias === undefined
1511
+ ? {}
1512
+ : { alias: ref.explicitAlias }),
1301
1513
  })),
1302
1514
  params,
1303
1515
  columns,
@@ -303,16 +303,18 @@ class Validator {
303
303
  const activeSql = this.#render(logical.template, 'active', []);
304
304
  const activeStructure = this.#inspect(activeSql, location);
305
305
  this.#validateStatementShape(activeSql, activeStructure, logical.declaration, location);
306
- this.#validateDeterminism(activeSql, logical.declaration.statement.span);
307
306
  let relations;
308
307
  try {
309
308
  relations = scanRelations(activeSql, this.#ir);
310
309
  }
311
310
  catch (error) {
311
+ this.#validateDeterminism(activeSql, logical.declaration.statement.span);
312
312
  // Preserve SQLite's source-spanned diagnostics for invalid relations.
313
313
  this.#validateSqlite(activeSql, logical, []);
314
314
  this.#fail('SYQL6002_INVALID_SQL', logical.declaration.statement.span, error instanceof Error ? error.message : String(error));
315
315
  }
316
+ const boundedCtes = this.#boundedCtes(activeSql, relations, location);
317
+ this.#validateDeterminism(activeSql, logical.declaration.statement.span, new Set(boundedCtes.values()));
316
318
  const refs = relations.tables;
317
319
  this.#validatePortableProfile(activeSql, logical.declaration.statement.span, refs);
318
320
  this.#validateSqlite(activeSql, logical, refs);
@@ -363,7 +365,7 @@ class Validator {
363
365
  }
364
366
  }
365
367
  const reactive = this.#inferReactive(logical, markerSql, relations, bindSymbols);
366
- const identityTerms = this.#proveIdentity(logical.declaration, activeSql, relations, analysis);
368
+ const identityTerms = this.#proveIdentity(logical.declaration, activeSql, relations, analysis, boundedCtes);
367
369
  const identity = identityTerms?.map((term) => term.field);
368
370
  this.#validateStableOrder(logical.declaration, activeSql, activeStructure, sort, identityTerms);
369
371
  const reactiveWithIdentity = {
@@ -538,7 +540,7 @@ class Validator {
538
540
  this.#fail('SYQL6002_INVALID_SQL', query.statement.span, 'empty SQL statement');
539
541
  }
540
542
  }
541
- #validateDeterminism(sql, span) {
543
+ #validateDeterminism(sql, span, boundedLimits = new Set()) {
542
544
  const tokens = significant(lexSyqlSqlSource(span.file, sql));
543
545
  const functionStack = [];
544
546
  let depth = 0;
@@ -548,7 +550,8 @@ class Validator {
548
550
  const next = tokens[index + 1];
549
551
  if (depth > 0 &&
550
552
  token.kind === 'identifier' &&
551
- (lower === 'limit' || lower === 'offset')) {
553
+ (lower === 'limit' || lower === 'offset') &&
554
+ !boundedLimits.has(token.span.start.offset)) {
552
555
  this.#fail('SYQL6003_NONDETERMINISTIC_SQL', token.span, `nested ${token.text.toUpperCase()} is rejected until its local row identity and total order can be proven`);
553
556
  }
554
557
  if (token.kind === 'identifier' && lower === 'over') {
@@ -1317,13 +1320,41 @@ class Validator {
1317
1320
  .slice(insertion)
1318
1321
  .trimStart()}`.trim();
1319
1322
  }
1323
+ /**
1324
+ * SYQL §14.3 bounded narrow stage: a top-level CTE body may end in
1325
+ * `LIMIT <integer>`. Returns the offset of each such LIMIT by CTE name;
1326
+ * {@link #proveIdentity} then requires the body's total order.
1327
+ */
1328
+ #boundedCtes(sql, relations, location) {
1329
+ const bounded = new Map();
1330
+ for (const cte of relations.cteScopes) {
1331
+ const tokens = significant(lexSyqlSqlSource(location, sql.slice(cte.start, cte.end))).filter((token) => token.kind !== 'eof');
1332
+ let depth = 0;
1333
+ for (const [index, token] of tokens.entries()) {
1334
+ if (token.text === ')')
1335
+ depth -= 1;
1336
+ if (token.text === '(')
1337
+ depth += 1;
1338
+ if (depth !== 0 || tokenLower(token) !== 'limit')
1339
+ continue;
1340
+ const size = tokens[index + 1];
1341
+ if (size?.kind === 'number' &&
1342
+ /^[1-9][0-9]*$/.test(size.text) &&
1343
+ index + 2 === tokens.length) {
1344
+ bounded.set(cte.name, cte.start + token.span.start.offset);
1345
+ }
1346
+ }
1347
+ }
1348
+ return bounded;
1349
+ }
1320
1350
  /**
1321
1351
  * Proves the result identity per SELECT scope. Each physical relation of a
1322
1352
  * scope contributes its primary key and each CTE relation the identity of
1323
1353
  * its body, except that a physical relation joined by a required inner
1324
1354
  * `ON rel.pk = cte.key` equality to a CTE is determined by that CTE row.
1325
1355
  */
1326
- #proveIdentity(query, sql, relations, analysis) {
1356
+ #proveIdentity(query, sql, relations, analysis, boundedCtes) {
1357
+ const unproven = () => this.#fail('SYQL6003_NONDETERMINISTIC_SQL', query.statement.span, 'a CTE LIMIT requires a proven CTE identity and a body ORDER BY that ends with it');
1327
1358
  const resultNames = new Set();
1328
1359
  for (const column of analysis.columns) {
1329
1360
  if (!IDENT_RE.test(column.name)) {
@@ -1369,6 +1400,8 @@ class Validator {
1369
1400
  if (relations.recursive ||
1370
1401
  (cleaned.match(/\bSELECT\b/gi)?.length ?? 0) !==
1371
1402
  relations.cteScopes.length + 1) {
1403
+ if (boundedCtes.size > 0)
1404
+ unproven();
1372
1405
  return undefined;
1373
1406
  }
1374
1407
  const cteKeys = new Map();
@@ -1483,13 +1516,41 @@ class Validator {
1483
1516
  return keys;
1484
1517
  };
1485
1518
  for (const cte of relations.cteScopes) {
1486
- if (cte.renamed)
1519
+ const bounded = boundedCtes.get(cte.name);
1520
+ if (cte.renamed) {
1521
+ if (bounded !== undefined)
1522
+ unproven();
1487
1523
  continue;
1524
+ }
1488
1525
  const text = cleaned.slice(cte.start, cte.end);
1489
1526
  const keys = scopeKeys(`cte:${cte.name}`, text);
1490
1527
  const items = splitSelectList(text);
1491
- if (keys === undefined || items === null)
1528
+ if (keys === undefined || items === null) {
1529
+ if (bounded !== undefined)
1530
+ unproven();
1492
1531
  continue;
1532
+ }
1533
+ if (bounded !== undefined) {
1534
+ // The body order is total when it ends with the body identity, each
1535
+ // term spelled as the identity's result alias or projected column.
1536
+ const order = /\border\s+by\b/i.exec(text);
1537
+ const limit = bounded - cte.start;
1538
+ if (order === null || order.index > limit)
1539
+ unproven();
1540
+ const terms = this.#splitOrderTerms(text.slice((order?.index ?? 0) + (order?.[0].length ?? 0), limit), query.statement.span);
1541
+ const suffix = terms.slice(terms.length - keys.length);
1542
+ if (terms.length < keys.length ||
1543
+ keys.some(({ index }, position) => {
1544
+ const item = items[index];
1545
+ const term = (suffix[position] ?? '')
1546
+ .replace(/\s+(?:asc|desc)$/i, '')
1547
+ .replace(/\s+/g, '')
1548
+ .toLowerCase();
1549
+ return (term !== item.expr.replace(/\s+/g, '').toLowerCase() &&
1550
+ term !== item.alias?.toLowerCase());
1551
+ }))
1552
+ unproven();
1553
+ }
1493
1554
  cteKeys.set(cte.name, keys.map(({ index }) => {
1494
1555
  const item = items[index];
1495
1556
  return item.alias ?? PLAIN_REF_RE.exec(item.expr)?.[2];
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/typegen",
3
- "version": "0.28.0",
3
+ "version": "0.29.0",
4
4
  "description": "Syncular schema-to-TypeScript type generator and the `syncular` CLI",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -56,9 +56,9 @@
56
56
  "!dist/**/*.test.d.ts"
57
57
  ],
58
58
  "dependencies": {
59
- "@syncular/core": "0.28.0"
59
+ "@syncular/core": "0.29.0"
60
60
  },
61
61
  "devDependencies": {
62
- "@syncular/server": "0.28.0"
62
+ "@syncular/server": "0.29.0"
63
63
  }
64
64
  }
package/src/query.ts CHANGED
@@ -778,11 +778,21 @@ export interface SqlRelations {
778
778
  readonly scopeAt: (offset: number) => string;
779
779
  }
780
780
 
781
- export function scanTableRefs(sql: string, ir: IrDocument): TableRef[] {
782
- return scanRelations(sql, ir).tables;
781
+ export function scanTableRefs(
782
+ sql: string,
783
+ ir: IrDocument,
784
+ lowered = false,
785
+ ): TableRef[] {
786
+ return scanRelations(sql, ir, lowered).tables;
783
787
  }
784
788
 
785
- export function scanRelations(sql: string, ir: IrDocument): SqlRelations {
789
+ /** `lowered` scans emitted SQL, whose FTS mapping reads
790
+ * ({@link lowerFtsSourceIds}) are internal and not relations of the query. */
791
+ export function scanRelations(
792
+ sql: string,
793
+ ir: IrDocument,
794
+ lowered = false,
795
+ ): SqlRelations {
786
796
  const tokens = lexSyqlSqlSource('query SQL', sql).filter(
787
797
  (token) => !isSyqlTrivia(token) && token.kind !== 'eof',
788
798
  );
@@ -794,6 +804,15 @@ export function scanRelations(sql: string, ir: IrDocument): SqlRelations {
794
804
  ),
795
805
  ]),
796
806
  );
807
+ const mappings = new Set(
808
+ lowered
809
+ ? ir.tables.flatMap((table) =>
810
+ table.ftsIndexes.map((index) =>
811
+ ftsMappingTable(index.name).toLowerCase(),
812
+ ),
813
+ )
814
+ : [],
815
+ );
797
816
  const closes = new Map<number, number>();
798
817
  const parents: number[] = [];
799
818
  const depths: number[] = [];
@@ -992,6 +1011,7 @@ export function scanRelations(sql: string, ir: IrDocument): SqlRelations {
992
1011
  });
993
1012
  continue;
994
1013
  }
1014
+ if (mappings.has(rawTable.toLowerCase())) continue;
995
1015
  const table = known.get(rawTable.toLowerCase());
996
1016
  if (table === undefined)
997
1017
  throw new TypegenError(
@@ -1582,12 +1602,233 @@ export function synthesizeDdl(ir: IrDocument): string {
1582
1602
  const tokenize = index.tokenize.replaceAll("'", "''");
1583
1603
  lines.push(
1584
1604
  `CREATE VIRTUAL TABLE ${index.name} USING fts5(${FTS_SOURCE_ID_COLUMN} UNINDEXED, ${index.columns.join(', ')}, tokenize='${tokenize}');`,
1605
+ `CREATE TABLE ${ftsMappingTable(index.name)} (id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE);`,
1585
1606
  );
1586
1607
  }
1587
1608
  }
1588
1609
  return lines.join('\n');
1589
1610
  }
1590
1611
 
1612
+ /** SPEC client-local FTS5 projections: the source-id mapping table whose `id`
1613
+ * equals the projection rowid. */
1614
+ export function ftsMappingTable(index: string): string {
1615
+ return `_syncular_fts_${index}`;
1616
+ }
1617
+
1618
+ const FTS_LOWERING_STOPS = new Set([
1619
+ 'join',
1620
+ 'inner',
1621
+ 'left',
1622
+ 'right',
1623
+ 'full',
1624
+ 'cross',
1625
+ 'natural',
1626
+ 'where',
1627
+ 'group',
1628
+ 'having',
1629
+ 'order',
1630
+ 'limit',
1631
+ 'window',
1632
+ 'union',
1633
+ 'except',
1634
+ 'intersect',
1635
+ ]);
1636
+ const SELECT_LIST_BOUNDARIES = new Set([
1637
+ 'select',
1638
+ 'from',
1639
+ 'where',
1640
+ 'group',
1641
+ 'having',
1642
+ 'order',
1643
+ 'limit',
1644
+ 'on',
1645
+ 'join',
1646
+ 'window',
1647
+ 'values',
1648
+ ]);
1649
+
1650
+ /**
1651
+ * SYQL §15 FTS source-id lowering. Reading an FTS projection's
1652
+ * `_syncular_source_id` makes FTS5 fetch the content row of every hit. Each
1653
+ * FTS relation introduced by `FROM` or an inner `JOIN` gains a join to its
1654
+ * mapping table on the projection rowid, and every reference that SQLite binds
1655
+ * to that relation reads the mapping's `source_id` instead. The mapping
1656
+ * columns carry `__syql_fts_` names, so no authored unqualified name changes
1657
+ * its binding. A bare projected reference keeps its result name through an
1658
+ * explicit alias. References this pass cannot bind with certainty keep
1659
+ * reading the projection; both read the same value.
1660
+ */
1661
+ export function lowerFtsSourceIds(sql: string, ir: IrDocument): string {
1662
+ const ftsNames = new Set(
1663
+ ir.tables.flatMap((table) => table.ftsIndexes.map((index) => index.name)),
1664
+ );
1665
+ const relations = scanRelations(sql, ir);
1666
+ if (!relations.tables.some((ref) => ftsNames.has(ref.table))) return sql;
1667
+ const tokens = lexSyqlSqlSource('query SQL', sql).filter(
1668
+ (token) => !isSyqlTrivia(token) && token.kind !== 'eof',
1669
+ );
1670
+ const depths: number[] = [];
1671
+ const closes = new Map<number, number>();
1672
+ const stack: number[] = [];
1673
+ tokens.forEach((token, index) => {
1674
+ if (token.text === ')') {
1675
+ const open = stack.pop();
1676
+ if (open !== undefined) closes.set(open, index);
1677
+ }
1678
+ depths[index] = stack.length;
1679
+ if (token.text === '(') stack.push(index);
1680
+ });
1681
+ const lower = (index: number): string | undefined => {
1682
+ const token = tokens[index];
1683
+ return token?.kind === 'identifier' ? token.text.toLowerCase() : undefined;
1684
+ };
1685
+ // Every SELECT scope as an offset range; the outer statement spans all.
1686
+ const scopes = [
1687
+ { start: -1, end: sql.length + 1 },
1688
+ ...tokens.flatMap((token, index) => {
1689
+ const close = tokens[closes.get(index) ?? -1];
1690
+ const next = lower(index + 1);
1691
+ return token.text === '(' &&
1692
+ close !== undefined &&
1693
+ (next === 'select' || next === 'with' || next === 'values')
1694
+ ? [{ start: token.span.start.offset, end: close.span.start.offset }]
1695
+ : [];
1696
+ }),
1697
+ ];
1698
+ const scopeAt = (offset: number) =>
1699
+ scopes
1700
+ .filter((scope) => scope.start < offset && offset < scope.end)
1701
+ .reduce((inner, scope) =>
1702
+ scope.end - scope.start < inner.end - inner.start ? scope : inner,
1703
+ );
1704
+ const tokenAt = (offset: number) =>
1705
+ tokens.findIndex((token) => token.span.start.offset === offset);
1706
+ const bindings = [
1707
+ ...relations.tables.map((ref) => ({ alias: ref.alias, start: ref.start })),
1708
+ ...relations.cteRefs.map((ref) => ({ alias: ref.alias, start: ref.start })),
1709
+ ].map((relation) => ({ ...relation, scope: scopeAt(relation.start) }));
1710
+
1711
+ const edits: { start: number; end: number; text: string }[] = [];
1712
+ const lowered = new Map<number, string>();
1713
+ for (const ref of relations.tables) {
1714
+ if (!ftsNames.has(ref.table) || ref.nullable) continue;
1715
+ const at = tokenAt(ref.start);
1716
+ const operator = lower(at - 1);
1717
+ if (operator !== 'from' && operator !== 'join') continue;
1718
+ let after = at + 1;
1719
+ if (lower(after) === 'as') after += 1;
1720
+ if (ref.explicitAlias !== undefined) after += 1;
1721
+ const depth = depths[at] ?? 0;
1722
+ let insertAt = (tokens[after - 1] as SyqlToken).span.end.offset;
1723
+ if (lower(after) === 'on') {
1724
+ let cursor = after + 1;
1725
+ while (
1726
+ cursor < tokens.length &&
1727
+ (depths[cursor] ?? 0) >= depth &&
1728
+ !(
1729
+ depths[cursor] === depth &&
1730
+ (tokens[cursor]?.text === ',' ||
1731
+ FTS_LOWERING_STOPS.has(lower(cursor) ?? ''))
1732
+ )
1733
+ )
1734
+ cursor += 1;
1735
+ insertAt = (tokens[cursor - 1] as SyqlToken).span.end.offset;
1736
+ } else if (lower(after) === 'using') {
1737
+ const close = tokens[closes.get(after + 1) ?? -1];
1738
+ if (close === undefined) continue;
1739
+ insertAt = close.span.end.offset;
1740
+ }
1741
+ const n = lowered.size + 1;
1742
+ const mapping = `__syql_fts_${n}`;
1743
+ lowered.set(ref.start, `${mapping}.__syql_fts_source_${n}`);
1744
+ edits.push({
1745
+ start: insertAt,
1746
+ end: insertAt,
1747
+ text: ` JOIN (SELECT id AS __syql_fts_rowid_${n}, source_id AS __syql_fts_source_${n} FROM ${ftsMappingTable(ref.table)}) AS ${mapping} ON ${mapping}.__syql_fts_rowid_${n} = ${(tokens[ref.explicitAlias === undefined ? at : after - 1] as SyqlToken).text}.rowid`,
1748
+ });
1749
+ }
1750
+ if (lowered.size === 0) return sql;
1751
+
1752
+ // An authored `AS _syncular_source_id` alias could capture an unqualified
1753
+ // ORDER BY reference; leave unqualified references alone then.
1754
+ const aliased = tokens.some(
1755
+ (token, index) =>
1756
+ lower(index - 1) === 'as' &&
1757
+ sqlIdentifier(token)?.toLowerCase() === FTS_SOURCE_ID_COLUMN,
1758
+ );
1759
+ for (let index = 0; index < tokens.length; index += 1) {
1760
+ const token = tokens[index] as SyqlToken;
1761
+ if (
1762
+ sqlIdentifier(token)?.toLowerCase() !== FTS_SOURCE_ID_COLUMN ||
1763
+ lower(index - 1) === 'as' ||
1764
+ tokens[index + 1]?.text === '.' ||
1765
+ tokens[index + 1]?.text === '('
1766
+ )
1767
+ continue;
1768
+ const qualified = tokens[index - 1]?.text === '.';
1769
+ const first = qualified ? index - 2 : index;
1770
+ const firstToken = tokens[first] as SyqlToken;
1771
+ const scope = scopeAt(token.span.start.offset);
1772
+ let target: string | undefined;
1773
+ if (qualified) {
1774
+ const qualifier = sqlIdentifier(firstToken)?.toLowerCase();
1775
+ const bound = bindings
1776
+ .filter(
1777
+ (relation) =>
1778
+ relation.alias.toLowerCase() === qualifier &&
1779
+ relation.scope.start < token.span.start.offset &&
1780
+ token.span.start.offset < relation.scope.end,
1781
+ )
1782
+ .reduce<(typeof bindings)[number] | undefined>(
1783
+ (inner, relation) =>
1784
+ inner === undefined ||
1785
+ relation.scope.end - relation.scope.start <
1786
+ inner.scope.end - inner.scope.start
1787
+ ? relation
1788
+ : inner,
1789
+ undefined,
1790
+ );
1791
+ target = bound === undefined ? undefined : lowered.get(bound.start);
1792
+ } else if (!aliased) {
1793
+ const own = bindings.filter((relation) => relation.scope === scope);
1794
+ target = own.length === 1 ? lowered.get(own[0]?.start ?? -1) : undefined;
1795
+ }
1796
+ if (target === undefined) continue;
1797
+ // A bare select-list item keeps its result name.
1798
+ let boundary = first - 1;
1799
+ while (
1800
+ boundary >= 0 &&
1801
+ !(
1802
+ depths[boundary] === depths[first] &&
1803
+ SELECT_LIST_BOUNDARIES.has(lower(boundary) ?? '')
1804
+ ) &&
1805
+ (depths[boundary] ?? 0) >= (depths[first] ?? 0)
1806
+ )
1807
+ boundary -= 1;
1808
+ const previous = tokens[first - 1]?.text.toLowerCase();
1809
+ const next = tokens[index + 1]?.text.toLowerCase();
1810
+ const bare =
1811
+ lower(boundary) === 'select' &&
1812
+ (previous === ',' ||
1813
+ previous === 'select' ||
1814
+ previous === 'distinct' ||
1815
+ previous === 'all') &&
1816
+ (next === ',' || next === 'from');
1817
+ edits.push({
1818
+ start: firstToken.span.start.offset,
1819
+ end: token.span.end.offset,
1820
+ text: bare ? `${target} AS ${token.text}` : target,
1821
+ });
1822
+ }
1823
+ return edits
1824
+ .sort((a, b) => b.start - a.start || b.end - a.end)
1825
+ .reduce(
1826
+ (text, edit) =>
1827
+ text.slice(0, edit.start) + edit.text + text.slice(edit.end),
1828
+ sql,
1829
+ );
1830
+ }
1831
+
1591
1832
  /**
1592
1833
  * Analyze ONE already-split statement against the IR + a prepared-DDL DB. The
1593
1834
  * name is pre-resolved (path-derived or a `-- name:` override) by the caller;
@@ -1840,18 +2081,37 @@ export function analyzeStatement(
1840
2081
 
1841
2082
  const reactive = inferReactiveMetadata(sql, refs, ir, params, columns);
1842
2083
 
2084
+ const physical = lowerFtsSourceIds(sql, ir);
2085
+ if (physical !== sql) {
2086
+ const lowered = analyze(physical);
2087
+ if (
2088
+ lowered.paramsCount !== described.paramsCount ||
2089
+ lowered.columnNames.length !== described.columnNames.length ||
2090
+ lowered.columnNames.some((n, i) => n !== described.columnNames[i])
2091
+ ) {
2092
+ throw new TypegenError(
2093
+ file,
2094
+ `internal: FTS source-id lowering changed the result columns [${described.columnNames.join(', ')}] to [${lowered.columnNames.join(', ')}]`,
2095
+ );
2096
+ }
2097
+ }
2098
+
1843
2099
  return {
1844
2100
  name,
1845
2101
  file,
1846
2102
  sourceSql,
1847
- sql,
1848
- positionalSql: toPositionalSql(sql),
1849
- relations: scanTableRefs(toPositionalSql(sql), ir).map((ref) => ({
1850
- table: ref.table,
1851
- start: ref.start,
1852
- end: ref.end,
1853
- ...(ref.explicitAlias === undefined ? {} : { alias: ref.explicitAlias }),
1854
- })),
2103
+ sql: physical,
2104
+ positionalSql: toPositionalSql(physical),
2105
+ relations: scanTableRefs(toPositionalSql(physical), ir, true).map(
2106
+ (ref) => ({
2107
+ table: ref.table,
2108
+ start: ref.start,
2109
+ end: ref.end,
2110
+ ...(ref.explicitAlias === undefined
2111
+ ? {}
2112
+ : { alias: ref.explicitAlias }),
2113
+ }),
2114
+ ),
1855
2115
  params,
1856
2116
  columns,
1857
2117
  tables,
@@ -465,11 +465,11 @@ class Validator {
465
465
  logical.declaration,
466
466
  location,
467
467
  );
468
- this.#validateDeterminism(activeSql, logical.declaration.statement.span);
469
468
  let relations: SqlRelations;
470
469
  try {
471
470
  relations = scanRelations(activeSql, this.#ir);
472
471
  } catch (error) {
472
+ this.#validateDeterminism(activeSql, logical.declaration.statement.span);
473
473
  // Preserve SQLite's source-spanned diagnostics for invalid relations.
474
474
  this.#validateSqlite(activeSql, logical, []);
475
475
  this.#fail(
@@ -478,6 +478,12 @@ class Validator {
478
478
  error instanceof Error ? error.message : String(error),
479
479
  );
480
480
  }
481
+ const boundedCtes = this.#boundedCtes(activeSql, relations, location);
482
+ this.#validateDeterminism(
483
+ activeSql,
484
+ logical.declaration.statement.span,
485
+ new Set(boundedCtes.values()),
486
+ );
481
487
  const refs = relations.tables;
482
488
  this.#validatePortableProfile(
483
489
  activeSql,
@@ -584,6 +590,7 @@ class Validator {
584
590
  activeSql,
585
591
  relations,
586
592
  analysis,
593
+ boundedCtes,
587
594
  );
588
595
  const identity = identityTerms?.map((term) => term.field);
589
596
  this.#validateStableOrder(
@@ -830,7 +837,11 @@ class Validator {
830
837
  }
831
838
  }
832
839
 
833
- #validateDeterminism(sql: string, span: SyqlSourceSpan): void {
840
+ #validateDeterminism(
841
+ sql: string,
842
+ span: SyqlSourceSpan,
843
+ boundedLimits: ReadonlySet<number> = new Set(),
844
+ ): void {
834
845
  const tokens = significant(lexSyqlSqlSource(span.file, sql));
835
846
  const functionStack: Array<string | undefined> = [];
836
847
  let depth = 0;
@@ -841,7 +852,8 @@ class Validator {
841
852
  if (
842
853
  depth > 0 &&
843
854
  token.kind === 'identifier' &&
844
- (lower === 'limit' || lower === 'offset')
855
+ (lower === 'limit' || lower === 'offset') &&
856
+ !boundedLimits.has(token.span.start.offset)
845
857
  ) {
846
858
  this.#fail(
847
859
  'SYQL6003_NONDETERMINISTIC_SQL',
@@ -1944,6 +1956,39 @@ class Validator {
1944
1956
  .trimStart()}`.trim();
1945
1957
  }
1946
1958
 
1959
+ /**
1960
+ * SYQL §14.3 bounded narrow stage: a top-level CTE body may end in
1961
+ * `LIMIT <integer>`. Returns the offset of each such LIMIT by CTE name;
1962
+ * {@link #proveIdentity} then requires the body's total order.
1963
+ */
1964
+ #boundedCtes(
1965
+ sql: string,
1966
+ relations: SqlRelations,
1967
+ location: string,
1968
+ ): ReadonlyMap<string, number> {
1969
+ const bounded = new Map<string, number>();
1970
+ for (const cte of relations.cteScopes) {
1971
+ const tokens = significant(
1972
+ lexSyqlSqlSource(location, sql.slice(cte.start, cte.end)),
1973
+ ).filter((token) => token.kind !== 'eof');
1974
+ let depth = 0;
1975
+ for (const [index, token] of tokens.entries()) {
1976
+ if (token.text === ')') depth -= 1;
1977
+ if (token.text === '(') depth += 1;
1978
+ if (depth !== 0 || tokenLower(token) !== 'limit') continue;
1979
+ const size = tokens[index + 1];
1980
+ if (
1981
+ size?.kind === 'number' &&
1982
+ /^[1-9][0-9]*$/.test(size.text) &&
1983
+ index + 2 === tokens.length
1984
+ ) {
1985
+ bounded.set(cte.name, cte.start + token.span.start.offset);
1986
+ }
1987
+ }
1988
+ }
1989
+ return bounded;
1990
+ }
1991
+
1947
1992
  /**
1948
1993
  * Proves the result identity per SELECT scope. Each physical relation of a
1949
1994
  * scope contributes its primary key and each CTE relation the identity of
@@ -1955,7 +2000,14 @@ class Validator {
1955
2000
  sql: string,
1956
2001
  relations: SqlRelations,
1957
2002
  analysis: AnalyzedQuery,
2003
+ boundedCtes: ReadonlyMap<string, number>,
1958
2004
  ): readonly IdentityTerm[] | undefined {
2005
+ const unproven = (): never =>
2006
+ this.#fail(
2007
+ 'SYQL6003_NONDETERMINISTIC_SQL',
2008
+ query.statement.span,
2009
+ 'a CTE LIMIT requires a proven CTE identity and a body ORDER BY that ends with it',
2010
+ );
1959
2011
  const resultNames = new Set<string>();
1960
2012
  for (const column of analysis.columns) {
1961
2013
  if (!IDENT_RE.test(column.name)) {
@@ -2020,6 +2072,7 @@ class Validator {
2020
2072
  (cleaned.match(/\bSELECT\b/gi)?.length ?? 0) !==
2021
2073
  relations.cteScopes.length + 1
2022
2074
  ) {
2075
+ if (boundedCtes.size > 0) unproven();
2023
2076
  return undefined;
2024
2077
  }
2025
2078
  const cteKeys = new Map<string, readonly string[]>();
@@ -2149,11 +2202,45 @@ class Validator {
2149
2202
  return keys;
2150
2203
  };
2151
2204
  for (const cte of relations.cteScopes) {
2152
- if (cte.renamed) continue;
2205
+ const bounded = boundedCtes.get(cte.name);
2206
+ if (cte.renamed) {
2207
+ if (bounded !== undefined) unproven();
2208
+ continue;
2209
+ }
2153
2210
  const text = cleaned.slice(cte.start, cte.end);
2154
2211
  const keys = scopeKeys(`cte:${cte.name}`, text);
2155
2212
  const items = splitSelectList(text);
2156
- if (keys === undefined || items === null) continue;
2213
+ if (keys === undefined || items === null) {
2214
+ if (bounded !== undefined) unproven();
2215
+ continue;
2216
+ }
2217
+ if (bounded !== undefined) {
2218
+ // The body order is total when it ends with the body identity, each
2219
+ // term spelled as the identity's result alias or projected column.
2220
+ const order = /\border\s+by\b/i.exec(text);
2221
+ const limit = bounded - cte.start;
2222
+ if (order === null || order.index > limit) unproven();
2223
+ const terms = this.#splitOrderTerms(
2224
+ text.slice((order?.index ?? 0) + (order?.[0].length ?? 0), limit),
2225
+ query.statement.span,
2226
+ );
2227
+ const suffix = terms.slice(terms.length - keys.length);
2228
+ if (
2229
+ terms.length < keys.length ||
2230
+ keys.some(({ index }, position) => {
2231
+ const item = items[index] as SelectItem;
2232
+ const term = (suffix[position] ?? '')
2233
+ .replace(/\s+(?:asc|desc)$/i, '')
2234
+ .replace(/\s+/g, '')
2235
+ .toLowerCase();
2236
+ return (
2237
+ term !== item.expr.replace(/\s+/g, '').toLowerCase() &&
2238
+ term !== item.alias?.toLowerCase()
2239
+ );
2240
+ })
2241
+ )
2242
+ unproven();
2243
+ }
2157
2244
  cteKeys.set(
2158
2245
  cte.name,
2159
2246
  keys.map(({ index }) => {