latticesql 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +67 -12
- package/dist/cli.js +389 -32
- package/dist/index.cjs +375 -26
- package/dist/index.d.cts +33 -1
- package/dist/index.d.ts +33 -1
- package/dist/index.js +389 -32
- package/package.json +1 -1
package/dist/cli.js
CHANGED
|
@@ -321,7 +321,15 @@ import { join as join3 } from "path";
|
|
|
321
321
|
import { existsSync as existsSync3, readFileSync as readFileSync3 } from "fs";
|
|
322
322
|
|
|
323
323
|
// src/render/writer.ts
|
|
324
|
-
import {
|
|
324
|
+
import {
|
|
325
|
+
writeFileSync as writeFileSync2,
|
|
326
|
+
mkdirSync as mkdirSync2,
|
|
327
|
+
renameSync,
|
|
328
|
+
copyFileSync,
|
|
329
|
+
unlinkSync,
|
|
330
|
+
existsSync as existsSync2,
|
|
331
|
+
readFileSync as readFileSync2
|
|
332
|
+
} from "fs";
|
|
325
333
|
import { createHash } from "crypto";
|
|
326
334
|
import { dirname as dirname3, join as join2 } from "path";
|
|
327
335
|
import { tmpdir } from "os";
|
|
@@ -334,7 +342,16 @@ function atomicWrite(filePath, content) {
|
|
|
334
342
|
if (currentHash === newHash) return false;
|
|
335
343
|
const tmp = join2(tmpdir(), `lattice-${randomBytes(8).toString("hex")}.tmp`);
|
|
336
344
|
writeFileSync2(tmp, content, "utf8");
|
|
337
|
-
|
|
345
|
+
try {
|
|
346
|
+
renameSync(tmp, filePath);
|
|
347
|
+
} catch (err) {
|
|
348
|
+
if (err.code === "EXDEV") {
|
|
349
|
+
copyFileSync(tmp, filePath);
|
|
350
|
+
unlinkSync(tmp);
|
|
351
|
+
} else {
|
|
352
|
+
throw err;
|
|
353
|
+
}
|
|
354
|
+
}
|
|
338
355
|
return true;
|
|
339
356
|
}
|
|
340
357
|
function existingHash(filePath) {
|
|
@@ -380,15 +397,21 @@ function writeManifest(outputDir, manifest) {
|
|
|
380
397
|
|
|
381
398
|
// src/db/sqlite.ts
|
|
382
399
|
import Database from "better-sqlite3";
|
|
400
|
+
var DDL_RE = /^\s*(CREATE|ALTER|DROP|PRAGMA)\b/i;
|
|
401
|
+
var DEFAULT_CACHE_MAX = 500;
|
|
383
402
|
var SQLiteAdapter = class {
|
|
384
403
|
_db = null;
|
|
385
404
|
_path;
|
|
386
405
|
_wal;
|
|
387
406
|
_busyTimeout;
|
|
407
|
+
/** Cached prepared statements keyed by SQL string. */
|
|
408
|
+
_stmtCache = /* @__PURE__ */ new Map();
|
|
409
|
+
_cacheMax;
|
|
388
410
|
constructor(path, options) {
|
|
389
411
|
this._path = path;
|
|
390
412
|
this._wal = options?.wal ?? true;
|
|
391
413
|
this._busyTimeout = options?.busyTimeout ?? 5e3;
|
|
414
|
+
this._cacheMax = options?.cacheMax ?? DEFAULT_CACHE_MAX;
|
|
392
415
|
}
|
|
393
416
|
get db() {
|
|
394
417
|
if (!this._db) throw new Error("SQLiteAdapter: not open \u2014 call open() first");
|
|
@@ -402,20 +425,46 @@ var SQLiteAdapter = class {
|
|
|
402
425
|
}
|
|
403
426
|
}
|
|
404
427
|
close() {
|
|
428
|
+
this._stmtCache.clear();
|
|
405
429
|
this._db?.close();
|
|
406
430
|
this._db = null;
|
|
407
431
|
}
|
|
432
|
+
/** Clear the prepared-statement cache. Call after DDL changes (schema/migrations). */
|
|
433
|
+
clearStatementCache() {
|
|
434
|
+
this._stmtCache.clear();
|
|
435
|
+
}
|
|
436
|
+
/**
|
|
437
|
+
* Return a cached prepared statement for the given SQL, or compile and cache a new one.
|
|
438
|
+
* DDL statements bypass the cache entirely.
|
|
439
|
+
*/
|
|
440
|
+
_cachedPrepare(sql) {
|
|
441
|
+
if (DDL_RE.test(sql)) {
|
|
442
|
+
return this.db.prepare(sql);
|
|
443
|
+
}
|
|
444
|
+
let stmt = this._stmtCache.get(sql);
|
|
445
|
+
if (stmt) return stmt;
|
|
446
|
+
stmt = this.db.prepare(sql);
|
|
447
|
+
if (this._stmtCache.size >= this._cacheMax) {
|
|
448
|
+
this._stmtCache.clear();
|
|
449
|
+
}
|
|
450
|
+
this._stmtCache.set(sql, stmt);
|
|
451
|
+
return stmt;
|
|
452
|
+
}
|
|
408
453
|
run(sql, params = []) {
|
|
409
|
-
this.
|
|
454
|
+
this._cachedPrepare(sql).run(...params);
|
|
410
455
|
}
|
|
411
456
|
get(sql, params = []) {
|
|
412
|
-
return this.
|
|
457
|
+
return this._cachedPrepare(sql).get(...params);
|
|
413
458
|
}
|
|
414
459
|
all(sql, params = []) {
|
|
415
|
-
return this.
|
|
460
|
+
return this._cachedPrepare(sql).all(...params);
|
|
461
|
+
}
|
|
462
|
+
/** Execute raw SQL that may contain multiple statements (e.g. migrations). */
|
|
463
|
+
exec(sql) {
|
|
464
|
+
this.db.exec(sql);
|
|
416
465
|
}
|
|
417
466
|
prepare(sql) {
|
|
418
|
-
const stmt = this.
|
|
467
|
+
const stmt = this._cachedPrepare(sql);
|
|
419
468
|
return {
|
|
420
469
|
run: (...params) => stmt.run(...params),
|
|
421
470
|
get: (...params) => stmt.get(...params),
|
|
@@ -509,19 +558,40 @@ var SchemaManager = class {
|
|
|
509
558
|
});
|
|
510
559
|
}
|
|
511
560
|
/** Run explicit versioned migrations in order, idempotently */
|
|
512
|
-
applyMigrations(adapter, migrations) {
|
|
561
|
+
applyMigrations(adapter, migrations, validator) {
|
|
513
562
|
const sorted = [...migrations].sort((a, b) => {
|
|
514
563
|
const va = String(a.version);
|
|
515
564
|
const vb = String(b.version);
|
|
516
565
|
return va.localeCompare(vb, void 0, { numeric: true });
|
|
517
566
|
});
|
|
567
|
+
if (validator) {
|
|
568
|
+
const errors = [];
|
|
569
|
+
for (const m of sorted) {
|
|
570
|
+
const versionStr = String(m.version);
|
|
571
|
+
const exists = adapter.get("SELECT 1 FROM __lattice_migrations WHERE version = ?", [
|
|
572
|
+
versionStr
|
|
573
|
+
]);
|
|
574
|
+
if (!exists) {
|
|
575
|
+
const result = validator(m.sql);
|
|
576
|
+
if (!result.valid) {
|
|
577
|
+
errors.push(
|
|
578
|
+
`Migration ${versionStr}: ${(result.errors ?? ["invalid SQL"]).join("; ")}`
|
|
579
|
+
);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
if (errors.length > 0) {
|
|
584
|
+
throw new Error(`Migration validation failed:
|
|
585
|
+
${errors.join("\n")}`);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
518
588
|
for (const m of sorted) {
|
|
519
589
|
const versionStr = String(m.version);
|
|
520
590
|
const exists = adapter.get("SELECT 1 FROM __lattice_migrations WHERE version = ?", [
|
|
521
591
|
versionStr
|
|
522
592
|
]);
|
|
523
593
|
if (!exists) {
|
|
524
|
-
adapter.
|
|
594
|
+
adapter.exec(m.sql);
|
|
525
595
|
adapter.run("INSERT INTO __lattice_migrations (version, applied_at) VALUES (?, ?)", [
|
|
526
596
|
versionStr,
|
|
527
597
|
(/* @__PURE__ */ new Date()).toISOString()
|
|
@@ -644,7 +714,7 @@ var Sanitizer = class {
|
|
|
644
714
|
|
|
645
715
|
// src/render/engine.ts
|
|
646
716
|
import { join as join5, basename, isAbsolute, resolve as resolve3 } from "path";
|
|
647
|
-
import { mkdirSync as mkdirSync3, existsSync as existsSync5, copyFileSync } from "fs";
|
|
717
|
+
import { mkdirSync as mkdirSync3, existsSync as existsSync5, copyFileSync as copyFileSync2 } from "fs";
|
|
648
718
|
|
|
649
719
|
// src/render/token-budget.ts
|
|
650
720
|
function estimateTokens(text) {
|
|
@@ -859,6 +929,223 @@ function truncateContent(content, budget) {
|
|
|
859
929
|
if (budget === void 0 || content.length <= budget) return content;
|
|
860
930
|
return content.slice(0, budget) + "\n\n*[truncated \u2014 context budget exceeded]*";
|
|
861
931
|
}
|
|
932
|
+
var BATCH_CHUNK_SIZE = 500;
|
|
933
|
+
function groupBy(rows, keyCol) {
|
|
934
|
+
const map = /* @__PURE__ */ new Map();
|
|
935
|
+
for (const row of rows) {
|
|
936
|
+
const key = String(row[keyCol] ?? "");
|
|
937
|
+
let arr = map.get(key);
|
|
938
|
+
if (!arr) {
|
|
939
|
+
arr = [];
|
|
940
|
+
map.set(key, arr);
|
|
941
|
+
}
|
|
942
|
+
arr.push(row);
|
|
943
|
+
}
|
|
944
|
+
return map;
|
|
945
|
+
}
|
|
946
|
+
function buildBatchClauses(params, opts, tableAlias) {
|
|
947
|
+
let sql = "";
|
|
948
|
+
const prefix = tableAlias ? `${tableAlias}.` : "";
|
|
949
|
+
for (const f of effectiveFilters(opts)) {
|
|
950
|
+
if (!SAFE_COL_RE.test(f.col)) continue;
|
|
951
|
+
switch (f.op) {
|
|
952
|
+
case "eq":
|
|
953
|
+
sql += ` AND ${prefix}"${f.col}" = ?`;
|
|
954
|
+
params.push(f.val);
|
|
955
|
+
break;
|
|
956
|
+
case "ne":
|
|
957
|
+
sql += ` AND ${prefix}"${f.col}" != ?`;
|
|
958
|
+
params.push(f.val);
|
|
959
|
+
break;
|
|
960
|
+
case "gt":
|
|
961
|
+
sql += ` AND ${prefix}"${f.col}" > ?`;
|
|
962
|
+
params.push(f.val);
|
|
963
|
+
break;
|
|
964
|
+
case "gte":
|
|
965
|
+
sql += ` AND ${prefix}"${f.col}" >= ?`;
|
|
966
|
+
params.push(f.val);
|
|
967
|
+
break;
|
|
968
|
+
case "lt":
|
|
969
|
+
sql += ` AND ${prefix}"${f.col}" < ?`;
|
|
970
|
+
params.push(f.val);
|
|
971
|
+
break;
|
|
972
|
+
case "lte":
|
|
973
|
+
sql += ` AND ${prefix}"${f.col}" <= ?`;
|
|
974
|
+
params.push(f.val);
|
|
975
|
+
break;
|
|
976
|
+
case "like":
|
|
977
|
+
sql += ` AND ${prefix}"${f.col}" LIKE ?`;
|
|
978
|
+
params.push(f.val);
|
|
979
|
+
break;
|
|
980
|
+
case "in": {
|
|
981
|
+
const arr = f.val;
|
|
982
|
+
if (arr.length === 0) {
|
|
983
|
+
sql += " AND 0";
|
|
984
|
+
} else {
|
|
985
|
+
sql += ` AND ${prefix}"${f.col}" IN (${arr.map(() => "?").join(", ")})`;
|
|
986
|
+
params.push(...arr);
|
|
987
|
+
}
|
|
988
|
+
break;
|
|
989
|
+
}
|
|
990
|
+
case "isNull":
|
|
991
|
+
sql += ` AND ${prefix}"${f.col}" IS NULL`;
|
|
992
|
+
break;
|
|
993
|
+
case "isNotNull":
|
|
994
|
+
sql += ` AND ${prefix}"${f.col}" IS NOT NULL`;
|
|
995
|
+
break;
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
if (opts.orderBy) {
|
|
999
|
+
if (typeof opts.orderBy === "string") {
|
|
1000
|
+
if (SAFE_COL_RE.test(opts.orderBy)) {
|
|
1001
|
+
const dir = opts.orderDir === "desc" ? "DESC" : "ASC";
|
|
1002
|
+
sql += ` ORDER BY ${prefix}"${opts.orderBy}" ${dir}`;
|
|
1003
|
+
}
|
|
1004
|
+
} else {
|
|
1005
|
+
const clauses = opts.orderBy.filter((spec) => SAFE_COL_RE.test(spec.col)).map((spec) => `${prefix}"${spec.col}" ${spec.dir === "desc" ? "DESC" : "ASC"}`);
|
|
1006
|
+
if (clauses.length > 0) {
|
|
1007
|
+
sql += ` ORDER BY ${clauses.join(", ")}`;
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
return sql;
|
|
1012
|
+
}
|
|
1013
|
+
function batchQuery(adapter, buildSql, inValues, extraParams) {
|
|
1014
|
+
if (inValues.length === 0) return [];
|
|
1015
|
+
const allRows = [];
|
|
1016
|
+
for (let i = 0; i < inValues.length; i += BATCH_CHUNK_SIZE) {
|
|
1017
|
+
const chunk = inValues.slice(i, i + BATCH_CHUNK_SIZE);
|
|
1018
|
+
const placeholders = chunk.map(() => "?").join(", ");
|
|
1019
|
+
const sql = buildSql(placeholders);
|
|
1020
|
+
allRows.push(...adapter.all(sql, [...chunk, ...extraParams]));
|
|
1021
|
+
}
|
|
1022
|
+
return allRows;
|
|
1023
|
+
}
|
|
1024
|
+
function batchPrefetchEntitySources(files, allEntityRows, entityPk, adapter, protection) {
|
|
1025
|
+
const results = /* @__PURE__ */ new Map();
|
|
1026
|
+
const unbatched = /* @__PURE__ */ new Set();
|
|
1027
|
+
const allPkValues = allEntityRows.map((r) => r[entityPk]);
|
|
1028
|
+
for (const [filename, spec] of Object.entries(files)) {
|
|
1029
|
+
const source = spec.source;
|
|
1030
|
+
if (source.type === "self") {
|
|
1031
|
+
continue;
|
|
1032
|
+
}
|
|
1033
|
+
if (source.type === "custom" || source.type === "enriched") {
|
|
1034
|
+
unbatched.add(filename);
|
|
1035
|
+
continue;
|
|
1036
|
+
}
|
|
1037
|
+
if (source.type === "hasMany") {
|
|
1038
|
+
if (protection?.protectedTables.has(source.table)) {
|
|
1039
|
+
unbatched.add(filename);
|
|
1040
|
+
continue;
|
|
1041
|
+
}
|
|
1042
|
+
const ref = source.references ?? entityPk;
|
|
1043
|
+
const pkValues = allEntityRows.map((r) => r[ref]);
|
|
1044
|
+
const extraParams = [];
|
|
1045
|
+
const clauses = buildBatchClauses(extraParams, source);
|
|
1046
|
+
const rows = batchQuery(
|
|
1047
|
+
adapter,
|
|
1048
|
+
(ph) => `SELECT * FROM "${source.table}" WHERE "${source.foreignKey}" IN (${ph})${clauses}`,
|
|
1049
|
+
pkValues,
|
|
1050
|
+
extraParams
|
|
1051
|
+
);
|
|
1052
|
+
const grouped = groupBy(rows, source.foreignKey);
|
|
1053
|
+
if (source.limit !== void 0 && source.limit > 0) {
|
|
1054
|
+
for (const [key, arr] of grouped) {
|
|
1055
|
+
if (arr.length > source.limit) grouped.set(key, arr.slice(0, source.limit));
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
results.set(filename, grouped);
|
|
1059
|
+
continue;
|
|
1060
|
+
}
|
|
1061
|
+
if (source.type === "manyToMany") {
|
|
1062
|
+
if (protection?.protectedTables.has(source.remoteTable)) {
|
|
1063
|
+
unbatched.add(filename);
|
|
1064
|
+
continue;
|
|
1065
|
+
}
|
|
1066
|
+
const remotePk = source.references ?? "id";
|
|
1067
|
+
const extraParams = [];
|
|
1068
|
+
const clauses = buildBatchClauses(extraParams, source, "r");
|
|
1069
|
+
let selectCols = "r.*";
|
|
1070
|
+
if (source.junctionColumns?.length) {
|
|
1071
|
+
const jCols = source.junctionColumns.map((jc) => {
|
|
1072
|
+
if (typeof jc === "string") {
|
|
1073
|
+
if (!SAFE_COL_RE.test(jc)) return null;
|
|
1074
|
+
return `j."${jc}"`;
|
|
1075
|
+
}
|
|
1076
|
+
if (!SAFE_COL_RE.test(jc.col) || !SAFE_COL_RE.test(jc.as)) return null;
|
|
1077
|
+
return `j."${jc.col}" AS "${jc.as}"`;
|
|
1078
|
+
}).filter(Boolean);
|
|
1079
|
+
if (jCols.length > 0) selectCols += ", " + jCols.join(", ");
|
|
1080
|
+
}
|
|
1081
|
+
const batchKeyCol = "__lattice_batch_key";
|
|
1082
|
+
const rows = batchQuery(
|
|
1083
|
+
adapter,
|
|
1084
|
+
(ph) => `SELECT ${selectCols}, j."${source.localKey}" AS "${batchKeyCol}" FROM "${source.remoteTable}" r JOIN "${source.junctionTable}" j ON j."${source.remoteKey}" = r."${remotePk}" WHERE j."${source.localKey}" IN (${ph})${clauses}`,
|
|
1085
|
+
allPkValues,
|
|
1086
|
+
extraParams
|
|
1087
|
+
);
|
|
1088
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
1089
|
+
for (const row of rows) {
|
|
1090
|
+
const key = String(row[batchKeyCol] ?? "");
|
|
1091
|
+
delete row[batchKeyCol];
|
|
1092
|
+
let arr = grouped.get(key);
|
|
1093
|
+
if (!arr) {
|
|
1094
|
+
arr = [];
|
|
1095
|
+
grouped.set(key, arr);
|
|
1096
|
+
}
|
|
1097
|
+
arr.push(row);
|
|
1098
|
+
}
|
|
1099
|
+
if (source.limit !== void 0 && source.limit > 0) {
|
|
1100
|
+
for (const [key, arr] of grouped) {
|
|
1101
|
+
if (arr.length > source.limit) grouped.set(key, arr.slice(0, source.limit));
|
|
1102
|
+
}
|
|
1103
|
+
}
|
|
1104
|
+
results.set(filename, grouped);
|
|
1105
|
+
continue;
|
|
1106
|
+
}
|
|
1107
|
+
if (source.type === "belongsTo") {
|
|
1108
|
+
if (protection?.protectedTables.has(source.table)) {
|
|
1109
|
+
unbatched.add(filename);
|
|
1110
|
+
continue;
|
|
1111
|
+
}
|
|
1112
|
+
const fkValues = [
|
|
1113
|
+
...new Set(allEntityRows.map((r) => r[source.foreignKey]).filter((v) => v != null))
|
|
1114
|
+
];
|
|
1115
|
+
if (fkValues.length === 0) {
|
|
1116
|
+
results.set(filename, /* @__PURE__ */ new Map());
|
|
1117
|
+
continue;
|
|
1118
|
+
}
|
|
1119
|
+
const refCol = source.references ?? "id";
|
|
1120
|
+
const extraParams = [];
|
|
1121
|
+
const clauses = buildBatchClauses(extraParams, source);
|
|
1122
|
+
const rows = batchQuery(
|
|
1123
|
+
adapter,
|
|
1124
|
+
(ph) => `SELECT * FROM "${source.table}" WHERE "${refCol}" IN (${ph})${clauses}`,
|
|
1125
|
+
fkValues,
|
|
1126
|
+
extraParams
|
|
1127
|
+
);
|
|
1128
|
+
const lookup = /* @__PURE__ */ new Map();
|
|
1129
|
+
for (const row of rows) {
|
|
1130
|
+
lookup.set(String(row[refCol] ?? ""), row);
|
|
1131
|
+
}
|
|
1132
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
1133
|
+
for (const entityRow of allEntityRows) {
|
|
1134
|
+
const fkVal = entityRow[source.foreignKey];
|
|
1135
|
+
const pkVal = String(entityRow[entityPk] ?? "");
|
|
1136
|
+
if (fkVal == null) {
|
|
1137
|
+
grouped.set(pkVal, []);
|
|
1138
|
+
} else {
|
|
1139
|
+
const related = lookup.get(String(fkVal));
|
|
1140
|
+
grouped.set(pkVal, related ? [related] : []);
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
results.set(filename, grouped);
|
|
1144
|
+
continue;
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
return { results, unbatched };
|
|
1148
|
+
}
|
|
862
1149
|
|
|
863
1150
|
// src/render/markdown.ts
|
|
864
1151
|
function frontmatter(fields) {
|
|
@@ -1021,7 +1308,7 @@ ${tmpl.perRow.body(row)}
|
|
|
1021
1308
|
|
|
1022
1309
|
// src/lifecycle/cleanup.ts
|
|
1023
1310
|
import { join as join4 } from "path";
|
|
1024
|
-
import { existsSync as existsSync4, readdirSync, unlinkSync, rmdirSync, statSync } from "fs";
|
|
1311
|
+
import { existsSync as existsSync4, readdirSync, unlinkSync as unlinkSync2, rmdirSync, statSync } from "fs";
|
|
1025
1312
|
function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, manifest, options = {}, newManifest) {
|
|
1026
1313
|
const result = {
|
|
1027
1314
|
directoriesRemoved: [],
|
|
@@ -1063,7 +1350,7 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
|
|
|
1063
1350
|
if (globalProtected.has(filename)) continue;
|
|
1064
1351
|
const filePath = join4(entityDir, filename);
|
|
1065
1352
|
if (!existsSync4(filePath)) continue;
|
|
1066
|
-
if (!options.dryRun)
|
|
1353
|
+
if (!options.dryRun) unlinkSync2(filePath);
|
|
1067
1354
|
options.onOrphan?.(filePath, "file");
|
|
1068
1355
|
result.filesRemoved.push(filePath);
|
|
1069
1356
|
}
|
|
@@ -1108,7 +1395,7 @@ function cleanupEntityContexts(outputDir, entityContexts, currentSlugsByTable, m
|
|
|
1108
1395
|
if (globalProtected.has(filename)) continue;
|
|
1109
1396
|
const filePath = join4(entityDir, filename);
|
|
1110
1397
|
if (!existsSync4(filePath)) continue;
|
|
1111
|
-
if (!options.dryRun)
|
|
1398
|
+
if (!options.dryRun) unlinkSync2(filePath);
|
|
1112
1399
|
options.onOrphan?.(filePath, "file");
|
|
1113
1400
|
result.filesRemoved.push(filePath);
|
|
1114
1401
|
}
|
|
@@ -1159,7 +1446,9 @@ var RenderEngine = class {
|
|
|
1159
1446
|
}
|
|
1160
1447
|
}
|
|
1161
1448
|
if (!def.prioritizeBy) {
|
|
1162
|
-
rows.sort(
|
|
1449
|
+
rows.sort(
|
|
1450
|
+
(a, b) => (b._reward_total ?? 0) - (a._reward_total ?? 0)
|
|
1451
|
+
);
|
|
1163
1452
|
}
|
|
1164
1453
|
}
|
|
1165
1454
|
if (def.enrich) {
|
|
@@ -1261,13 +1550,25 @@ var RenderEngine = class {
|
|
|
1261
1550
|
counters.skipped++;
|
|
1262
1551
|
}
|
|
1263
1552
|
}
|
|
1553
|
+
const protection = protectedTables.size > 0 ? { protectedTables, currentTable: table } : void 0;
|
|
1554
|
+
const mergedFiles = {};
|
|
1555
|
+
for (const [filename, spec] of Object.entries(def.files)) {
|
|
1556
|
+
const mergeDefaults = def.sourceDefaults && spec.source.type !== "self" && spec.source.type !== "custom" && spec.source.type !== "enriched";
|
|
1557
|
+
const source = mergeDefaults ? { ...def.sourceDefaults, ...spec.source } : spec.source;
|
|
1558
|
+
mergedFiles[filename] = { source, limit: spec.budget };
|
|
1559
|
+
}
|
|
1560
|
+
const batch = batchPrefetchEntitySources(
|
|
1561
|
+
mergedFiles,
|
|
1562
|
+
allRows,
|
|
1563
|
+
entityPk,
|
|
1564
|
+
this._adapter,
|
|
1565
|
+
protection
|
|
1566
|
+
);
|
|
1264
1567
|
for (const entityRow of allRows) {
|
|
1265
1568
|
const rawSlug = def.slug(entityRow);
|
|
1266
1569
|
const slug = rawSlug.replace(/[\u00A0\u2000-\u200B\u202F\u205F\u3000]/g, " ").replace(/[\x00-\x1F\x7F]/g, "");
|
|
1267
1570
|
if (/[^a-zA-Z0-9.\-_ @(),#&'+:;!~\[\]]/.test(slug)) {
|
|
1268
|
-
throw new Error(
|
|
1269
|
-
`Invalid slug "${slug}": contains characters outside the allowed set`
|
|
1270
|
-
);
|
|
1571
|
+
throw new Error(`Invalid slug "${slug}": contains characters outside the allowed set`);
|
|
1271
1572
|
}
|
|
1272
1573
|
const entityDir = def.directory ? join5(outputDir, def.directory(entityRow)) : join5(outputDir, directoryRoot, slug);
|
|
1273
1574
|
const resolvedDir = resolve3(entityDir);
|
|
@@ -1284,7 +1585,7 @@ var RenderEngine = class {
|
|
|
1284
1585
|
const destPath = join5(entityDir, basename(absPath));
|
|
1285
1586
|
if (!existsSync5(destPath)) {
|
|
1286
1587
|
try {
|
|
1287
|
-
|
|
1588
|
+
copyFileSync2(absPath, destPath);
|
|
1288
1589
|
filesWritten.push(destPath);
|
|
1289
1590
|
} catch {
|
|
1290
1591
|
}
|
|
@@ -1294,11 +1595,19 @@ var RenderEngine = class {
|
|
|
1294
1595
|
}
|
|
1295
1596
|
const renderedFiles = /* @__PURE__ */ new Map();
|
|
1296
1597
|
const entityFileHashes = {};
|
|
1297
|
-
const
|
|
1598
|
+
const entityPkVal = String(entityRow[entityPk] ?? "");
|
|
1298
1599
|
for (const [filename, spec] of Object.entries(def.files)) {
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1600
|
+
let rows;
|
|
1601
|
+
if (spec.source.type === "self") {
|
|
1602
|
+
rows = [entityRow];
|
|
1603
|
+
} else if (batch.unbatched.has(filename)) {
|
|
1604
|
+
const source = mergedFiles[filename].source;
|
|
1605
|
+
rows = resolveEntitySource(source, entityRow, entityPk, this._adapter, protection);
|
|
1606
|
+
} else if (batch.results.has(filename)) {
|
|
1607
|
+
rows = batch.results.get(filename).get(entityPkVal) ?? [];
|
|
1608
|
+
} else {
|
|
1609
|
+
rows = [];
|
|
1610
|
+
}
|
|
1302
1611
|
if (spec.omitIfEmpty && rows.length === 0) continue;
|
|
1303
1612
|
const renderFn = compileEntityRender(spec.render);
|
|
1304
1613
|
const content = truncateContent(renderFn(rows), spec.budget);
|
|
@@ -1778,10 +2087,10 @@ async function storeEmbedding(adapter, table, pk, row, config) {
|
|
|
1778
2087
|
);
|
|
1779
2088
|
}
|
|
1780
2089
|
function removeEmbedding(adapter, table, pk) {
|
|
1781
|
-
adapter.run(
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
);
|
|
2090
|
+
adapter.run(`DELETE FROM "${EMBEDDINGS_TABLE}" WHERE "table_name" = ? AND "row_pk" = ?`, [
|
|
2091
|
+
table,
|
|
2092
|
+
pk
|
|
2093
|
+
]);
|
|
1785
2094
|
}
|
|
1786
2095
|
function cosineSimilarity(a, b) {
|
|
1787
2096
|
const len = Math.min(a.length, b.length);
|
|
@@ -1842,6 +2151,10 @@ var Lattice = class {
|
|
|
1842
2151
|
_encryptionKeyRaw;
|
|
1843
2152
|
/** Current task context string for relevance filtering. */
|
|
1844
2153
|
_taskContext = "";
|
|
2154
|
+
/** Per-table write version counter, incremented on every mutation. */
|
|
2155
|
+
_tableWriteVersion = /* @__PURE__ */ new Map();
|
|
2156
|
+
/** Snapshot of write versions at the last successful render. */
|
|
2157
|
+
_lastRenderedVersions = /* @__PURE__ */ new Map();
|
|
1845
2158
|
_auditHandlers = [];
|
|
1846
2159
|
_renderHandlers = [];
|
|
1847
2160
|
_writebackHandlers = [];
|
|
@@ -1935,8 +2248,9 @@ var Lattice = class {
|
|
|
1935
2248
|
this._adapter.open();
|
|
1936
2249
|
this._schema.applySchema(this._adapter);
|
|
1937
2250
|
if (options.migrations?.length) {
|
|
1938
|
-
this._schema.applyMigrations(this._adapter, options.migrations);
|
|
2251
|
+
this._schema.applyMigrations(this._adapter, options.migrations, options.validateMigrationSQL);
|
|
1939
2252
|
}
|
|
2253
|
+
this._adapter.clearStatementCache();
|
|
1940
2254
|
for (const tableName of this._schema.getTables().keys()) {
|
|
1941
2255
|
const rows = this._adapter.all(`PRAGMA table_info("${tableName}")`);
|
|
1942
2256
|
this._columnCache.set(tableName, new Set(rows.map((r) => r.name)));
|
|
@@ -1960,6 +2274,7 @@ var Lattice = class {
|
|
|
1960
2274
|
return Promise.reject(new Error("Lattice: not initialized \u2014 call init() first"));
|
|
1961
2275
|
}
|
|
1962
2276
|
this._schema.applyMigrations(this._adapter, migrations);
|
|
2277
|
+
this._adapter.clearStatementCache();
|
|
1963
2278
|
for (const tableName of this._schema.getTables().keys()) {
|
|
1964
2279
|
const rows = this._adapter.all(`PRAGMA table_info("${tableName}")`);
|
|
1965
2280
|
this._columnCache.set(tableName, new Set(rows.map((r) => r.name)));
|
|
@@ -1984,6 +2299,20 @@ var Lattice = class {
|
|
|
1984
2299
|
this._taskContext = context;
|
|
1985
2300
|
return this;
|
|
1986
2301
|
}
|
|
2302
|
+
/**
|
|
2303
|
+
* Mark one or all tables as dirty so the next render() will run a full cycle.
|
|
2304
|
+
* Use this after writing directly via the `db` escape hatch.
|
|
2305
|
+
*/
|
|
2306
|
+
markDirty(table) {
|
|
2307
|
+
if (table) {
|
|
2308
|
+
this._bumpWriteVersion(table);
|
|
2309
|
+
} else {
|
|
2310
|
+
for (const t of this._schema.getTables().keys()) {
|
|
2311
|
+
this._bumpWriteVersion(t);
|
|
2312
|
+
}
|
|
2313
|
+
}
|
|
2314
|
+
return this;
|
|
2315
|
+
}
|
|
1987
2316
|
/** Return the current task context string. */
|
|
1988
2317
|
getTaskContext() {
|
|
1989
2318
|
return this._taskContext;
|
|
@@ -2104,6 +2433,7 @@ var Lattice = class {
|
|
|
2104
2433
|
const rawPk = rowWithPk[pkCol];
|
|
2105
2434
|
const pkValue = rawPk != null ? String(rawPk) : "";
|
|
2106
2435
|
this._sanitizer.emitAudit(table, "update", pkValue);
|
|
2436
|
+
this._bumpWriteVersion(table);
|
|
2107
2437
|
return Promise.resolve(pkValue);
|
|
2108
2438
|
}
|
|
2109
2439
|
upsertBy(table, col, val, row) {
|
|
@@ -2280,6 +2610,7 @@ var Lattice = class {
|
|
|
2280
2610
|
AND (deleted_at IS NULL OR deleted_at = '')`,
|
|
2281
2611
|
[sourceFile, ...currentKeys]
|
|
2282
2612
|
);
|
|
2613
|
+
this._bumpWriteVersion(table);
|
|
2283
2614
|
}
|
|
2284
2615
|
return Promise.resolve(count);
|
|
2285
2616
|
}
|
|
@@ -2336,6 +2667,7 @@ var Lattice = class {
|
|
|
2336
2667
|
`${verb} INTO "${junctionTable}" (${colNames}) VALUES (${placeholders})`,
|
|
2337
2668
|
Object.values(filtered)
|
|
2338
2669
|
);
|
|
2670
|
+
this._bumpWriteVersion(junctionTable);
|
|
2339
2671
|
return Promise.resolve();
|
|
2340
2672
|
}
|
|
2341
2673
|
/**
|
|
@@ -2351,6 +2683,7 @@ var Lattice = class {
|
|
|
2351
2683
|
`DELETE FROM "${junctionTable}" WHERE ${where}`,
|
|
2352
2684
|
entries.map(([, v]) => v)
|
|
2353
2685
|
);
|
|
2686
|
+
this._bumpWriteVersion(junctionTable);
|
|
2354
2687
|
return Promise.resolve();
|
|
2355
2688
|
}
|
|
2356
2689
|
// -------------------------------------------------------------------------
|
|
@@ -2554,9 +2887,7 @@ var Lattice = class {
|
|
|
2554
2887
|
if (notInit) return notInit;
|
|
2555
2888
|
const def = this._schema.getTables().get(table);
|
|
2556
2889
|
if (!def?.rewardTracking) {
|
|
2557
|
-
return Promise.reject(
|
|
2558
|
-
new Error(`Table "${table}" does not have rewardTracking enabled`)
|
|
2559
|
-
);
|
|
2890
|
+
return Promise.reject(new Error(`Table "${table}" does not have rewardTracking enabled`));
|
|
2560
2891
|
}
|
|
2561
2892
|
const vals = Object.values(scores);
|
|
2562
2893
|
if (vals.length === 0) return Promise.resolve();
|
|
@@ -2566,6 +2897,7 @@ var Lattice = class {
|
|
|
2566
2897
|
`UPDATE "${table}" SET "_reward_total" = ("_reward_total" * "_reward_count" + ?) / ("_reward_count" + 1), "_reward_count" = "_reward_count" + 1 WHERE ${clause}`,
|
|
2567
2898
|
[avg, ...pkParams]
|
|
2568
2899
|
);
|
|
2900
|
+
this._bumpWriteVersion(table);
|
|
2569
2901
|
return Promise.resolve();
|
|
2570
2902
|
}
|
|
2571
2903
|
// -------------------------------------------------------------------------
|
|
@@ -2585,9 +2917,7 @@ var Lattice = class {
|
|
|
2585
2917
|
if (notInit) return notInit;
|
|
2586
2918
|
const def = this._schema.getTables().get(table);
|
|
2587
2919
|
if (!def?.embeddings) {
|
|
2588
|
-
return Promise.reject(
|
|
2589
|
-
new Error(`Table "${table}" does not have embeddings configured`)
|
|
2590
|
-
);
|
|
2920
|
+
return Promise.reject(new Error(`Table "${table}" does not have embeddings configured`));
|
|
2591
2921
|
}
|
|
2592
2922
|
const pkCol = this._schema.getPrimaryKey(table)[0] ?? "id";
|
|
2593
2923
|
return searchByEmbedding(
|
|
@@ -2674,13 +3004,27 @@ var Lattice = class {
|
|
|
2674
3004
|
const notInit = this._notInitError();
|
|
2675
3005
|
if (notInit) return notInit;
|
|
2676
3006
|
const result = await this._render.render(outputDir);
|
|
3007
|
+
for (const [t, v] of this._tableWriteVersion) {
|
|
3008
|
+
this._lastRenderedVersions.set(t, v);
|
|
3009
|
+
}
|
|
2677
3010
|
for (const h of this._renderHandlers) h(result);
|
|
2678
3011
|
return result;
|
|
2679
3012
|
}
|
|
3013
|
+
/**
|
|
3014
|
+
* Returns `true` when no table has been written to since the last render.
|
|
3015
|
+
* Useful for consumers implementing their own polling/watch loops to skip
|
|
3016
|
+
* redundant render cycles.
|
|
3017
|
+
*/
|
|
3018
|
+
isDirty() {
|
|
3019
|
+
return !this._isClean();
|
|
3020
|
+
}
|
|
2680
3021
|
async sync(outputDir) {
|
|
2681
3022
|
const notInit = this._notInitError();
|
|
2682
3023
|
if (notInit) return notInit;
|
|
2683
3024
|
const renderResult = await this._render.render(outputDir);
|
|
3025
|
+
for (const [t, v] of this._tableWriteVersion) {
|
|
3026
|
+
this._lastRenderedVersions.set(t, v);
|
|
3027
|
+
}
|
|
2684
3028
|
for (const h of this._renderHandlers) h(renderResult);
|
|
2685
3029
|
const writebackProcessed = await this._writeback.process();
|
|
2686
3030
|
return { ...renderResult, writebackProcessed };
|
|
@@ -2841,8 +3185,21 @@ var Lattice = class {
|
|
|
2841
3185
|
}
|
|
2842
3186
|
return { clauses, params };
|
|
2843
3187
|
}
|
|
3188
|
+
/** Increment the write version for a table, marking it dirty for render. */
|
|
3189
|
+
_bumpWriteVersion(table) {
|
|
3190
|
+
this._tableWriteVersion.set(table, (this._tableWriteVersion.get(table) ?? 0) + 1);
|
|
3191
|
+
}
|
|
3192
|
+
/** True when no table has been written to since the last render. */
|
|
3193
|
+
_isClean() {
|
|
3194
|
+
if (this._lastRenderedVersions.size === 0) return false;
|
|
3195
|
+
for (const [table, version] of this._tableWriteVersion) {
|
|
3196
|
+
if (version !== (this._lastRenderedVersions.get(table) ?? -1)) return false;
|
|
3197
|
+
}
|
|
3198
|
+
return true;
|
|
3199
|
+
}
|
|
2844
3200
|
/** Returns a rejected Promise if not initialized; null if ready. */
|
|
2845
3201
|
_fireWriteHooks(table, op, row, pk, changedColumns) {
|
|
3202
|
+
this._bumpWriteVersion(table);
|
|
2846
3203
|
for (const hook of this._writeHooks) {
|
|
2847
3204
|
if (hook.table !== table) continue;
|
|
2848
3205
|
if (!hook.on.includes(op)) continue;
|