@mistriwork/engine 1.0.0-rc.34 → 1.0.0-rc.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  import { ANSI_STYLE, Refusal } from '../adapter.js';
2
+ import { cachedStatement as prepared } from '../sqliteHandles.js';
2
3
  import { chooseColumnReference, deleteRuleOf, } from '../../shared/schema.js';
3
4
  import { mapScopeCatalog } from '../../shared/scopeCatalog.js';
4
5
  import { referenceKey } from './scopedTransaction.js';
@@ -127,7 +128,7 @@ export function sqliteKeyFacts(db) {
127
128
  const definitions = new Map();
128
129
  const definitionOf = (table) => {
129
130
  if (!definitions.has(table.id)) {
130
- const row = db.prepare("SELECT sql FROM main.sqlite_schema WHERE type = 'table' AND name = ?").get(table.name);
131
+ const row = prepared(db, "SELECT sql FROM main.sqlite_schema WHERE type = 'table' AND name = ?").get(table.name);
131
132
  definitions.set(table.id, typeof row?.sql === 'string' ? row.sql : undefined);
132
133
  }
133
134
  return definitions.get(table.id);
@@ -170,14 +171,12 @@ export function sqliteKeyFacts(db) {
170
171
  // A rowid alias holds integers alone, compared by no collation; it has no index of its own.
171
172
  if (primary.length === 1 && table.columns.find((column) => column.name === primary[0])?.identity)
172
173
  return [primary];
173
- const index = db
174
- .prepare(`PRAGMA main.index_list(${ANSI_STYLE.quote(table.name)})`)
174
+ const index = prepared(db, `PRAGMA main.index_list(${ANSI_STYLE.quote(table.name)})`)
175
175
  .all()
176
176
  .find((row) => row.origin === 'pk');
177
177
  if (typeof index?.name !== 'string')
178
178
  return [];
179
- const indexed = db
180
- .prepare(`PRAGMA main.index_xinfo(${ANSI_STYLE.quote(index.name)})`)
179
+ const indexed = prepared(db, `PRAGMA main.index_xinfo(${ANSI_STYLE.quote(index.name)})`)
181
180
  .all()
182
181
  .filter((row) => row.key === 1);
183
182
  const same = primary.every((name) => {
@@ -196,7 +195,7 @@ export function sqliteKeyFacts(db) {
196
195
  */
197
196
  export function sqliteBlockedTables(db) {
198
197
  const blocked = new Map();
199
- for (const table of db.prepare('PRAGMA main.table_list').all()) {
198
+ for (const table of prepared(db, 'PRAGMA main.table_list').all()) {
200
199
  if (table.schema !== 'main' || typeof table.name !== 'string')
201
200
  continue;
202
201
  if (table.type === 'virtual' || table.type === 'shadow')
@@ -216,12 +215,10 @@ export function readSqliteReadCatalog(db) {
216
215
  }
217
216
  function scanSqliteScopeCatalog(db) {
218
217
  try {
219
- if (db.prepare('PRAGMA foreign_keys').get()?.foreign_keys !== 1)
218
+ if (prepared(db, 'PRAGMA foreign_keys').get()?.foreign_keys !== 1)
220
219
  return refuse();
221
- const entries = db
222
- .prepare("SELECT name, type FROM main.sqlite_schema WHERE type IN ('table', 'view') ORDER BY name")
223
- .all();
224
- const listed = db.prepare('PRAGMA main.table_list').all();
220
+ const entries = prepared(db, "SELECT name, type FROM main.sqlite_schema WHERE type IN ('table', 'view') ORDER BY name").all();
221
+ const listed = prepared(db, 'PRAGMA main.table_list').all();
225
222
  // Unknown PRAGMAs can return nothing without throwing. Never interpret
226
223
  // unavailable table-kind metadata as proof that a table is ordinary.
227
224
  if (!listed.some((table) => table.schema === 'main' && table.name === 'sqlite_schema' && table.type === 'table'))
@@ -231,9 +228,7 @@ function scanSqliteScopeCatalog(db) {
231
228
  it only validates its row, is read from that text per table below (sqliteTriggers.ts, scopeTriggers.ts). */
232
229
  const definitions = new Map();
233
230
  for (const namespace of ['main', 'temp']) {
234
- const triggers = db
235
- .prepare(`SELECT tbl_name AS name, name AS trigger, sql FROM ${namespace}.sqlite_schema WHERE type = 'trigger' ORDER BY name`)
236
- .all();
231
+ const triggers = prepared(db, `SELECT tbl_name AS name, name AS trigger, sql FROM ${namespace}.sqlite_schema WHERE type = 'trigger' ORDER BY name`).all();
237
232
  for (const trigger of triggers) {
238
233
  if (typeof trigger.name !== 'string' || typeof trigger.trigger !== 'string')
239
234
  return refuse();
@@ -245,7 +240,7 @@ function scanSqliteScopeCatalog(db) {
245
240
  }
246
241
  const triggered = new Map();
247
242
  const shadowed = new Set();
248
- for (const table of db.prepare("SELECT name FROM temp.sqlite_schema WHERE type IN ('table', 'view')").all()) {
243
+ for (const table of prepared(db, "SELECT name FROM temp.sqlite_schema WHERE type IN ('table', 'view')").all()) {
249
244
  if (typeof table.name !== 'string')
250
245
  return refuse();
251
246
  shadowed.add(folded(table.name));
@@ -262,7 +257,7 @@ function scanSqliteScopeCatalog(db) {
262
257
  !['table', 'view', 'virtual', 'shadow'].includes(kind.type) ||
263
258
  (entry.type === 'view') !== (kind.type === 'view'))
264
259
  return refuse();
265
- const columns = db.prepare(`PRAGMA main.table_xinfo(${ANSI_STYLE.quote(entry.name)})`).all();
260
+ const columns = prepared(db, `PRAGMA main.table_xinfo(${ANSI_STYLE.quote(entry.name)})`).all();
266
261
  identities(columns);
267
262
  if (!columns.length || columns.length !== kind.ncol)
268
263
  return refuse();
@@ -279,7 +274,7 @@ function scanSqliteScopeCatalog(db) {
279
274
  const primary = columns.filter((column) => column.pk > 0).sort((a, b) => a.pk - b.pk);
280
275
  if (primary.some((column, i) => column.pk !== i + 1))
281
276
  return refuse();
282
- const indexes = db.prepare(`PRAGMA main.index_list(${ANSI_STYLE.quote(entry.name)})`).all();
277
+ const indexes = prepared(db, `PRAGMA main.index_list(${ANSI_STYLE.quote(entry.name)})`).all();
283
278
  // INTEGER PRIMARY KEY DESC has a PK index and is not a rowid alias.
284
279
  const identity = kind.type === 'table' &&
285
280
  kind.wr === 0 &&
@@ -329,7 +324,7 @@ function scanSqliteScopeCatalog(db) {
329
324
  const byName = identities(tables);
330
325
  const foreignKeys = [];
331
326
  for (const table of tables) {
332
- const rows = db.prepare(`PRAGMA main.foreign_key_list(${ANSI_STYLE.quote(table.name)})`).all();
327
+ const rows = prepared(db, `PRAGMA main.foreign_key_list(${ANSI_STYLE.quote(table.name)})`).all();
333
328
  const groups = new Map();
334
329
  for (const row of rows) {
335
330
  if (!Number.isInteger(row.id) ||
@@ -1,3 +1,4 @@
1
+ export { cachedStatement, schemaFingerprint } from './sqliteHandles.js';
1
2
  import { DatabaseSync, type SQLInputValue } from 'node:sqlite';
2
3
  import { type Adapter, type ScopedReads, type Filter, type QueryRequest, type QueryResult, type SqlRequest, type SqlResult, type DeleteRequest, type DeleteResult, type UpdateRequest, type UpdateResult, type ViewResult, type BreakdownResult, TransactionOperations, type WriteTransaction, type InsertRequest, type InsertResult, type ExplainRequest, type ExplainResult, type ProfileLimits, type ProfileResult, type CountItem, type AggregateRequest, type AggregateResult } from './adapter.js';
3
4
  import type { TriggerAcceptance } from './scoped/triggerProof.js';
@@ -9,6 +9,8 @@
9
9
  */
10
10
  import { statSync } from 'node:fs';
11
11
  import { waitForLocks, assertIndependentWrite, writeToFile, prepareGuarded, transactionEpoch, storeEpoch, schemaFingerprint, } from './sqliteHandles.js';
12
+ /* The tenant sources keep their catalog and store-layout checks by these (tenantReadSource.ts, tenantStore.ts). */
13
+ export { cachedStatement, schemaFingerprint } from './sqliteHandles.js';
12
14
  import { captureScopeCatalog, mapScopeCatalog } from '../shared/scopeCatalog.js';
13
15
  import { inlineRows } from '../shared/anyRows.js';
14
16
  import { asksAboutRows } from '../shared/filter.js';
@@ -41,6 +43,49 @@ import { flagsFromDDL, vocabularyFromDDL } from '../shared/vocabulary.js';
41
43
  import { coerceForColumn, columnKind, deleteRuleOf, } from '../shared/schema.js';
42
44
  /** A generated column, virtual (2) or stored (3): the database writes it from its expression. */
43
45
  const generated = (column) => column.hidden === 2 || column.hidden === 3;
46
+ /* The catalog answers a write transaction trusts outlive the adapter that read them: a writer reopened, or a second
47
+ one on the file, starts warm. Kept per file, by the identity of the file the handle opened (device, inode and birth
48
+ time, so a new file at a reused inode is a new key) and never across files; a file with no birth time to tell it
49
+ from its inode's next tenant, and an in-memory database, keep their own. Every entry is still served only under the
50
+ epoch the reader holds (epochCache.ts), and frozen, since another adapter reads the same value. */
51
+ const sharedCatalogs = new Map();
52
+ function catalogCacheOf(identity) {
53
+ if (identity === undefined)
54
+ return new EpochCache();
55
+ const cache = sharedCatalogs.get(identity) ?? new EpochCache();
56
+ sharedCatalogs.delete(identity);
57
+ sharedCatalogs.set(identity, cache);
58
+ while (sharedCatalogs.size > 32)
59
+ sharedCatalogs.delete(sharedCatalogs.keys().next().value);
60
+ return cache;
61
+ }
62
+ function frozen(value) {
63
+ if (!value || typeof value !== 'object' || Object.isFrozen(value))
64
+ return value;
65
+ if (value instanceof Map)
66
+ for (const [key, each] of value) {
67
+ frozen(key);
68
+ frozen(each);
69
+ }
70
+ else if (value instanceof Set)
71
+ for (const each of value)
72
+ frozen(each);
73
+ else
74
+ for (const each of Object.values(value))
75
+ frozen(each);
76
+ return Object.freeze(value);
77
+ }
78
+ /* What the scope catalog was resolved with is part of its key: a caller judging trigger effects reads another
79
+ catalog than one that does not. */
80
+ const resolverIds = new WeakMap();
81
+ let resolvers = 0;
82
+ const resolverKey = (resolve) => {
83
+ if (!resolve)
84
+ return 'scope';
85
+ if (!resolverIds.has(resolve))
86
+ resolverIds.set(resolve, ++resolvers);
87
+ return `scope:${resolverIds.get(resolve)}`;
88
+ };
44
89
  const verifiedCommandAdapters = new WeakSet();
45
90
  /** Identity-only native provenance; no caller can register a lookalike. */
46
91
  export const isSqliteVerifiedCommandSource = (value) => !!value && typeof value === 'object' && !isProxy(value) && verifiedCommandAdapters.has(value);
@@ -59,8 +104,8 @@ export class SqliteAdapter {
59
104
  /* Catalog answers a write transaction trusts — the journal's layout, the
60
105
  scope catalog, a table's existence and its columns — kept under the
61
106
  schema epoch the transaction read (transactionEpoch), so a warm command
62
- asks the catalog nothing (epochCache.ts). */
63
- catalogCache = new EpochCache();
107
+ asks the catalog nothing (epochCache.ts). Shared by every adapter on the file (catalogCacheOf). */
108
+ catalogCache;
64
109
  /** The epoch the open write transaction read, on the handle it read it on. */
65
110
  txEpoch;
66
111
  constructor(file, label = file) {
@@ -76,9 +121,12 @@ export class SqliteAdapter {
76
121
  if (typeof filename === 'string' && filename) {
77
122
  const info = statSync(filename, { bigint: true });
78
123
  this.writeIdentity = `${info.dev}:${info.ino}`;
124
+ this.catalogCache = catalogCacheOf(info.birthtimeNs > 0n ? `${info.dev}:${info.ino}:${info.birthtimeNs}` : undefined);
79
125
  }
80
- else
126
+ else {
81
127
  this.writeIdentity = Symbol('sqlite-memory');
128
+ this.catalogCache = catalogCacheOf(undefined);
129
+ }
82
130
  waitForLocks(this.db);
83
131
  }
84
132
  catch (error) {
@@ -1445,7 +1493,12 @@ export class SqliteAdapter {
1445
1493
  verified();
1446
1494
  };
1447
1495
  if (scope !== undefined) {
1448
- const catalog = this.catalogCache.throughSync(epoch, 'scope', () => readSqliteScopeCatalog(handle, effects?.resolveTriggerEffects));
1496
+ /* Its schema captured once here, so the evidence mapping below is the same object every write and is
1497
+ made once per schema rather than copied and checked twice a command (mapScopeCatalog). */
1498
+ const catalog = this.catalogCache.throughSync(epoch, resolverKey(effects?.resolveTriggerEffects), () => {
1499
+ const read = readSqliteScopeCatalog(handle, effects?.resolveTriggerEffects);
1500
+ return frozen({ ...read, schema: captureScopeCatalog(read.schema) });
1501
+ });
1449
1502
  physicalCatalog = catalog;
1450
1503
  journal?.admit?.(handle, catalog, scope);
1451
1504
  // Keep the complete physical graph for row/FK enforcement. Runtime
@@ -1655,7 +1708,7 @@ export class SqliteAdapter {
1655
1708
  the first write of an epoch reads the list, and no later one asks. */
1656
1709
  const epoch = this.epochOf(db);
1657
1710
  if (epoch !== undefined) {
1658
- const names = this.catalogCache.throughSync(epoch, 'tables', () => new Set(db.prepare(`SELECT name FROM sqlite_master WHERE type IN ('table','view')`).all().map((row) => row.name)));
1711
+ const names = this.catalogCache.throughSync(epoch, 'tables', () => frozen(new Set(db.prepare(`SELECT name FROM sqlite_master WHERE type IN ('table','view')`).all().map((row) => row.name))));
1659
1712
  if (!names.has(name))
1660
1713
  throw new Error(`No table or view called "${name}" in this database.`);
1661
1714
  return name;
@@ -1668,7 +1721,7 @@ export class SqliteAdapter {
1668
1721
  /** A table's columns, as a write reads them: from the catalog, or inside a
1669
1722
  * write transaction from what was read under the epoch it holds. */
1670
1723
  tableInfo(table, db) {
1671
- return this.catalogCache.throughSync(this.epochOf(db), `columns:${table}`, () => db.prepare(`PRAGMA main.table_xinfo(${ANSI_STYLE.quote(table)})`).all().filter((c) => c.hidden !== 1));
1724
+ return this.catalogCache.throughSync(this.epochOf(db), `columns:${table}`, () => frozen(db.prepare(`PRAGMA main.table_xinfo(${ANSI_STYLE.quote(table)})`).all().filter((c) => c.hidden !== 1)));
1672
1725
  }
1673
1726
  /** Whether a relation is a view, as a write reads it: kept under the epoch, as its columns are. */
1674
1727
  isView(table, db) {
@@ -44,6 +44,8 @@ export declare function transactionEpoch(db: DatabaseSync, journal?: {
44
44
  * this contract, as it is for captureSchemaLifetime: no business operation can send one.
45
45
  */
46
46
  export declare function schemaFingerprint(db: DatabaseSync): () => string;
47
+ /** `db.prepare(sql)`, kept per handle: for the catalog's and a store layout's questions, asked again and again. */
48
+ export declare function cachedStatement(db: DatabaseSync, sql: string): StatementSync;
47
49
  /** The schema and whether a store's meta row says its version, in one statement; undefined when the store is not there. */
48
50
  export declare function storeEpoch(db: DatabaseSync, meta: {
49
51
  table: string;
@@ -136,9 +136,26 @@ export function transactionEpoch(db, journal) {
136
136
  * this contract, as it is for captureSchemaLifetime: no business operation can send one.
137
137
  */
138
138
  export function schemaFingerprint(db) {
139
- const main = db.prepare('PRAGMA main.schema_version'), temp = db.prepare('PRAGMA temp.schema_version');
139
+ const main = cachedStatement(db, 'PRAGMA main.schema_version'), temp = cachedStatement(db, 'PRAGMA temp.schema_version');
140
140
  return () => `${main.get()?.schema_version}.${temp.get()?.schema_version}`;
141
141
  }
142
+ /* A catalog scan asks the same few hundred questions every time it runs, and preparing them was most of its cost (a
143
+ field report); a store's layout check asks its few on every command. Each handle keeps its own, by text: a statement is bound to its connection, and SQLite prepares one
144
+ again by itself when the schema under it has moved. A catalog of more tables than the bound starts the map over. */
145
+ const statements = new WeakMap();
146
+ /** `db.prepare(sql)`, kept per handle: for the catalog's and a store layout's questions, asked again and again. */
147
+ export function cachedStatement(db, sql) {
148
+ let kept = statements.get(db);
149
+ if (!kept)
150
+ statements.set(db, (kept = new Map()));
151
+ let statement = kept.get(sql);
152
+ if (!statement) {
153
+ if (kept.size >= 2048)
154
+ kept.clear();
155
+ kept.set(sql, (statement = db.prepare(sql)));
156
+ }
157
+ return statement;
158
+ }
142
159
  /** The schema and whether a store's meta row says its version, in one statement; undefined when the store is not there. */
143
160
  export function storeEpoch(db, meta) {
144
161
  try {
@@ -38,12 +38,17 @@ export async function explainStatement(spanId) {
38
38
  return explain ? explain() : undefined;
39
39
  }
40
40
  const EXPLAINABLE = new Set(['SELECT', 'WITH']);
41
+ /* A catalog read or a PRAGMA: the engine's own checks, which run by the dozen
42
+ around every command, and each span cost two random ids even in a trace the
43
+ exporter drops (a field report). They are spans only in a sampled trace. */
44
+ const INTERNAL = /^\s*PRAGMA\b|\b(?:sqlite_schema|sqlite_master|sqlite_temp_schema|sqlite_temp_master|pg_catalog|information_schema)\b/i;
41
45
  /** A statement's span, begun but not made current: a statement has no children. The caller ends it. */
42
46
  function beginStatement(dialect, text, params, explain) {
43
47
  const { operation, table, summary } = summarizeStatement(text);
44
48
  const span = startSpan(summary, {
45
49
  kind: 'client',
46
50
  childOnly: true,
51
+ sampledOnly: INTERNAL.test(text),
47
52
  attributes: {
48
53
  'db.system.name': dialect,
49
54
  'db.operation.name': operation,
@@ -95,6 +95,8 @@ interface SpanOptions {
95
95
  links?: readonly SpanLink[];
96
96
  /** Start only inside an existing span: a statement outside any request, job or command is not worth a trace. */
97
97
  childOnly?: boolean;
98
+ /** Start only beneath a sampled trace: the internal reads a statement span would otherwise open by the dozen (drivers.ts). */
99
+ sampledOnly?: boolean;
98
100
  }
99
101
  /** Begin a span; the caller ends it. Prefer `inSpan`, which also makes it current. */
100
102
  export declare function startSpan(name: string, options?: SpanOptions): Span;
@@ -151,6 +151,8 @@ export function startSpan(name, options = {}) {
151
151
  // Beneath a dropped span nothing is traced: the observer's own EXPLAIN is not a trace.
152
152
  if (parent?.discarded)
153
153
  return NOOP;
154
+ if (options.sampledOnly && !parent?.record.sampled)
155
+ return NOOP;
154
156
  const remote = parent ? undefined : options.remote;
155
157
  const traceId = parent?.traceId ?? remote?.traceId ?? hex(16);
156
158
  const record = {
@@ -19,6 +19,13 @@ const frozenDerived = new WeakSet();
19
19
  const derivedCaptures = new WeakMap();
20
20
  /** Test-only: how many catalogs were copied and checked in full. */
21
21
  export const scopeCatalogCounters = { captured: 0 };
22
+ /* Every object inside a catalog captured below, with what copying it would
23
+ count. A catalog made from a captured one (a mapping, a pruning) shares its
24
+ frozen parts, and copying those again for each write was most of a capture's
25
+ work (a field report). They are detached plain data that nothing can change,
26
+ so copyData keeps them; the checks after it still read every one. Only seal()
27
+ adds to it, so a caller's object, frozen or not, is always copied. */
28
+ const sealed = new WeakMap();
22
29
  /** Validate descriptors before reading values, including non-authoritative metadata. */
23
30
  function copyData(input) {
24
31
  let nodes = 0;
@@ -38,6 +45,12 @@ function copyData(input) {
38
45
  return value;
39
46
  if (typeof value !== 'object' || ancestors.has(value))
40
47
  return refuse();
48
+ const kept = sealed.get(value);
49
+ if (kept) {
50
+ if ((nodes += kept.nodes - 1) > 262_144 || depth + kept.height > 16 || (characters += kept.characters) > 16_777_216)
51
+ return refuse();
52
+ return value;
53
+ }
41
54
  const array = Array.isArray(value);
42
55
  const proto = Object.getPrototypeOf(value);
43
56
  if (array ? proto !== Array.prototype : proto !== Object.prototype && proto !== null)
@@ -89,6 +102,26 @@ function freeze(value) {
89
102
  }
90
103
  return value;
91
104
  }
105
+ /** Freeze a checked catalog all the way down and remember each part's size for copyData. */
106
+ function seal(value) {
107
+ if (typeof value === 'string')
108
+ return { nodes: 1, characters: value.length, height: 0 };
109
+ if (!value || typeof value !== 'object')
110
+ return { nodes: 1, characters: 0, height: 0 };
111
+ const known = sealed.get(value);
112
+ if (known)
113
+ return known;
114
+ const size = { nodes: 1, characters: 0, height: 0 };
115
+ for (const [key, child] of Object.entries(value)) {
116
+ const inner = seal(child);
117
+ size.nodes += inner.nodes;
118
+ size.characters += inner.characters + (Array.isArray(value) ? 0 : key.length);
119
+ size.height = Math.max(size.height, inner.height + 1);
120
+ }
121
+ Object.freeze(value);
122
+ sealed.set(value, size);
123
+ return size;
124
+ }
92
125
  const fkOrder = (a, b) => order(a.from.table, b.from.table) || order(a.name, b.name);
93
126
  /** Whether `schema` is a catalog this module captured: detached, validated
94
127
  * and deeply frozen, so nothing can change it after a reader binds to it. */
@@ -268,7 +301,8 @@ export function captureScopeCatalog(schema) {
268
301
  endpoint(column.references.table, [column.references.column]);
269
302
  }
270
303
  }
271
- const frozen = freeze(captured);
304
+ seal(captured);
305
+ const frozen = captured;
272
306
  capturedCatalogs.add(frozen);
273
307
  if (frozenDerived.has(schema))
274
308
  derivedCaptures.set(schema, frozen);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mistriwork/engine",
3
- "version": "1.0.0-rc.34",
3
+ "version": "1.0.0-rc.35",
4
4
  "description": "The engine both products stand on: the adapters (SQLite, Postgres), the catalog, the query language, keyset pages, the HTTP kit and the shared rules. No React.",
5
5
  "homepage": "https://mistriwork.dev",
6
6
  "bugs": {