@starkeep/storage-sqlite 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/adapter-DoVOV6K-.d.cts +126 -0
- package/dist/adapter-DoVOV6K-.d.ts +126 -0
- package/dist/index.cjs +617 -214
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +172 -71
- package/dist/index.d.ts +172 -71
- package/dist/index.js +645 -193
- package/dist/index.js.map +1 -1
- package/dist/node-entry.cjs +45 -0
- package/dist/node-entry.cjs.map +1 -0
- package/dist/node-entry.d.cts +32 -0
- package/dist/node-entry.d.ts +32 -0
- package/dist/node-entry.js +18 -0
- package/dist/node-entry.js.map +1 -0
- package/package.json +10 -5
package/dist/index.js
CHANGED
|
@@ -1,14 +1,35 @@
|
|
|
1
1
|
// src/adapter.ts
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
import { nextCursorFrom } from "@starkeep/storage-adapter";
|
|
3
|
+
import {
|
|
4
|
+
serializeHLC as serializeHLC2,
|
|
5
|
+
deserializeHLC as deserializeHLC2,
|
|
6
|
+
getCategory,
|
|
7
|
+
isKnownType,
|
|
8
|
+
sqliteMetadataTableName as sqliteMetadataTableName2,
|
|
9
|
+
typeCategory,
|
|
10
|
+
METADATA_DISCRIMINANT_COLUMN,
|
|
11
|
+
METADATA_DELETED_AT_COLUMN
|
|
12
|
+
} from "@starkeep/protocol-primitives";
|
|
6
13
|
import {
|
|
7
14
|
StorageError,
|
|
8
15
|
TransactionError,
|
|
9
|
-
|
|
16
|
+
buildAppAggregateQuery,
|
|
17
|
+
buildAppRowQuery,
|
|
18
|
+
collectAggregatePage,
|
|
19
|
+
collectRowPage,
|
|
20
|
+
sharedQueryExcludesSoftDeleted,
|
|
21
|
+
sharedQuerySchema,
|
|
22
|
+
sharedQueryTableName,
|
|
23
|
+
SQLITE_APP_QUERY_DIALECT,
|
|
10
24
|
buildGetLabel,
|
|
11
25
|
buildLabelNodeWatermarks,
|
|
26
|
+
buildBucketDigest,
|
|
27
|
+
buildScanSinceForNode,
|
|
28
|
+
collectSince,
|
|
29
|
+
mergeDigestBuckets,
|
|
30
|
+
planNodeScans,
|
|
31
|
+
toDigestBuckets,
|
|
32
|
+
DEFAULT_BUCKET_PREFIX_LENGTH,
|
|
12
33
|
buildLabelRetraction,
|
|
13
34
|
buildLabelValueReplacementTombstone,
|
|
14
35
|
buildLabelSnapshotUpsert,
|
|
@@ -16,15 +37,25 @@ import {
|
|
|
16
37
|
buildLabelsByRecordIds,
|
|
17
38
|
buildQueryLabels,
|
|
18
39
|
buildTombstoneLabelsForRecord,
|
|
40
|
+
buildRestoreLabelsForRecord,
|
|
19
41
|
groupLabelsByRecordId,
|
|
20
|
-
|
|
42
|
+
emptyLabelPage,
|
|
43
|
+
LABEL_QUERY_TARGET,
|
|
44
|
+
labelPageFrom,
|
|
45
|
+
planFindByLabel,
|
|
21
46
|
paginateLabelScan,
|
|
22
47
|
rowToLabel as rowToLabel2
|
|
23
48
|
} from "@starkeep/storage-adapter";
|
|
24
49
|
import { sql as kSql } from "kysely";
|
|
25
50
|
|
|
26
51
|
// src/serialization.ts
|
|
27
|
-
import {
|
|
52
|
+
import {
|
|
53
|
+
serializeHLC,
|
|
54
|
+
deserializeHLC,
|
|
55
|
+
createStarkeepId,
|
|
56
|
+
isStandInRole,
|
|
57
|
+
standInSlot
|
|
58
|
+
} from "@starkeep/protocol-primitives";
|
|
28
59
|
import { rowToLabel, labelToRow } from "@starkeep/storage-adapter";
|
|
29
60
|
function recordToRow(record) {
|
|
30
61
|
return {
|
|
@@ -41,7 +72,12 @@ function recordToRow(record) {
|
|
|
41
72
|
size_bytes: record.sizeBytes,
|
|
42
73
|
original_filename: record.originalFilename,
|
|
43
74
|
origin_app_id: record.originAppId,
|
|
44
|
-
parent_id: record.parentId
|
|
75
|
+
parent_id: record.parentId,
|
|
76
|
+
stand_in_role: record.standInRole,
|
|
77
|
+
fidelity: record.fidelity,
|
|
78
|
+
canonical_threshold: record.canonicalThreshold,
|
|
79
|
+
self_canonical: record.selfCanonical ? 1 : 0,
|
|
80
|
+
stand_in_slot: standInSlot(record)
|
|
45
81
|
};
|
|
46
82
|
}
|
|
47
83
|
function rowToRecord(row) {
|
|
@@ -59,11 +95,23 @@ function rowToRecord(row) {
|
|
|
59
95
|
sizeBytes: row.size_bytes,
|
|
60
96
|
originalFilename: row.original_filename,
|
|
61
97
|
originAppId: row.origin_app_id,
|
|
62
|
-
parentId: row.parent_id ? createStarkeepId(row.parent_id) : null
|
|
98
|
+
parentId: row.parent_id ? createStarkeepId(row.parent_id) : null,
|
|
99
|
+
standInRole: isStandInRole(row.stand_in_role) ? row.stand_in_role : null,
|
|
100
|
+
// DSQL returns a bigint column as a string; the value is small, so a
|
|
101
|
+
// Number is exact.
|
|
102
|
+
fidelity: row.fidelity === null || row.fidelity === void 0 ? null : Number(row.fidelity),
|
|
103
|
+
canonicalThreshold: row.canonical_threshold === null || row.canonical_threshold === void 0 ? null : Number(row.canonical_threshold),
|
|
104
|
+
selfCanonical: row.self_canonical === 1
|
|
63
105
|
};
|
|
64
106
|
}
|
|
65
107
|
|
|
66
108
|
// src/query-builder.ts
|
|
109
|
+
import {
|
|
110
|
+
buildRecordCount,
|
|
111
|
+
buildRecordSelect,
|
|
112
|
+
buildRecordTypeCounts
|
|
113
|
+
} from "@starkeep/storage-adapter";
|
|
114
|
+
import { sqliteMetadataTableName } from "@starkeep/protocol-primitives";
|
|
67
115
|
import {
|
|
68
116
|
DummyDriver,
|
|
69
117
|
Kysely,
|
|
@@ -71,24 +119,6 @@ import {
|
|
|
71
119
|
SqliteIntrospector,
|
|
72
120
|
SqliteQueryCompiler
|
|
73
121
|
} from "kysely";
|
|
74
|
-
var FIELD_MAP = {
|
|
75
|
-
id: "id",
|
|
76
|
-
type: "type",
|
|
77
|
-
createdAt: "created_at",
|
|
78
|
-
updatedAt: "updated_at",
|
|
79
|
-
deletedAt: "deleted_at",
|
|
80
|
-
version: "version",
|
|
81
|
-
contentHash: "content_hash",
|
|
82
|
-
objectStorageKey: "object_storage_key",
|
|
83
|
-
mimeType: "mime_type",
|
|
84
|
-
sizeBytes: "size_bytes",
|
|
85
|
-
originAppId: "origin_app_id",
|
|
86
|
-
parentId: "parent_id",
|
|
87
|
-
originalFilename: "original_filename"
|
|
88
|
-
};
|
|
89
|
-
function mapField(field) {
|
|
90
|
-
return FIELD_MAP[field] ?? field;
|
|
91
|
-
}
|
|
92
122
|
var compiler = new Kysely({
|
|
93
123
|
dialect: {
|
|
94
124
|
createAdapter: () => new SqliteAdapter(),
|
|
@@ -97,83 +127,70 @@ var compiler = new Kysely({
|
|
|
97
127
|
createQueryCompiler: () => new SqliteQueryCompiler()
|
|
98
128
|
}
|
|
99
129
|
});
|
|
130
|
+
var DIALECT = {
|
|
131
|
+
records: "shared_records",
|
|
132
|
+
labels: "shared_record_labels",
|
|
133
|
+
metadataTable: sqliteMetadataTableName
|
|
134
|
+
};
|
|
100
135
|
function buildSelectQuery(query) {
|
|
101
|
-
|
|
102
|
-
if (query.type) {
|
|
103
|
-
qb = qb.where("type", "=", query.type);
|
|
104
|
-
}
|
|
105
|
-
if (query.filters) {
|
|
106
|
-
for (const filter of query.filters) {
|
|
107
|
-
qb = applyFilter(qb, filter);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
if (query.cursor) {
|
|
111
|
-
qb = qb.where("id", ">", query.cursor);
|
|
112
|
-
}
|
|
113
|
-
if (query.sort && query.sort.length > 0) {
|
|
114
|
-
for (const s of query.sort) {
|
|
115
|
-
qb = qb.orderBy(mapField(s.field), s.direction === "desc" ? "desc" : "asc");
|
|
116
|
-
}
|
|
117
|
-
} else {
|
|
118
|
-
qb = qb.orderBy("id", "asc");
|
|
119
|
-
}
|
|
120
|
-
if (query.limit) {
|
|
121
|
-
qb = qb.limit(query.limit + 1);
|
|
122
|
-
}
|
|
123
|
-
const compiled = qb.compile();
|
|
136
|
+
const compiled = buildRecordSelect(compiler, DIALECT, query);
|
|
124
137
|
return { sql: compiled.sql, params: [...compiled.parameters] };
|
|
125
138
|
}
|
|
126
|
-
function
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
case "gt":
|
|
134
|
-
return qb.where(col, ">", filter.value);
|
|
135
|
-
case "gte":
|
|
136
|
-
return qb.where(col, ">=", filter.value);
|
|
137
|
-
case "lt":
|
|
138
|
-
return qb.where(col, "<", filter.value);
|
|
139
|
-
case "lte":
|
|
140
|
-
return qb.where(col, "<=", filter.value);
|
|
141
|
-
case "in":
|
|
142
|
-
return qb.where(col, "in", filter.value);
|
|
143
|
-
case "like":
|
|
144
|
-
return qb.where(col, "like", `%${filter.value}%`);
|
|
145
|
-
case "isNull":
|
|
146
|
-
return qb.where((eb) => eb(col, "is", null));
|
|
147
|
-
case "isNotNull":
|
|
148
|
-
return qb.where((eb) => eb(col, "is not", null));
|
|
149
|
-
default:
|
|
150
|
-
return qb;
|
|
151
|
-
}
|
|
139
|
+
function buildCountQuery(query) {
|
|
140
|
+
const compiled = buildRecordCount(compiler, DIALECT, query);
|
|
141
|
+
return { sql: compiled.sql, params: [...compiled.parameters] };
|
|
142
|
+
}
|
|
143
|
+
function buildTypeCountsQuery(query) {
|
|
144
|
+
const compiled = buildRecordTypeCounts(compiler, DIALECT, query);
|
|
145
|
+
return { sql: compiled.sql, params: [...compiled.parameters] };
|
|
152
146
|
}
|
|
153
147
|
|
|
154
148
|
// src/schema/bootstrap.ts
|
|
155
149
|
import { sql } from "kysely";
|
|
156
|
-
import {
|
|
150
|
+
import {
|
|
151
|
+
CATEGORIES,
|
|
152
|
+
sqliteMetadataDdl,
|
|
153
|
+
metadataIndexDdls,
|
|
154
|
+
hasMetadataTable
|
|
155
|
+
} from "@starkeep/protocol-primitives";
|
|
156
|
+
function applyConnectionPragmas(db) {
|
|
157
|
+
db.exec("PRAGMA case_sensitive_like = ON");
|
|
158
|
+
}
|
|
157
159
|
function applyLocalSchemaDdl(db) {
|
|
158
160
|
db.exec("PRAGMA journal_mode = WAL");
|
|
159
161
|
db.exec("PRAGMA foreign_keys = ON");
|
|
162
|
+
applyConnectionPragmas(db);
|
|
160
163
|
db.exec(
|
|
161
|
-
compiler.schema.createTable("shared_records").ifNotExists().addColumn("id", "text", (c) => c.primaryKey()).addColumn("type", "text", (c) => c.notNull()).addColumn("created_at", "text", (c) => c.notNull()).addColumn("updated_at", "text", (c) => c.notNull()).addColumn("node_id", "text", (c) => c.notNull()).addColumn("deleted_at", "text").addColumn("version", "integer", (c) => c.notNull().defaultTo(1)).addColumn("content_hash", "text", (c) => c.notNull()).addColumn("object_storage_key", "text", (c) => c.notNull()).addColumn("mime_type", "text").addColumn("size_bytes", "integer", (c) => c.notNull()).addColumn("original_filename", "text").addColumn("origin_app_id", "text", (c) => c.notNull()).addColumn("parent_id", "text").
|
|
164
|
+
compiler.schema.createTable("shared_records").ifNotExists().addColumn("id", "text", (c) => c.primaryKey()).addColumn("type", "text", (c) => c.notNull()).addColumn("created_at", "text", (c) => c.notNull()).addColumn("updated_at", "text", (c) => c.notNull()).addColumn("node_id", "text", (c) => c.notNull()).addColumn("deleted_at", "text").addColumn("version", "integer", (c) => c.notNull().defaultTo(1)).addColumn("content_hash", "text", (c) => c.notNull()).addColumn("object_storage_key", "text", (c) => c.notNull()).addColumn("mime_type", "text").addColumn("size_bytes", "integer", (c) => c.notNull()).addColumn("original_filename", "text").addColumn("origin_app_id", "text", (c) => c.notNull()).addColumn("parent_id", "text").addColumn(
|
|
165
|
+
"stand_in_role",
|
|
166
|
+
"text",
|
|
167
|
+
(c) => c.check(sql`stand_in_role IN ('canonical', 'smaller')`)
|
|
168
|
+
).addColumn("fidelity", "integer").addColumn("canonical_threshold", "integer").addColumn("self_canonical", "integer").addColumn("stand_in_slot", "text").compile().sql
|
|
162
169
|
);
|
|
163
170
|
const sharedRecordsIndexes = [
|
|
164
171
|
compiler.schema.createIndex("idx_shared_records_type").ifNotExists().on("shared_records").column("type"),
|
|
165
172
|
compiler.schema.createIndex("idx_shared_records_node_watermark").ifNotExists().on("shared_records").columns(["node_id", "updated_at"]),
|
|
166
173
|
compiler.schema.createIndex("idx_shared_records_origin_app").ifNotExists().on("shared_records").column("origin_app_id"),
|
|
167
174
|
compiler.schema.createIndex("idx_shared_records_parent_id").ifNotExists().on("shared_records").column("parent_id"),
|
|
168
|
-
//
|
|
169
|
-
//
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
|
|
175
|
+
// Record-level dedup: "is there already a live record for these bytes
|
|
176
|
+
// under this parent?", asked on **every** registration.
|
|
177
|
+
//
|
|
178
|
+
// The (filename, content_hash) unique index below cannot serve it — its
|
|
179
|
+
// leading column is the filename, so a hash-only lookup would scan the
|
|
180
|
+
// table once per write. Leading with content_hash is what makes the dedup
|
|
181
|
+
// check affordable enough to be unconditional, and unconditional is what
|
|
182
|
+
// makes "one object key, one record" an invariant rather than a hope.
|
|
183
|
+
compiler.schema.createIndex("idx_shared_records_content_hash").ifNotExists().on("shared_records").columns(["content_hash", "parent_id", "deleted_at"])
|
|
173
184
|
];
|
|
174
185
|
for (const index of sharedRecordsIndexes) {
|
|
175
186
|
db.exec(index.compile().sql);
|
|
176
187
|
}
|
|
188
|
+
db.exec(
|
|
189
|
+
compiler.schema.createIndex("uq_shared_records_stand_in_slot").ifNotExists().unique().on("shared_records").columns(["parent_id", "stand_in_slot"]).compile().sql
|
|
190
|
+
);
|
|
191
|
+
db.exec(
|
|
192
|
+
compiler.schema.createIndex("uq_shared_records_parent_filename_hash").ifNotExists().unique().on("shared_records").expression(sql`COALESCE(parent_id, ''), COALESCE(original_filename, ''), content_hash`).where(sql.ref("deleted_at"), "is", null).compile().sql
|
|
193
|
+
);
|
|
177
194
|
db.exec(
|
|
178
195
|
compiler.schema.createTable("shared_access_grants").ifNotExists().addColumn("app_id", "text", (c) => c.notNull()).addColumn("type_id", "text", (c) => c.notNull()).addColumn("access", "text", (c) => c.notNull().check(sql`access IN ('read', 'readwrite')`)).addColumn("metadata_write", "integer", (c) => c.notNull().defaultTo(0)).addColumn("created_at", "text", (c) => c.notNull().defaultTo(sql`(datetime('now'))`)).addPrimaryKeyConstraint("pk_shared_access_grants", ["app_id", "type_id"]).compile().sql
|
|
179
196
|
);
|
|
@@ -201,6 +218,9 @@ function applyLocalSchemaDdl(db) {
|
|
|
201
218
|
for (const index of sharedRecordLabelsIndexes) {
|
|
202
219
|
db.exec(index.compile().sql);
|
|
203
220
|
}
|
|
221
|
+
db.exec(
|
|
222
|
+
compiler.schema.createTable("shared_object_availability").ifNotExists().addColumn("object_storage_key", "text", (c) => c.primaryKey()).addColumn("state", "text", (c) => c.notNull()).addColumn("tier", "text").addColumn("expected_latency_hours", "integer").addColumn("ready_at_ms", "integer").addColumn("restored_until_ms", "integer").addColumn("observed_at_ms", "integer", (c) => c.notNull()).compile().sql
|
|
223
|
+
);
|
|
204
224
|
db.exec(
|
|
205
225
|
compiler.schema.createTable("shared_app_label_keys").ifNotExists().addColumn("app_id", "text", (c) => c.notNull()).addColumn("key", "text", (c) => c.notNull()).addColumn("description", "text").addColumn("created_at", "text", (c) => c.notNull().defaultTo(sql`(datetime('now'))`)).addPrimaryKeyConstraint("pk_shared_app_label_keys", ["app_id", "key"]).compile().sql
|
|
206
226
|
);
|
|
@@ -226,8 +246,9 @@ function applyLocalSchemaDdl(db) {
|
|
|
226
246
|
compiler.schema.createTable("app_syncable_namespaces").ifNotExists().addColumn("app_id", "text", (c) => c.primaryKey()).addColumn("tables_json", "text", (c) => c.notNull()).addColumn("files_enabled", "integer", (c) => c.notNull().defaultTo(0)).addColumn("created_at", "text", (c) => c.notNull().defaultTo(sql`(datetime('now'))`)).compile().sql
|
|
227
247
|
);
|
|
228
248
|
for (const c of CATEGORIES) {
|
|
229
|
-
if (c.id
|
|
249
|
+
if (!hasMetadataTable(c.id)) continue;
|
|
230
250
|
db.exec(sqliteMetadataDdl(c));
|
|
251
|
+
for (const index of metadataIndexDdls(c, "sqlite")) db.exec(index.sql);
|
|
231
252
|
}
|
|
232
253
|
}
|
|
233
254
|
function initializeLocalSchema(db) {
|
|
@@ -241,29 +262,60 @@ function initializeLocalSchema(db) {
|
|
|
241
262
|
}
|
|
242
263
|
}
|
|
243
264
|
|
|
265
|
+
// src/row-values.ts
|
|
266
|
+
function booleanColumnNames(columns) {
|
|
267
|
+
if (!columns) return null;
|
|
268
|
+
const names = columns.filter((c) => c.type === "boolean").map((c) => c.name);
|
|
269
|
+
return names.length > 0 ? new Set(names) : null;
|
|
270
|
+
}
|
|
271
|
+
function fromSqliteRows(rows, booleanColumns) {
|
|
272
|
+
if (!booleanColumns || booleanColumns.size === 0) return rows;
|
|
273
|
+
return rows.map((row) => {
|
|
274
|
+
const out = { ...row };
|
|
275
|
+
for (const name of booleanColumns) {
|
|
276
|
+
const value = out[name];
|
|
277
|
+
if (typeof value === "number") out[name] = value !== 0;
|
|
278
|
+
}
|
|
279
|
+
return out;
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
function toSqliteParam(value) {
|
|
283
|
+
if (typeof value === "boolean") return value ? 1 : 0;
|
|
284
|
+
return value;
|
|
285
|
+
}
|
|
286
|
+
|
|
244
287
|
// src/adapter.ts
|
|
288
|
+
var AVAILABILITY = "shared_object_availability";
|
|
289
|
+
function rowToAvailability(row) {
|
|
290
|
+
return {
|
|
291
|
+
objectStorageKey: row.object_storage_key,
|
|
292
|
+
state: row.state,
|
|
293
|
+
tier: row.tier,
|
|
294
|
+
expectedLatencyHours: row.expected_latency_hours,
|
|
295
|
+
readyAtMs: row.ready_at_ms,
|
|
296
|
+
restoredUntilMs: row.restored_until_ms,
|
|
297
|
+
observedAtMs: row.observed_at_ms
|
|
298
|
+
};
|
|
299
|
+
}
|
|
245
300
|
var LABELS = {
|
|
246
301
|
table: "shared_record_labels"
|
|
247
302
|
};
|
|
303
|
+
var RECORDS = "shared_records";
|
|
248
304
|
var SqliteDatabaseAdapter = class {
|
|
249
305
|
database = null;
|
|
250
306
|
options;
|
|
307
|
+
driver;
|
|
251
308
|
constructor(options) {
|
|
252
309
|
this.options = options;
|
|
310
|
+
this.driver = options.driver;
|
|
253
311
|
}
|
|
254
312
|
async init() {
|
|
255
313
|
if (this.database) return;
|
|
256
|
-
|
|
257
|
-
const dir = dirname(this.options.path);
|
|
258
|
-
if (!existsSync(dir)) {
|
|
259
|
-
mkdirSync(dir, { recursive: true });
|
|
260
|
-
}
|
|
261
|
-
}
|
|
262
|
-
this.database = new DatabaseSync(this.options.path);
|
|
314
|
+
this.database = this.driver.open(this.options.path);
|
|
263
315
|
initializeLocalSchema(this.database);
|
|
264
316
|
}
|
|
265
317
|
async close() {
|
|
266
|
-
this.database
|
|
318
|
+
if (this.database) this.driver.close(this.database);
|
|
267
319
|
this.database = null;
|
|
268
320
|
}
|
|
269
321
|
async healthCheck() {
|
|
@@ -280,25 +332,39 @@ var SqliteDatabaseAdapter = class {
|
|
|
280
332
|
return this.database;
|
|
281
333
|
}
|
|
282
334
|
/**
|
|
283
|
-
*
|
|
284
|
-
*
|
|
285
|
-
* database file.
|
|
335
|
+
* The raw SQLite connection, so sibling subsystems (the sync engine's change
|
|
336
|
+
* log and state store, the resident set) can create side tables in the same
|
|
337
|
+
* database file. Only valid after `init()`.
|
|
338
|
+
*
|
|
339
|
+
* Typed as {@link RawDatabase} rather than `DatabaseSync`: returning the
|
|
340
|
+
* concrete Node type is what made a second driver impossible, since every
|
|
341
|
+
* consumer was then nailed to `node:sqlite`, which React Native does not
|
|
342
|
+
* have. `DatabaseSync` satisfies the narrower interface structurally, so
|
|
343
|
+
* nothing about this connection changes — only what callers may reach for.
|
|
286
344
|
*/
|
|
287
345
|
getRawDatabase() {
|
|
288
346
|
return this.getDatabase();
|
|
289
347
|
}
|
|
290
|
-
|
|
291
|
-
|
|
348
|
+
// The casts these three used to carry named `DatabaseSync["prepare"]` to
|
|
349
|
+
// borrow node:sqlite's parameter types. `RawStatement` declares `unknown[]`
|
|
350
|
+
// directly, so they were already no-ops — and they were the last thing in
|
|
351
|
+
// this file referring to a concrete driver.
|
|
352
|
+
//
|
|
353
|
+
// All three map their parameters through `toSqliteParam`, which is what makes
|
|
354
|
+
// `row-values.ts`'s claim true — every bind on this connection, reads and
|
|
355
|
+
// writes alike. It was true only of the query path until 2026-09-10, so a
|
|
356
|
+
// declared `boolean` written through `putMetadata` reached the driver as a
|
|
357
|
+
// JavaScript boolean and threw "Provided value cannot be bound to SQLite
|
|
358
|
+
// parameter". Nothing had exercised it because no metadata column was a
|
|
359
|
+
// boolean until `exif_present`.
|
|
360
|
+
runStmt(sql4, ...params) {
|
|
361
|
+
this.getDatabase().prepare(sql4).run(...params.map(toSqliteParam));
|
|
292
362
|
}
|
|
293
|
-
getRow(
|
|
294
|
-
return this.getDatabase().prepare(
|
|
295
|
-
...params
|
|
296
|
-
);
|
|
363
|
+
getRow(sql4, ...params) {
|
|
364
|
+
return this.getDatabase().prepare(sql4).get(...params.map(toSqliteParam));
|
|
297
365
|
}
|
|
298
|
-
allRows(
|
|
299
|
-
return this.getDatabase().prepare(
|
|
300
|
-
...params
|
|
301
|
-
);
|
|
366
|
+
allRows(sql4, ...params) {
|
|
367
|
+
return this.getDatabase().prepare(sql4).all(...params.map(toSqliteParam));
|
|
302
368
|
}
|
|
303
369
|
async put(record) {
|
|
304
370
|
const row = recordToRow(record);
|
|
@@ -319,7 +385,7 @@ var SqliteDatabaseAdapter = class {
|
|
|
319
385
|
}
|
|
320
386
|
async delete(id, hlc) {
|
|
321
387
|
const ts = serializeHLC2(hlc);
|
|
322
|
-
const query = compiler.updateTable("shared_records").set({ deleted_at: ts, updated_at: ts, node_id: hlc.nodeId }).where("id", "=", id).compile();
|
|
388
|
+
const query = compiler.updateTable("shared_records").set({ deleted_at: ts, updated_at: ts, node_id: hlc.nodeId, stand_in_slot: null }).where("id", "=", id).compile();
|
|
323
389
|
this.runStmt(query.sql, ...query.parameters);
|
|
324
390
|
}
|
|
325
391
|
async getNodeWatermarks() {
|
|
@@ -331,18 +397,100 @@ var SqliteDatabaseAdapter = class {
|
|
|
331
397
|
}
|
|
332
398
|
return out;
|
|
333
399
|
}
|
|
400
|
+
async querySince(peerWatermarks, limit) {
|
|
401
|
+
const scans = planNodeScans(await this.getNodeWatermarks(), peerWatermarks);
|
|
402
|
+
const { rows, hasMore, truncated } = await collectSince(
|
|
403
|
+
scans,
|
|
404
|
+
limit,
|
|
405
|
+
async (scan, remaining) => {
|
|
406
|
+
const compiled = buildScanSinceForNode(compiler, RECORDS, scan, remaining);
|
|
407
|
+
return this.allRows(compiled.sql, ...compiled.parameters);
|
|
408
|
+
},
|
|
409
|
+
(row) => deserializeHLC2(row["updated_at"])
|
|
410
|
+
);
|
|
411
|
+
return { rows: rows.map(rowToRecord), hasMore, truncated };
|
|
412
|
+
}
|
|
413
|
+
async queryLabelsSince(peerWatermarks, limit) {
|
|
414
|
+
const scans = planNodeScans(await this.getLabelNodeWatermarks(), peerWatermarks);
|
|
415
|
+
const { rows, hasMore, truncated } = await collectSince(
|
|
416
|
+
scans,
|
|
417
|
+
limit,
|
|
418
|
+
async (scan, remaining) => {
|
|
419
|
+
const compiled = buildScanSinceForNode(compiler, LABELS.table, scan, remaining);
|
|
420
|
+
return this.allRows(compiled.sql, ...compiled.parameters);
|
|
421
|
+
},
|
|
422
|
+
(row) => deserializeHLC2(row["updated_at"])
|
|
423
|
+
);
|
|
424
|
+
return { rows: rows.map(rowToLabel2), hasMore, truncated };
|
|
425
|
+
}
|
|
426
|
+
async bucketDigest(prefixLength = DEFAULT_BUCKET_PREFIX_LENGTH) {
|
|
427
|
+
const out = [];
|
|
428
|
+
for (const table of [RECORDS, LABELS.table]) {
|
|
429
|
+
const compiled = buildBucketDigest(compiler, table, prefixLength);
|
|
430
|
+
out.push(
|
|
431
|
+
...toDigestBuckets(
|
|
432
|
+
this.allRows(compiled.sql, ...compiled.parameters)
|
|
433
|
+
)
|
|
434
|
+
);
|
|
435
|
+
}
|
|
436
|
+
return mergeDigestBuckets(out);
|
|
437
|
+
}
|
|
334
438
|
async query(query) {
|
|
335
|
-
const { sql:
|
|
336
|
-
const rows = this.allRows(
|
|
439
|
+
const { sql: sql4, params } = buildSelectQuery(query);
|
|
440
|
+
const rows = this.allRows(sql4, ...params);
|
|
337
441
|
const limit = query.limit;
|
|
338
442
|
const hasMore = limit ? rows.length > limit : false;
|
|
339
443
|
const resultRows = hasMore ? rows.slice(0, limit) : rows;
|
|
340
444
|
return {
|
|
341
445
|
records: resultRows.map(rowToRecord),
|
|
342
|
-
|
|
446
|
+
// Cut from the row itself, ordering values and all — see `nextCursorFrom`.
|
|
447
|
+
// The page carries those values back under reserved aliases precisely so
|
|
448
|
+
// this does not have to recompute what the database ordered on.
|
|
449
|
+
nextCursor: hasMore ? nextCursorFrom(query, resultRows[resultRows.length - 1]) : null,
|
|
343
450
|
hasMore
|
|
344
451
|
};
|
|
345
452
|
}
|
|
453
|
+
async countRecords(query) {
|
|
454
|
+
const { sql: sql4, params } = buildCountQuery(query);
|
|
455
|
+
const row = this.getRow(sql4, ...params);
|
|
456
|
+
return row?.total ?? 0;
|
|
457
|
+
}
|
|
458
|
+
async countRecordsByType(query) {
|
|
459
|
+
const { sql: sql4, params } = buildTypeCountsQuery(query);
|
|
460
|
+
const rows = this.allRows(sql4, ...params);
|
|
461
|
+
return rows.map((row) => ({
|
|
462
|
+
type: row.type,
|
|
463
|
+
count: Number(row.count),
|
|
464
|
+
latestUpdatedAt: row.latest_updated_at
|
|
465
|
+
}));
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* See `DatabaseAdapter.queryShared`. Compiled by the same builder the
|
|
469
|
+
* app-syncable plane uses, against the table the target names on this engine.
|
|
470
|
+
*/
|
|
471
|
+
async queryShared(target, query, options = {}) {
|
|
472
|
+
const schema = sharedQuerySchema(target);
|
|
473
|
+
const table = sharedQueryTableName(target, "sqlite");
|
|
474
|
+
const build = {
|
|
475
|
+
serverWhere: options.serverWhere,
|
|
476
|
+
excludeSoftDeleted: sharedQueryExcludesSoftDeleted(target),
|
|
477
|
+
...options.softDeleted ? { softDeleted: options.softDeleted } : {}
|
|
478
|
+
};
|
|
479
|
+
const booleans = booleanColumnNames(schema.columns);
|
|
480
|
+
if (query.mode === "aggregate") {
|
|
481
|
+
const compiled2 = buildAppAggregateQuery(compiler, table, query, SQLITE_APP_QUERY_DIALECT, build);
|
|
482
|
+
return collectAggregatePage(query, fromSqliteRows(this.selectCompiled(compiled2), booleans));
|
|
483
|
+
}
|
|
484
|
+
const compiled = buildAppRowQuery(compiler, table, query, build);
|
|
485
|
+
return collectRowPage(query, fromSqliteRows(this.selectCompiled(compiled), booleans));
|
|
486
|
+
}
|
|
487
|
+
/** Bind and run a compiled SELECT, adapting values SQLite cannot bind. */
|
|
488
|
+
selectCompiled(compiled) {
|
|
489
|
+
return this.allRows(
|
|
490
|
+
compiled.sql,
|
|
491
|
+
...compiled.parameters.map(toSqliteParam)
|
|
492
|
+
);
|
|
493
|
+
}
|
|
346
494
|
async batch(operations) {
|
|
347
495
|
this.getDatabase().exec("BEGIN");
|
|
348
496
|
try {
|
|
@@ -377,13 +525,9 @@ var SqliteDatabaseAdapter = class {
|
|
|
377
525
|
throw new TransactionError("Transaction failed", error);
|
|
378
526
|
}
|
|
379
527
|
}
|
|
380
|
-
async putMetadata(
|
|
381
|
-
const table =
|
|
382
|
-
const values =
|
|
383
|
-
for (const [key, value] of Object.entries(row)) {
|
|
384
|
-
if (key === "recordId") continue;
|
|
385
|
-
values[key] = value;
|
|
386
|
-
}
|
|
528
|
+
async putMetadata(recordType, row) {
|
|
529
|
+
const table = sqliteMetadataTableName2(recordType);
|
|
530
|
+
const values = metadataValues(recordType, row);
|
|
387
531
|
const updateColumns = Object.keys(values).filter((c) => c !== "record_id");
|
|
388
532
|
const query = compiler.insertInto(table).values(values).onConflict(
|
|
389
533
|
(oc) => updateColumns.length > 0 ? oc.column("record_id").doUpdateSet(
|
|
@@ -393,29 +537,40 @@ var SqliteDatabaseAdapter = class {
|
|
|
393
537
|
this.runStmt(query.sql, ...query.parameters);
|
|
394
538
|
}
|
|
395
539
|
async getMetadata(typeId, recordId) {
|
|
396
|
-
const table =
|
|
540
|
+
const table = sqliteMetadataTableName2(typeId);
|
|
397
541
|
const query = compiler.selectFrom(table).selectAll().where("record_id", "=", recordId).compile();
|
|
398
542
|
const row = this.getRow(query.sql, ...query.parameters);
|
|
399
543
|
if (!row) return null;
|
|
400
|
-
return columnsToMetadataRow(recordId, row);
|
|
544
|
+
return columnsToMetadataRow(recordId, typeId, row);
|
|
401
545
|
}
|
|
402
546
|
async getMetadataByIds(typeId, recordIds) {
|
|
403
547
|
const result = /* @__PURE__ */ new Map();
|
|
404
548
|
if (recordIds.length === 0) return result;
|
|
405
|
-
const table =
|
|
549
|
+
const table = sqliteMetadataTableName2(typeId);
|
|
406
550
|
const query = compiler.selectFrom(table).selectAll().where("record_id", "in", recordIds).compile();
|
|
407
551
|
const rows = this.allRows(query.sql, ...query.parameters);
|
|
408
552
|
for (const row of rows) {
|
|
409
553
|
const recordId = row["record_id"];
|
|
410
|
-
result.set(recordId, columnsToMetadataRow(recordId, row));
|
|
554
|
+
result.set(recordId, columnsToMetadataRow(recordId, typeId, row));
|
|
411
555
|
}
|
|
412
556
|
return result;
|
|
413
557
|
}
|
|
414
558
|
async deleteMetadata(typeId, recordId) {
|
|
415
|
-
const table =
|
|
559
|
+
const table = sqliteMetadataTableName2(typeId);
|
|
416
560
|
const query = compiler.deleteFrom(table).where("record_id", "=", recordId).compile();
|
|
417
561
|
this.runStmt(query.sql, ...query.parameters);
|
|
418
562
|
}
|
|
563
|
+
async tombstoneMetadata(typeId, recordId, hlc) {
|
|
564
|
+
this.setMetadataTombstone(typeId, recordId, serializeHLC2(hlc));
|
|
565
|
+
}
|
|
566
|
+
async restoreMetadata(typeId, recordId) {
|
|
567
|
+
this.setMetadataTombstone(typeId, recordId, null);
|
|
568
|
+
}
|
|
569
|
+
setMetadataTombstone(typeId, recordId, value) {
|
|
570
|
+
const table = sqliteMetadataTableName2(typeId);
|
|
571
|
+
const query = compiler.updateTable(table).set({ [METADATA_DELETED_AT_COLUMN]: value }).where("record_id", "=", recordId).compile();
|
|
572
|
+
this.runStmt(query.sql, ...query.parameters);
|
|
573
|
+
}
|
|
419
574
|
// ---- Cross-app record labels -------------------------------------------
|
|
420
575
|
//
|
|
421
576
|
// The SQL for all of these is built in `@starkeep/storage-adapter`, shared
|
|
@@ -465,15 +620,22 @@ var SqliteDatabaseAdapter = class {
|
|
|
465
620
|
async getLabelsByRecordIds(recordIds) {
|
|
466
621
|
if (recordIds.length === 0) return /* @__PURE__ */ new Map();
|
|
467
622
|
const query = buildLabelsByRecordIds(compiler, LABELS, recordIds);
|
|
468
|
-
return groupLabelsByRecordId(
|
|
469
|
-
this.allRows(query.sql, ...query.parameters)
|
|
470
|
-
);
|
|
623
|
+
return groupLabelsByRecordId(this.allRows(query.sql, ...query.parameters));
|
|
471
624
|
}
|
|
625
|
+
/**
|
|
626
|
+
* The reverse label read, run through the query grammar — see
|
|
627
|
+
* `label-find.ts`. The grant rides in as `serverWhere` rather than as one of
|
|
628
|
+
* the caller's own predicates, which is the separation every other shared
|
|
629
|
+
* query draws.
|
|
630
|
+
*/
|
|
472
631
|
async findByLabel(query) {
|
|
473
|
-
const
|
|
474
|
-
if (!
|
|
475
|
-
|
|
476
|
-
|
|
632
|
+
const plan = planFindByLabel(query);
|
|
633
|
+
if (!plan) return emptyLabelPage();
|
|
634
|
+
return labelPageFrom(
|
|
635
|
+
await this.queryShared(LABEL_QUERY_TARGET, plan.query, {
|
|
636
|
+
serverWhere: plan.serverWhere
|
|
637
|
+
})
|
|
638
|
+
);
|
|
477
639
|
}
|
|
478
640
|
// ---- Label sync ---------------------------------------------------------
|
|
479
641
|
async putLabel(label) {
|
|
@@ -497,21 +659,96 @@ var SqliteDatabaseAdapter = class {
|
|
|
497
659
|
for (const row of rows) out[row.node_id] = deserializeHLC2(row.max_updated_at);
|
|
498
660
|
return out;
|
|
499
661
|
}
|
|
662
|
+
// ---- Object availability ------------------------------------------------
|
|
663
|
+
async getAvailability(objectStorageKeys) {
|
|
664
|
+
const out = /* @__PURE__ */ new Map();
|
|
665
|
+
if (objectStorageKeys.length === 0) return out;
|
|
666
|
+
const compiled = compiler.selectFrom(AVAILABILITY).selectAll().where("object_storage_key", "in", objectStorageKeys).compile();
|
|
667
|
+
const rows = this.allRows(compiled.sql, ...compiled.parameters);
|
|
668
|
+
for (const row of rows) out.set(row.object_storage_key, rowToAvailability(row));
|
|
669
|
+
return out;
|
|
670
|
+
}
|
|
671
|
+
async putAvailability(row) {
|
|
672
|
+
const compiled = compiler.insertInto(AVAILABILITY).values({
|
|
673
|
+
object_storage_key: row.objectStorageKey,
|
|
674
|
+
state: row.state,
|
|
675
|
+
tier: row.tier,
|
|
676
|
+
expected_latency_hours: row.expectedLatencyHours,
|
|
677
|
+
ready_at_ms: row.readyAtMs,
|
|
678
|
+
restored_until_ms: row.restoredUntilMs,
|
|
679
|
+
observed_at_ms: row.observedAtMs
|
|
680
|
+
}).onConflict(
|
|
681
|
+
(oc) => oc.column("object_storage_key").doUpdateSet((eb) => ({
|
|
682
|
+
state: eb.ref("excluded.state"),
|
|
683
|
+
tier: eb.ref("excluded.tier"),
|
|
684
|
+
expected_latency_hours: eb.ref("excluded.expected_latency_hours"),
|
|
685
|
+
ready_at_ms: eb.ref("excluded.ready_at_ms"),
|
|
686
|
+
restored_until_ms: eb.ref("excluded.restored_until_ms"),
|
|
687
|
+
observed_at_ms: eb.ref("excluded.observed_at_ms")
|
|
688
|
+
}))
|
|
689
|
+
).compile();
|
|
690
|
+
this.runStmt(compiled.sql, ...compiled.parameters);
|
|
691
|
+
}
|
|
692
|
+
async countRestoringObjects() {
|
|
693
|
+
const compiled = compiler.selectFrom(AVAILABILITY).innerJoin(
|
|
694
|
+
"shared_records",
|
|
695
|
+
"shared_records.object_storage_key",
|
|
696
|
+
`${AVAILABILITY}.object_storage_key`
|
|
697
|
+
).select(({ fn }) => [
|
|
698
|
+
fn.count(`${AVAILABILITY}.object_storage_key`).as("object_count"),
|
|
699
|
+
fn.sum("shared_records.size_bytes").as("bytes")
|
|
700
|
+
]).where(`${AVAILABILITY}.state`, "=", kSql.lit("restoring")).compile();
|
|
701
|
+
const row = this.getRow(
|
|
702
|
+
compiled.sql,
|
|
703
|
+
...compiled.parameters
|
|
704
|
+
);
|
|
705
|
+
return { objectCount: row?.object_count ?? 0, bytes: row?.bytes ?? 0 };
|
|
706
|
+
}
|
|
500
707
|
async tombstoneLabelsForRecord(recordId, hlc) {
|
|
501
708
|
const query = buildTombstoneLabelsForRecord(compiler, LABELS, recordId, hlc);
|
|
502
709
|
this.runStmt(query.sql, ...query.parameters);
|
|
503
710
|
}
|
|
711
|
+
async restoreLabelsForRecord(recordId, deletedAt, hlc) {
|
|
712
|
+
const query = buildRestoreLabelsForRecord(compiler, LABELS, recordId, deletedAt, hlc);
|
|
713
|
+
this.runStmt(query.sql, ...query.parameters);
|
|
714
|
+
}
|
|
504
715
|
};
|
|
505
|
-
function columnsToMetadataRow(recordId, columns) {
|
|
716
|
+
function columnsToMetadataRow(recordId, typeId, columns) {
|
|
717
|
+
const category = typeCategory(typeId) ?? typeId;
|
|
718
|
+
const booleans = booleanColumnNames(getCategory(category)?.metadataColumns);
|
|
506
719
|
const row = { recordId };
|
|
507
720
|
for (const [key, value] of Object.entries(columns)) {
|
|
508
|
-
if (key === "record_id") continue;
|
|
509
|
-
row[key] = value;
|
|
721
|
+
if (key === "record_id" || key === METADATA_DELETED_AT_COLUMN) continue;
|
|
722
|
+
row[key] = booleans?.has(key) && typeof value === "number" ? value !== 0 : value;
|
|
510
723
|
}
|
|
511
724
|
return row;
|
|
512
725
|
}
|
|
726
|
+
function metadataValues(recordType, row) {
|
|
727
|
+
if (!isKnownType(recordType)) {
|
|
728
|
+
throw new StorageError(
|
|
729
|
+
`putMetadata needs the record's own type, not "${recordType}": the ${METADATA_DISCRIMINANT_COLUMN} column gates every read of this row`
|
|
730
|
+
);
|
|
731
|
+
}
|
|
732
|
+
const values = {
|
|
733
|
+
record_id: row.recordId,
|
|
734
|
+
[METADATA_DISCRIMINANT_COLUMN]: recordType,
|
|
735
|
+
// Writing metadata asserts the row is live. Without this an app re-reporting
|
|
736
|
+
// a restored record's dimensions would write into a row still carrying its
|
|
737
|
+
// tombstone, and the one read that filters on the stamp would keep answering
|
|
738
|
+
// nothing. Server-owned either way: a wire row carrying it is ignored below.
|
|
739
|
+
[METADATA_DELETED_AT_COLUMN]: null
|
|
740
|
+
};
|
|
741
|
+
for (const [key, value] of Object.entries(row)) {
|
|
742
|
+
if (key === "recordId" || key === METADATA_DISCRIMINANT_COLUMN || key === METADATA_DELETED_AT_COLUMN) {
|
|
743
|
+
continue;
|
|
744
|
+
}
|
|
745
|
+
values[key] = value;
|
|
746
|
+
}
|
|
747
|
+
return values;
|
|
748
|
+
}
|
|
513
749
|
|
|
514
750
|
// src/app-syncable/namespace.ts
|
|
751
|
+
import { parseAppSyncableTables } from "@starkeep/shared-space-api";
|
|
515
752
|
function normalizeAppId(appId) {
|
|
516
753
|
return appId.toLowerCase().replace(/-/g, "_");
|
|
517
754
|
}
|
|
@@ -519,7 +756,7 @@ function appSyncableTableName(appId, tableName) {
|
|
|
519
756
|
return `${normalizeAppId(appId)}_syncable_${tableName}`;
|
|
520
757
|
}
|
|
521
758
|
function rowToNamespace(r) {
|
|
522
|
-
const tables =
|
|
759
|
+
const tables = parseAppSyncableTables(r.app_id, r.tables_json);
|
|
523
760
|
return {
|
|
524
761
|
appId: r.app_id,
|
|
525
762
|
tables,
|
|
@@ -571,11 +808,26 @@ var SqliteAppSyncableNamespaceStore = class {
|
|
|
571
808
|
// src/app-syncable/apply.ts
|
|
572
809
|
import { sql as sql2 } from "kysely";
|
|
573
810
|
import { serializeHLC as serializeHLC3, deserializeHLC as deserializeHLC3 } from "@starkeep/protocol-primitives";
|
|
811
|
+
import {
|
|
812
|
+
buildAppAggregateQuery as buildAppAggregateQuery2,
|
|
813
|
+
buildAppRowQuery as buildAppRowQuery2,
|
|
814
|
+
collectAggregatePage as collectAggregatePage2,
|
|
815
|
+
collectRowPage as collectRowPage2,
|
|
816
|
+
SQLITE_APP_QUERY_DIALECT as SQLITE_APP_QUERY_DIALECT2,
|
|
817
|
+
buildBucketDigest as buildBucketDigest2,
|
|
818
|
+
buildScanSinceForNode as buildScanSinceForNode2,
|
|
819
|
+
collectSince as collectSince2,
|
|
820
|
+
planNodeScans as planNodeScans2,
|
|
821
|
+
requireKeyedWhere,
|
|
822
|
+
rowToWireEntry,
|
|
823
|
+
toDigestBuckets as toDigestBuckets2,
|
|
824
|
+
DEFAULT_BUCKET_PREFIX_LENGTH as DEFAULT_BUCKET_PREFIX_LENGTH2
|
|
825
|
+
} from "@starkeep/storage-adapter";
|
|
574
826
|
function runCompiled(db, compiled) {
|
|
575
|
-
db.prepare(compiled.sql).run(...compiled.parameters);
|
|
827
|
+
db.prepare(compiled.sql).run(...compiled.parameters.map(toSqliteParam));
|
|
576
828
|
}
|
|
577
829
|
function allCompiled(db, compiled) {
|
|
578
|
-
return db.prepare(compiled.sql).all(...compiled.parameters);
|
|
830
|
+
return db.prepare(compiled.sql).all(...compiled.parameters.map(toSqliteParam));
|
|
579
831
|
}
|
|
580
832
|
var SqliteAppSyncableApplier = class {
|
|
581
833
|
constructor(db, namespace) {
|
|
@@ -587,9 +839,7 @@ var SqliteAppSyncableApplier = class {
|
|
|
587
839
|
apply(entry) {
|
|
588
840
|
const ns = this.namespace.get(entry.appId);
|
|
589
841
|
if (!ns) {
|
|
590
|
-
throw new Error(
|
|
591
|
-
`SqliteAppSyncableApplier: app "${entry.appId}" not installed`
|
|
592
|
-
);
|
|
842
|
+
throw new Error(`SqliteAppSyncableApplier: app "${entry.appId}" not installed`);
|
|
593
843
|
}
|
|
594
844
|
const tableInfo = ns.tables.find((t) => t.name === entry.table);
|
|
595
845
|
if (!tableInfo) {
|
|
@@ -602,9 +852,9 @@ var SqliteAppSyncableApplier = class {
|
|
|
602
852
|
if (entry.op === "insert") {
|
|
603
853
|
this.applyInsert(fullName, pkColumns, entry);
|
|
604
854
|
} else if (entry.op === "update") {
|
|
605
|
-
this.applyUpdate(fullName, entry);
|
|
855
|
+
this.applyUpdate(fullName, pkColumns, entry);
|
|
606
856
|
} else {
|
|
607
|
-
this.applyDelete(fullName, entry);
|
|
857
|
+
this.applyDelete(fullName, pkColumns, entry);
|
|
608
858
|
}
|
|
609
859
|
}
|
|
610
860
|
applyInsert(fullName, pkColumns, entry) {
|
|
@@ -628,10 +878,10 @@ var SqliteAppSyncableApplier = class {
|
|
|
628
878
|
).compile()
|
|
629
879
|
);
|
|
630
880
|
}
|
|
631
|
-
applyUpdate(fullName, entry) {
|
|
881
|
+
applyUpdate(fullName, pkColumns, entry) {
|
|
632
882
|
const rawPatch = entry.row ?? {};
|
|
633
883
|
const patch = rawPatch["updated_at"] ? withNodeId(rawPatch, entry) : rawPatch;
|
|
634
|
-
const where = entry
|
|
884
|
+
const where = requireKeyedWhere(entry, "update", pkColumns);
|
|
635
885
|
const patchCols = Object.keys(patch);
|
|
636
886
|
const whereCols = Object.keys(where);
|
|
637
887
|
if (patchCols.length === 0) return;
|
|
@@ -641,36 +891,90 @@ var SqliteAppSyncableApplier = class {
|
|
|
641
891
|
query = query.where(c, "=", where[c]);
|
|
642
892
|
}
|
|
643
893
|
if (incomingUpdatedAt) {
|
|
644
|
-
query = query.where(
|
|
894
|
+
query = query.where(
|
|
895
|
+
(eb) => eb.or([eb("updated_at", "is", null), eb("updated_at", "<", incomingUpdatedAt)])
|
|
896
|
+
);
|
|
645
897
|
}
|
|
646
898
|
runCompiled(this.db, query.compile());
|
|
647
899
|
}
|
|
648
|
-
applyDelete(fullName, entry) {
|
|
649
|
-
const where = entry
|
|
900
|
+
applyDelete(fullName, pkColumns, entry) {
|
|
901
|
+
const where = requireKeyedWhere(entry, "delete", pkColumns);
|
|
650
902
|
const whereCols = Object.keys(where);
|
|
651
903
|
const incomingUpdatedAt = entry.row?.["updated_at"];
|
|
652
904
|
const ts = incomingUpdatedAt ?? serializeHLC3(entry.timestamp);
|
|
905
|
+
const carriesFullRow = entry.row !== void 0 && pkColumns.length > 0 && pkColumns.every((column) => entry.row?.[column] !== void 0);
|
|
906
|
+
if (carriesFullRow) {
|
|
907
|
+
const row = withNodeId({ ...entry.row, updated_at: ts, deleted_at: ts }, entry);
|
|
908
|
+
const updateCols = Object.keys(row).filter((c) => !pkColumns.includes(c));
|
|
909
|
+
runCompiled(
|
|
910
|
+
this.db,
|
|
911
|
+
compiler.insertInto(fullName).values({ ...row }).onConflict(
|
|
912
|
+
(oc) => oc.columns(pkColumns).doUpdateSet(
|
|
913
|
+
(eb) => Object.fromEntries(updateCols.map((c) => [c, eb.ref(`excluded.${c}`)]))
|
|
914
|
+
).where(sql2.ref("excluded.updated_at"), ">", sql2.ref(`${fullName}.updated_at`))
|
|
915
|
+
).compile()
|
|
916
|
+
);
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
653
919
|
let query = compiler.updateTable(fullName).set({ deleted_at: ts, updated_at: ts, node_id: nodeIdOf(ts, entry) });
|
|
654
920
|
for (const c of whereCols) {
|
|
655
921
|
query = query.where(c, "=", where[c]);
|
|
656
922
|
}
|
|
657
|
-
query = query.where(
|
|
658
|
-
(eb) => eb.or([eb("updated_at", "is", null), eb("updated_at", "<", ts)])
|
|
659
|
-
);
|
|
923
|
+
query = query.where((eb) => eb.or([eb("updated_at", "is", null), eb("updated_at", "<", ts)]));
|
|
660
924
|
runCompiled(this.db, query.compile());
|
|
661
925
|
}
|
|
662
|
-
/**
|
|
926
|
+
/**
|
|
927
|
+
* Whether this table exists in *this* database.
|
|
928
|
+
*
|
|
929
|
+
* Asked before the digest rather than inferred from a failed read, because
|
|
930
|
+
* those are two different answers and only one of them is `[]`. An app that
|
|
931
|
+
* is not installed here genuinely holds nothing; a table that is locked,
|
|
932
|
+
* corrupt or unreadable holds an unknown number of rows, and reporting zero
|
|
933
|
+
* for it is the loudest possible false alarm — every bucket in it reads as a
|
|
934
|
+
* hole, in whichever direction the comparison runs.
|
|
935
|
+
*
|
|
936
|
+
* `sqlite_master` is the cheapest question that separates them, and asking it
|
|
937
|
+
* cannot itself fail for a reason worth swallowing: a database whose schema
|
|
938
|
+
* table will not read is not one whose app tables are merely absent.
|
|
939
|
+
*/
|
|
940
|
+
tableExists(fullName) {
|
|
941
|
+
const row = this.db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`).all(fullName);
|
|
942
|
+
return row.length > 0;
|
|
943
|
+
}
|
|
944
|
+
/**
|
|
945
|
+
* See `ScanCapableApplier.bucketDigest`. Missing table → `[]`; a table that
|
|
946
|
+
* exists and will not read **throws**, so the caller's `supported: false`
|
|
947
|
+
* path can do its job.
|
|
948
|
+
*
|
|
949
|
+
* This used to be `try { … } catch { return [] }`, which is the same mistake
|
|
950
|
+
* `scanSince` made: `[]` is the wire value for "this table holds nothing", not
|
|
951
|
+
* for "I could not count it". It made both engine-side "could not count ⇒
|
|
952
|
+
* supported: false" guards unreachable on every production path, because no
|
|
953
|
+
* real applier ever threw.
|
|
954
|
+
*/
|
|
955
|
+
async bucketDigest(appId, table, prefixLength = DEFAULT_BUCKET_PREFIX_LENGTH2) {
|
|
956
|
+
const fullName = appSyncableTableName(appId, table);
|
|
957
|
+
if (!this.tableExists(fullName)) return [];
|
|
958
|
+
const compiled = buildBucketDigest2(compiler, fullName, prefixLength);
|
|
959
|
+
return toDigestBuckets2(allCompiled(this.db, compiled));
|
|
960
|
+
}
|
|
961
|
+
/**
|
|
962
|
+
* See `ScanCapableApplier.getNodeWatermarks`. Missing table → `{}`; a table
|
|
963
|
+
* that exists and will not read **throws**.
|
|
964
|
+
*
|
|
965
|
+
* The distinction matters more here than the `{}` return suggests, because
|
|
966
|
+
* `scanSince` plans its authors from this map: an unreadable table that
|
|
967
|
+
* answered `{}` produced no scans, and the scan then reported "nothing owed,
|
|
968
|
+
* every author complete" — the exact silent-loss shape `scanSince`'s own catch
|
|
969
|
+
* was written to prevent, reached one frame earlier.
|
|
970
|
+
*/
|
|
663
971
|
async getNodeWatermarks(appId, table) {
|
|
664
972
|
const fullName = appSyncableTableName(appId, table);
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
);
|
|
671
|
-
} catch {
|
|
672
|
-
return {};
|
|
673
|
-
}
|
|
973
|
+
if (!this.tableExists(fullName)) return {};
|
|
974
|
+
const rows = allCompiled(
|
|
975
|
+
this.db,
|
|
976
|
+
compiler.selectFrom(fullName).select(({ fn }) => ["node_id", fn.max("updated_at").as("max_updated_at")]).groupBy("node_id").compile()
|
|
977
|
+
);
|
|
674
978
|
const out = {};
|
|
675
979
|
for (const row of rows) {
|
|
676
980
|
out[row.node_id] = deserializeHLC3(row.max_updated_at);
|
|
@@ -678,41 +982,74 @@ var SqliteAppSyncableApplier = class {
|
|
|
678
982
|
return out;
|
|
679
983
|
}
|
|
680
984
|
/**
|
|
681
|
-
* Pull-side synthesis:
|
|
682
|
-
*
|
|
683
|
-
*
|
|
684
|
-
*
|
|
685
|
-
* tiebreaker column is needed.
|
|
985
|
+
* Pull-side synthesis: rows the peer hasn't seen, per author, seeking the
|
|
986
|
+
* `(node_id, updated_at)` index rather than reading the table and filtering.
|
|
987
|
+
* See `ScanCapableApplier.scanSince` for the contract and
|
|
988
|
+
* `storage-adapter/src/database/since-queries.ts` for why it is a loop.
|
|
686
989
|
*/
|
|
687
|
-
async scanSince(appId, table,
|
|
990
|
+
async scanSince(appId, table, peerWatermarks, limit) {
|
|
688
991
|
const fullName = appSyncableTableName(appId, table);
|
|
689
|
-
const
|
|
690
|
-
const
|
|
691
|
-
let rows;
|
|
992
|
+
const scans = planNodeScans2(await this.getNodeWatermarks(appId, table), peerWatermarks);
|
|
993
|
+
const pkColumns = this.namespace.get(appId)?.tables.find((t) => t.name === table)?.pkColumns ?? [];
|
|
692
994
|
try {
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
995
|
+
const { rows, hasMore, truncated } = await collectSince2(
|
|
996
|
+
scans,
|
|
997
|
+
limit,
|
|
998
|
+
async (scan, remaining) => {
|
|
999
|
+
const compiled = buildScanSinceForNode2(compiler, fullName, scan, remaining);
|
|
1000
|
+
return allCompiled(this.db, compiled);
|
|
1001
|
+
},
|
|
1002
|
+
(row) => deserializeHLC3(row["updated_at"])
|
|
1003
|
+
);
|
|
1004
|
+
return {
|
|
1005
|
+
// Converted before the row becomes a wire entry: what goes on the wire
|
|
1006
|
+
// is what the app sees, and a peer's applier binds a boolean it can.
|
|
1007
|
+
rows: fromSqliteRows(rows, this.booleanColumnsOf(appId, table)).map((row) => rowToWireEntry(appId, table, row, pkColumns, deserializeHLC3)).filter((entry) => entry !== null),
|
|
1008
|
+
hasMore,
|
|
1009
|
+
truncated
|
|
1010
|
+
};
|
|
1011
|
+
} catch (err) {
|
|
1012
|
+
console.warn(
|
|
1013
|
+
`[app-syncable] scanSince failed for ${appId}.${table}: ${err.message}`
|
|
1014
|
+
);
|
|
1015
|
+
const truncated = {};
|
|
1016
|
+
for (const scan of scans) truncated[scan.nodeId] = null;
|
|
1017
|
+
return { rows: [], hasMore: true, truncated };
|
|
700
1018
|
}
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
1019
|
+
}
|
|
1020
|
+
/**
|
|
1021
|
+
* Run a parsed query. The read half of the app-data plane.
|
|
1022
|
+
*
|
|
1023
|
+
* Compiled by the shared builder rather than here, so the SQL this emits and
|
|
1024
|
+
* the SQL the DSQL applier emits are one piece of code — the two hand-written
|
|
1025
|
+
* read grammars this replaces had already drifted on their limit defaults
|
|
1026
|
+
* alone.
|
|
1027
|
+
*/
|
|
1028
|
+
async runQuery(appId, table, query, options = {}) {
|
|
709
1029
|
const fullName = appSyncableTableName(appId, table);
|
|
710
|
-
const
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
1030
|
+
const booleanColumns = this.booleanColumnsOf(appId, table);
|
|
1031
|
+
if (query.mode === "aggregate") {
|
|
1032
|
+
const compiled2 = buildAppAggregateQuery2(
|
|
1033
|
+
compiler,
|
|
1034
|
+
fullName,
|
|
1035
|
+
query,
|
|
1036
|
+
SQLITE_APP_QUERY_DIALECT2,
|
|
1037
|
+
options
|
|
1038
|
+
);
|
|
1039
|
+
return collectAggregatePage2(query, fromSqliteRows(this.selectRows(compiled2), booleanColumns));
|
|
714
1040
|
}
|
|
715
|
-
|
|
1041
|
+
const compiled = buildAppRowQuery2(compiler, fullName, query, options);
|
|
1042
|
+
return collectRowPage2(query, fromSqliteRows(this.selectRows(compiled), booleanColumns));
|
|
1043
|
+
}
|
|
1044
|
+
/** The names of a declared table's `boolean` columns, or null when it has none. */
|
|
1045
|
+
booleanColumnsOf(appId, table) {
|
|
1046
|
+
return booleanColumnNames(
|
|
1047
|
+
this.namespace.get(appId)?.tables.find((t) => t.name === table)?.columns
|
|
1048
|
+
);
|
|
1049
|
+
}
|
|
1050
|
+
/** Bind and run a compiled SELECT, adapting values SQLite cannot bind. */
|
|
1051
|
+
selectRows(compiled) {
|
|
1052
|
+
return this.db.prepare(compiled.sql).all(...compiled.parameters.map(toSqliteParam));
|
|
716
1053
|
}
|
|
717
1054
|
};
|
|
718
1055
|
function withNodeId(row, entry) {
|
|
@@ -727,23 +1064,138 @@ function nodeIdOf(updatedAt, entry) {
|
|
|
727
1064
|
}
|
|
728
1065
|
return entry.timestamp.nodeId;
|
|
729
1066
|
}
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
1067
|
+
|
|
1068
|
+
// src/library-owner.ts
|
|
1069
|
+
import { sql as sql3 } from "kysely";
|
|
1070
|
+
import { LOCAL_USER_SEGMENT, USER_SEGMENT_PATTERN } from "@starkeep/protocol-primitives";
|
|
1071
|
+
var OWNER_TABLE = "library_owner";
|
|
1072
|
+
function ensureOwnerTable(db) {
|
|
1073
|
+
db.exec(
|
|
1074
|
+
compiler.schema.createTable(OWNER_TABLE).ifNotExists().addColumn("id", "integer", (c) => c.primaryKey()).addColumn("user_id", "text", (c) => c.notNull()).compile().sql
|
|
1075
|
+
);
|
|
1076
|
+
}
|
|
1077
|
+
function boundUserId(db) {
|
|
1078
|
+
ensureOwnerTable(db);
|
|
1079
|
+
const row = db.prepare(
|
|
1080
|
+
compiler.selectFrom(OWNER_TABLE).select("user_id").where("id", "=", sql3.lit(0)).compile().sql
|
|
1081
|
+
).get();
|
|
1082
|
+
return row?.user_id ?? null;
|
|
1083
|
+
}
|
|
1084
|
+
function libraryUserSegment(db) {
|
|
1085
|
+
return boundUserId(db) ?? LOCAL_USER_SEGMENT;
|
|
1086
|
+
}
|
|
1087
|
+
function libraryKeyColumns(db) {
|
|
1088
|
+
return coreKeyColumns(db).filter(({ table }) => tableExists(db, table));
|
|
1089
|
+
}
|
|
1090
|
+
function coreKeyColumns(db) {
|
|
1091
|
+
const columns = [
|
|
1092
|
+
{ table: "shared_records", column: "object_storage_key", primaryKey: false },
|
|
1093
|
+
{ table: "shared_object_availability", column: "object_storage_key", primaryKey: true },
|
|
1094
|
+
{ table: "resident_keys", column: "object_storage_key", primaryKey: true },
|
|
1095
|
+
{ table: "resident_keys", column: "record_id", primaryKey: false }
|
|
1096
|
+
];
|
|
1097
|
+
const tables = db.prepare(
|
|
1098
|
+
compiler.selectFrom("sqlite_master").select("name").where("type", "=", sql3.lit("table")).compile().sql
|
|
1099
|
+
).all();
|
|
1100
|
+
for (const { name } of tables) {
|
|
1101
|
+
if (name.endsWith("__starkeep_sync_records")) {
|
|
1102
|
+
columns.push(
|
|
1103
|
+
{ table: name, column: "id", primaryKey: true },
|
|
1104
|
+
{ table: name, column: "object_storage_key", primaryKey: false }
|
|
1105
|
+
);
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
return columns;
|
|
1109
|
+
}
|
|
1110
|
+
function tableExists(db, table) {
|
|
1111
|
+
return db.prepare(
|
|
1112
|
+
compiler.selectFrom("sqlite_master").select("name").where("type", "=", sql3.lit("table")).where("name", "=", sql3.lit(table)).compile().sql
|
|
1113
|
+
).get() !== void 0;
|
|
1114
|
+
}
|
|
1115
|
+
function namespaces(column, userId) {
|
|
1116
|
+
return [
|
|
1117
|
+
{
|
|
1118
|
+
local: sql3.lit(`shared/${LOCAL_USER_SEGMENT}/%`),
|
|
1119
|
+
// Keeps the slash after `local`, so `shared/local/x` becomes `shared/<userId>/x`.
|
|
1120
|
+
rebound: sql3`'shared/' || ${sql3.lit(userId)} || substr(${sql3.ref(column)}, ${sql3.lit(`shared/${LOCAL_USER_SEGMENT}`.length + 1)})`
|
|
1121
|
+
},
|
|
1122
|
+
// apps/<appId>/local/syncable/... — the app id is the segment before `local`.
|
|
1123
|
+
{
|
|
1124
|
+
local: sql3.lit(`apps/%/${LOCAL_USER_SEGMENT}/syncable/%`),
|
|
1125
|
+
rebound: sql3`replace(${sql3.ref(column)}, ${sql3.lit(`/${LOCAL_USER_SEGMENT}/syncable/`)}, ${sql3.lit(`/${userId}/syncable/`)})`
|
|
1126
|
+
}
|
|
1127
|
+
];
|
|
1128
|
+
}
|
|
1129
|
+
function rebindColumn(db, { table, column, primaryKey }, userId) {
|
|
1130
|
+
let changed = 0;
|
|
1131
|
+
for (const { local, rebound } of namespaces(column, userId)) {
|
|
1132
|
+
if (primaryKey) {
|
|
1133
|
+
const duplicates = compiler.deleteFrom(table).where(sql3.ref(column), "like", local).where(
|
|
1134
|
+
sql3`${rebound} in (${compiler.selectFrom(table).select(sql3.ref(column).as("k"))})`
|
|
1135
|
+
).compile();
|
|
1136
|
+
db.prepare(duplicates.sql).run(...duplicates.parameters);
|
|
1137
|
+
}
|
|
1138
|
+
const update = compiler.updateTable(table).set({ [column]: rebound }).where(sql3.ref(column), "like", local).compile();
|
|
1139
|
+
const result = db.prepare(update.sql).run(...update.parameters);
|
|
1140
|
+
changed += Number(result.changes ?? 0);
|
|
1141
|
+
}
|
|
1142
|
+
return changed;
|
|
1143
|
+
}
|
|
1144
|
+
var LibraryBoundElsewhereError = class extends Error {
|
|
1145
|
+
constructor(boundTo, cloudSays) {
|
|
1146
|
+
super(
|
|
1147
|
+
`This library belongs to user ${boundTo}, but the cloud says this device syncs for user ${cloudSays}. Sync stays off so the two libraries do not mix.`
|
|
1148
|
+
);
|
|
1149
|
+
this.boundTo = boundTo;
|
|
1150
|
+
this.cloudSays = cloudSays;
|
|
1151
|
+
this.name = "LibraryBoundElsewhereError";
|
|
1152
|
+
}
|
|
1153
|
+
boundTo;
|
|
1154
|
+
cloudSays;
|
|
1155
|
+
};
|
|
1156
|
+
function checkLibraryBindable(db, userId) {
|
|
1157
|
+
if (!USER_SEGMENT_PATTERN.test(userId) || userId === LOCAL_USER_SEGMENT) {
|
|
1158
|
+
throw new Error(`cannot bind the library to ${JSON.stringify(userId)}`);
|
|
1159
|
+
}
|
|
1160
|
+
const current = boundUserId(db);
|
|
1161
|
+
if (current !== null && current !== userId) throw new LibraryBoundElsewhereError(current, userId);
|
|
1162
|
+
}
|
|
1163
|
+
function commitLibraryBinding(db, userId, extraKeyColumns = []) {
|
|
1164
|
+
checkLibraryBindable(db, userId);
|
|
1165
|
+
const columns = [...coreKeyColumns(db), ...extraKeyColumns].filter(
|
|
1166
|
+
({ table }) => tableExists(db, table)
|
|
1167
|
+
);
|
|
1168
|
+
let rewritten = 0;
|
|
1169
|
+
db.exec("BEGIN");
|
|
1170
|
+
try {
|
|
1171
|
+
for (const column of columns) rewritten += rebindColumn(db, column, userId);
|
|
1172
|
+
db.prepare(
|
|
1173
|
+
compiler.insertInto(OWNER_TABLE).values({ id: sql3.lit(0), user_id: sql3.raw("?") }).onConflict(
|
|
1174
|
+
(oc) => oc.column("id").doUpdateSet((eb) => ({ user_id: eb.ref("excluded.user_id") }))
|
|
1175
|
+
).compile().sql
|
|
1176
|
+
).run(userId);
|
|
1177
|
+
db.exec("COMMIT");
|
|
1178
|
+
} catch (err) {
|
|
1179
|
+
db.exec("ROLLBACK");
|
|
1180
|
+
throw err;
|
|
736
1181
|
}
|
|
737
|
-
return
|
|
1182
|
+
return rewritten;
|
|
738
1183
|
}
|
|
739
1184
|
export {
|
|
1185
|
+
LibraryBoundElsewhereError,
|
|
740
1186
|
SqliteAppSyncableApplier,
|
|
741
1187
|
SqliteAppSyncableNamespaceStore,
|
|
742
1188
|
SqliteDatabaseAdapter,
|
|
743
1189
|
appSyncableTableName,
|
|
1190
|
+
applyConnectionPragmas,
|
|
1191
|
+
boundUserId,
|
|
1192
|
+
checkLibraryBindable,
|
|
1193
|
+
commitLibraryBinding,
|
|
744
1194
|
deleteAppSyncableNamespace,
|
|
745
1195
|
getAppSyncableNamespace,
|
|
746
1196
|
initializeLocalSchema,
|
|
1197
|
+
libraryKeyColumns,
|
|
1198
|
+
libraryUserSegment,
|
|
747
1199
|
listAppSyncableNamespaces,
|
|
748
1200
|
compiler as sqliteCompiler,
|
|
749
1201
|
upsertAppSyncableNamespace
|