@starkeep/storage-sqlite 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/dist/{adapter-EloqOiMg.d.cts → adapter-DoVOV6K-.d.cts} +28 -3
- package/dist/{adapter-EloqOiMg.d.ts → adapter-DoVOV6K-.d.ts} +28 -3
- package/dist/index.cjs +537 -208
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +164 -13
- package/dist/index.d.ts +164 -13
- package/dist/index.js +565 -187
- package/dist/index.js.map +1 -1
- package/dist/node-entry.cjs.map +1 -1
- package/dist/node-entry.d.cts +1 -1
- package/dist/node-entry.d.ts +1 -1
- package/dist/node-entry.js.map +1 -1
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,11 +1,35 @@
|
|
|
1
1
|
// src/adapter.ts
|
|
2
|
-
import {
|
|
2
|
+
import { nextCursorFrom } from "@starkeep/storage-adapter";
|
|
3
|
+
import {
|
|
4
|
+
serializeHLC as serializeHLC2,
|
|
5
|
+
deserializeHLC as deserializeHLC2,
|
|
6
|
+
getCategory,
|
|
7
|
+
isKnownType,
|
|
8
|
+
sqliteMetadataTableName as sqliteMetadataTableName2,
|
|
9
|
+
typeCategory,
|
|
10
|
+
METADATA_DISCRIMINANT_COLUMN,
|
|
11
|
+
METADATA_DELETED_AT_COLUMN
|
|
12
|
+
} from "@starkeep/protocol-primitives";
|
|
3
13
|
import {
|
|
4
14
|
StorageError,
|
|
5
15
|
TransactionError,
|
|
6
|
-
|
|
16
|
+
buildAppAggregateQuery,
|
|
17
|
+
buildAppRowQuery,
|
|
18
|
+
collectAggregatePage,
|
|
19
|
+
collectRowPage,
|
|
20
|
+
sharedQueryExcludesSoftDeleted,
|
|
21
|
+
sharedQuerySchema,
|
|
22
|
+
sharedQueryTableName,
|
|
23
|
+
SQLITE_APP_QUERY_DIALECT,
|
|
7
24
|
buildGetLabel,
|
|
8
25
|
buildLabelNodeWatermarks,
|
|
26
|
+
buildBucketDigest,
|
|
27
|
+
buildScanSinceForNode,
|
|
28
|
+
collectSince,
|
|
29
|
+
mergeDigestBuckets,
|
|
30
|
+
planNodeScans,
|
|
31
|
+
toDigestBuckets,
|
|
32
|
+
DEFAULT_BUCKET_PREFIX_LENGTH,
|
|
9
33
|
buildLabelRetraction,
|
|
10
34
|
buildLabelValueReplacementTombstone,
|
|
11
35
|
buildLabelSnapshotUpsert,
|
|
@@ -13,15 +37,25 @@ import {
|
|
|
13
37
|
buildLabelsByRecordIds,
|
|
14
38
|
buildQueryLabels,
|
|
15
39
|
buildTombstoneLabelsForRecord,
|
|
40
|
+
buildRestoreLabelsForRecord,
|
|
16
41
|
groupLabelsByRecordId,
|
|
17
|
-
|
|
42
|
+
emptyLabelPage,
|
|
43
|
+
LABEL_QUERY_TARGET,
|
|
44
|
+
labelPageFrom,
|
|
45
|
+
planFindByLabel,
|
|
18
46
|
paginateLabelScan,
|
|
19
47
|
rowToLabel as rowToLabel2
|
|
20
48
|
} from "@starkeep/storage-adapter";
|
|
21
49
|
import { sql as kSql } from "kysely";
|
|
22
50
|
|
|
23
51
|
// src/serialization.ts
|
|
24
|
-
import {
|
|
52
|
+
import {
|
|
53
|
+
serializeHLC,
|
|
54
|
+
deserializeHLC,
|
|
55
|
+
createStarkeepId,
|
|
56
|
+
isStandInRole,
|
|
57
|
+
standInSlot
|
|
58
|
+
} from "@starkeep/protocol-primitives";
|
|
25
59
|
import { rowToLabel, labelToRow } from "@starkeep/storage-adapter";
|
|
26
60
|
function recordToRow(record) {
|
|
27
61
|
return {
|
|
@@ -38,7 +72,12 @@ function recordToRow(record) {
|
|
|
38
72
|
size_bytes: record.sizeBytes,
|
|
39
73
|
original_filename: record.originalFilename,
|
|
40
74
|
origin_app_id: record.originAppId,
|
|
41
|
-
parent_id: record.parentId
|
|
75
|
+
parent_id: record.parentId,
|
|
76
|
+
stand_in_role: record.standInRole,
|
|
77
|
+
fidelity: record.fidelity,
|
|
78
|
+
canonical_threshold: record.canonicalThreshold,
|
|
79
|
+
self_canonical: record.selfCanonical ? 1 : 0,
|
|
80
|
+
stand_in_slot: standInSlot(record)
|
|
42
81
|
};
|
|
43
82
|
}
|
|
44
83
|
function rowToRecord(row) {
|
|
@@ -56,11 +95,23 @@ function rowToRecord(row) {
|
|
|
56
95
|
sizeBytes: row.size_bytes,
|
|
57
96
|
originalFilename: row.original_filename,
|
|
58
97
|
originAppId: row.origin_app_id,
|
|
59
|
-
parentId: row.parent_id ? createStarkeepId(row.parent_id) : null
|
|
98
|
+
parentId: row.parent_id ? createStarkeepId(row.parent_id) : null,
|
|
99
|
+
standInRole: isStandInRole(row.stand_in_role) ? row.stand_in_role : null,
|
|
100
|
+
// DSQL returns a bigint column as a string; the value is small, so a
|
|
101
|
+
// Number is exact.
|
|
102
|
+
fidelity: row.fidelity === null || row.fidelity === void 0 ? null : Number(row.fidelity),
|
|
103
|
+
canonicalThreshold: row.canonical_threshold === null || row.canonical_threshold === void 0 ? null : Number(row.canonical_threshold),
|
|
104
|
+
selfCanonical: row.self_canonical === 1
|
|
60
105
|
};
|
|
61
106
|
}
|
|
62
107
|
|
|
63
108
|
// src/query-builder.ts
|
|
109
|
+
import {
|
|
110
|
+
buildRecordCount,
|
|
111
|
+
buildRecordSelect,
|
|
112
|
+
buildRecordTypeCounts
|
|
113
|
+
} from "@starkeep/storage-adapter";
|
|
114
|
+
import { sqliteMetadataTableName } from "@starkeep/protocol-primitives";
|
|
64
115
|
import {
|
|
65
116
|
DummyDriver,
|
|
66
117
|
Kysely,
|
|
@@ -68,24 +119,6 @@ import {
|
|
|
68
119
|
SqliteIntrospector,
|
|
69
120
|
SqliteQueryCompiler
|
|
70
121
|
} from "kysely";
|
|
71
|
-
var FIELD_MAP = {
|
|
72
|
-
id: "id",
|
|
73
|
-
type: "type",
|
|
74
|
-
createdAt: "created_at",
|
|
75
|
-
updatedAt: "updated_at",
|
|
76
|
-
deletedAt: "deleted_at",
|
|
77
|
-
version: "version",
|
|
78
|
-
contentHash: "content_hash",
|
|
79
|
-
objectStorageKey: "object_storage_key",
|
|
80
|
-
mimeType: "mime_type",
|
|
81
|
-
sizeBytes: "size_bytes",
|
|
82
|
-
originAppId: "origin_app_id",
|
|
83
|
-
parentId: "parent_id",
|
|
84
|
-
originalFilename: "original_filename"
|
|
85
|
-
};
|
|
86
|
-
function mapField(field) {
|
|
87
|
-
return FIELD_MAP[field] ?? field;
|
|
88
|
-
}
|
|
89
122
|
var compiler = new Kysely({
|
|
90
123
|
dialect: {
|
|
91
124
|
createAdapter: () => new SqliteAdapter(),
|
|
@@ -94,78 +127,45 @@ var compiler = new Kysely({
|
|
|
94
127
|
createQueryCompiler: () => new SqliteQueryCompiler()
|
|
95
128
|
}
|
|
96
129
|
});
|
|
130
|
+
var DIALECT = {
|
|
131
|
+
records: "shared_records",
|
|
132
|
+
labels: "shared_record_labels",
|
|
133
|
+
metadataTable: sqliteMetadataTableName
|
|
134
|
+
};
|
|
97
135
|
function buildSelectQuery(query) {
|
|
98
|
-
|
|
99
|
-
if (query.type) {
|
|
100
|
-
qb = qb.where("type", "=", query.type);
|
|
101
|
-
}
|
|
102
|
-
if (query.filters) {
|
|
103
|
-
for (const filter of query.filters) {
|
|
104
|
-
qb = applyFilter(qb, filter);
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
if (query.excludeLabel) {
|
|
108
|
-
const { appId, key } = query.excludeLabel;
|
|
109
|
-
qb = qb.where(
|
|
110
|
-
(eb) => eb.not(
|
|
111
|
-
eb.exists(
|
|
112
|
-
eb.selectFrom("shared_record_labels").select("record_id").whereRef("shared_record_labels.record_id", "=", "shared_records.id").where("app_id", "=", appId).where("key", "=", key).where("deleted_at", "is", null)
|
|
113
|
-
)
|
|
114
|
-
)
|
|
115
|
-
);
|
|
116
|
-
}
|
|
117
|
-
if (query.cursor) {
|
|
118
|
-
qb = qb.where("id", ">", query.cursor);
|
|
119
|
-
}
|
|
120
|
-
if (query.sort && query.sort.length > 0) {
|
|
121
|
-
for (const s of query.sort) {
|
|
122
|
-
qb = qb.orderBy(mapField(s.field), s.direction === "desc" ? "desc" : "asc");
|
|
123
|
-
}
|
|
124
|
-
} else {
|
|
125
|
-
qb = qb.orderBy("id", "asc");
|
|
126
|
-
}
|
|
127
|
-
if (query.limit) {
|
|
128
|
-
qb = qb.limit(query.limit + 1);
|
|
129
|
-
}
|
|
130
|
-
const compiled = qb.compile();
|
|
136
|
+
const compiled = buildRecordSelect(compiler, DIALECT, query);
|
|
131
137
|
return { sql: compiled.sql, params: [...compiled.parameters] };
|
|
132
138
|
}
|
|
133
|
-
function
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
case "gt":
|
|
141
|
-
return qb.where(col, ">", filter.value);
|
|
142
|
-
case "gte":
|
|
143
|
-
return qb.where(col, ">=", filter.value);
|
|
144
|
-
case "lt":
|
|
145
|
-
return qb.where(col, "<", filter.value);
|
|
146
|
-
case "lte":
|
|
147
|
-
return qb.where(col, "<=", filter.value);
|
|
148
|
-
case "in":
|
|
149
|
-
return qb.where(col, "in", filter.value);
|
|
150
|
-
case "like":
|
|
151
|
-
return qb.where(col, "like", `%${filter.value}%`);
|
|
152
|
-
case "isNull":
|
|
153
|
-
return qb.where((eb) => eb(col, "is", null));
|
|
154
|
-
case "isNotNull":
|
|
155
|
-
return qb.where((eb) => eb(col, "is not", null));
|
|
156
|
-
default:
|
|
157
|
-
return qb;
|
|
158
|
-
}
|
|
139
|
+
function buildCountQuery(query) {
|
|
140
|
+
const compiled = buildRecordCount(compiler, DIALECT, query);
|
|
141
|
+
return { sql: compiled.sql, params: [...compiled.parameters] };
|
|
142
|
+
}
|
|
143
|
+
function buildTypeCountsQuery(query) {
|
|
144
|
+
const compiled = buildRecordTypeCounts(compiler, DIALECT, query);
|
|
145
|
+
return { sql: compiled.sql, params: [...compiled.parameters] };
|
|
159
146
|
}
|
|
160
147
|
|
|
161
148
|
// src/schema/bootstrap.ts
|
|
162
149
|
import { sql } from "kysely";
|
|
163
|
-
import {
|
|
150
|
+
import {
|
|
151
|
+
CATEGORIES,
|
|
152
|
+
sqliteMetadataDdl,
|
|
153
|
+
metadataIndexDdls,
|
|
154
|
+
hasMetadataTable
|
|
155
|
+
} from "@starkeep/protocol-primitives";
|
|
156
|
+
function applyConnectionPragmas(db) {
|
|
157
|
+
db.exec("PRAGMA case_sensitive_like = ON");
|
|
158
|
+
}
|
|
164
159
|
function applyLocalSchemaDdl(db) {
|
|
165
160
|
db.exec("PRAGMA journal_mode = WAL");
|
|
166
161
|
db.exec("PRAGMA foreign_keys = ON");
|
|
162
|
+
applyConnectionPragmas(db);
|
|
167
163
|
db.exec(
|
|
168
|
-
compiler.schema.createTable("shared_records").ifNotExists().addColumn("id", "text", (c) => c.primaryKey()).addColumn("type", "text", (c) => c.notNull()).addColumn("created_at", "text", (c) => c.notNull()).addColumn("updated_at", "text", (c) => c.notNull()).addColumn("node_id", "text", (c) => c.notNull()).addColumn("deleted_at", "text").addColumn("version", "integer", (c) => c.notNull().defaultTo(1)).addColumn("content_hash", "text", (c) => c.notNull()).addColumn("object_storage_key", "text", (c) => c.notNull()).addColumn("mime_type", "text").addColumn("size_bytes", "integer", (c) => c.notNull()).addColumn("original_filename", "text").addColumn("origin_app_id", "text", (c) => c.notNull()).addColumn("parent_id", "text").
|
|
164
|
+
compiler.schema.createTable("shared_records").ifNotExists().addColumn("id", "text", (c) => c.primaryKey()).addColumn("type", "text", (c) => c.notNull()).addColumn("created_at", "text", (c) => c.notNull()).addColumn("updated_at", "text", (c) => c.notNull()).addColumn("node_id", "text", (c) => c.notNull()).addColumn("deleted_at", "text").addColumn("version", "integer", (c) => c.notNull().defaultTo(1)).addColumn("content_hash", "text", (c) => c.notNull()).addColumn("object_storage_key", "text", (c) => c.notNull()).addColumn("mime_type", "text").addColumn("size_bytes", "integer", (c) => c.notNull()).addColumn("original_filename", "text").addColumn("origin_app_id", "text", (c) => c.notNull()).addColumn("parent_id", "text").addColumn(
|
|
165
|
+
"stand_in_role",
|
|
166
|
+
"text",
|
|
167
|
+
(c) => c.check(sql`stand_in_role IN ('canonical', 'smaller')`)
|
|
168
|
+
).addColumn("fidelity", "integer").addColumn("canonical_threshold", "integer").addColumn("self_canonical", "integer").addColumn("stand_in_slot", "text").compile().sql
|
|
169
169
|
);
|
|
170
170
|
const sharedRecordsIndexes = [
|
|
171
171
|
compiler.schema.createIndex("idx_shared_records_type").ifNotExists().on("shared_records").column("type"),
|
|
@@ -180,16 +180,17 @@ function applyLocalSchemaDdl(db) {
|
|
|
180
180
|
// table once per write. Leading with content_hash is what makes the dedup
|
|
181
181
|
// check affordable enough to be unconditional, and unconditional is what
|
|
182
182
|
// makes "one object key, one record" an invariant rather than a hope.
|
|
183
|
-
compiler.schema.createIndex("idx_shared_records_content_hash").ifNotExists().on("shared_records").columns(["content_hash", "parent_id", "deleted_at"])
|
|
184
|
-
// Duplicate-file prevention: (filename + bytes) is unique among live
|
|
185
|
-
// records. Tombstoned rows (deleted_at IS NOT NULL) are excluded so a
|
|
186
|
-
// re-upload after delete is allowed. Records with NULL filename are not
|
|
187
|
-
// constrained — the rule requires both filename and content to match.
|
|
188
|
-
compiler.schema.createIndex("uq_shared_records_filename_hash").ifNotExists().unique().on("shared_records").columns(["original_filename", "content_hash"]).where(sql.ref("deleted_at"), "is", null).where(sql.ref("original_filename"), "is not", null)
|
|
183
|
+
compiler.schema.createIndex("idx_shared_records_content_hash").ifNotExists().on("shared_records").columns(["content_hash", "parent_id", "deleted_at"])
|
|
189
184
|
];
|
|
190
185
|
for (const index of sharedRecordsIndexes) {
|
|
191
186
|
db.exec(index.compile().sql);
|
|
192
187
|
}
|
|
188
|
+
db.exec(
|
|
189
|
+
compiler.schema.createIndex("uq_shared_records_stand_in_slot").ifNotExists().unique().on("shared_records").columns(["parent_id", "stand_in_slot"]).compile().sql
|
|
190
|
+
);
|
|
191
|
+
db.exec(
|
|
192
|
+
compiler.schema.createIndex("uq_shared_records_parent_filename_hash").ifNotExists().unique().on("shared_records").expression(sql`COALESCE(parent_id, ''), COALESCE(original_filename, ''), content_hash`).where(sql.ref("deleted_at"), "is", null).compile().sql
|
|
193
|
+
);
|
|
193
194
|
db.exec(
|
|
194
195
|
compiler.schema.createTable("shared_access_grants").ifNotExists().addColumn("app_id", "text", (c) => c.notNull()).addColumn("type_id", "text", (c) => c.notNull()).addColumn("access", "text", (c) => c.notNull().check(sql`access IN ('read', 'readwrite')`)).addColumn("metadata_write", "integer", (c) => c.notNull().defaultTo(0)).addColumn("created_at", "text", (c) => c.notNull().defaultTo(sql`(datetime('now'))`)).addPrimaryKeyConstraint("pk_shared_access_grants", ["app_id", "type_id"]).compile().sql
|
|
195
196
|
);
|
|
@@ -245,8 +246,9 @@ function applyLocalSchemaDdl(db) {
|
|
|
245
246
|
compiler.schema.createTable("app_syncable_namespaces").ifNotExists().addColumn("app_id", "text", (c) => c.primaryKey()).addColumn("tables_json", "text", (c) => c.notNull()).addColumn("files_enabled", "integer", (c) => c.notNull().defaultTo(0)).addColumn("created_at", "text", (c) => c.notNull().defaultTo(sql`(datetime('now'))`)).compile().sql
|
|
246
247
|
);
|
|
247
248
|
for (const c of CATEGORIES) {
|
|
248
|
-
if (c.id
|
|
249
|
+
if (!hasMetadataTable(c.id)) continue;
|
|
249
250
|
db.exec(sqliteMetadataDdl(c));
|
|
251
|
+
for (const index of metadataIndexDdls(c, "sqlite")) db.exec(index.sql);
|
|
250
252
|
}
|
|
251
253
|
}
|
|
252
254
|
function initializeLocalSchema(db) {
|
|
@@ -260,6 +262,28 @@ function initializeLocalSchema(db) {
|
|
|
260
262
|
}
|
|
261
263
|
}
|
|
262
264
|
|
|
265
|
+
// src/row-values.ts
|
|
266
|
+
function booleanColumnNames(columns) {
|
|
267
|
+
if (!columns) return null;
|
|
268
|
+
const names = columns.filter((c) => c.type === "boolean").map((c) => c.name);
|
|
269
|
+
return names.length > 0 ? new Set(names) : null;
|
|
270
|
+
}
|
|
271
|
+
function fromSqliteRows(rows, booleanColumns) {
|
|
272
|
+
if (!booleanColumns || booleanColumns.size === 0) return rows;
|
|
273
|
+
return rows.map((row) => {
|
|
274
|
+
const out = { ...row };
|
|
275
|
+
for (const name of booleanColumns) {
|
|
276
|
+
const value = out[name];
|
|
277
|
+
if (typeof value === "number") out[name] = value !== 0;
|
|
278
|
+
}
|
|
279
|
+
return out;
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
function toSqliteParam(value) {
|
|
283
|
+
if (typeof value === "boolean") return value ? 1 : 0;
|
|
284
|
+
return value;
|
|
285
|
+
}
|
|
286
|
+
|
|
263
287
|
// src/adapter.ts
|
|
264
288
|
var AVAILABILITY = "shared_object_availability";
|
|
265
289
|
function rowToAvailability(row) {
|
|
@@ -276,6 +300,7 @@ function rowToAvailability(row) {
|
|
|
276
300
|
var LABELS = {
|
|
277
301
|
table: "shared_record_labels"
|
|
278
302
|
};
|
|
303
|
+
var RECORDS = "shared_records";
|
|
279
304
|
var SqliteDatabaseAdapter = class {
|
|
280
305
|
database = null;
|
|
281
306
|
options;
|
|
@@ -324,14 +349,22 @@ var SqliteDatabaseAdapter = class {
|
|
|
324
349
|
// borrow node:sqlite's parameter types. `RawStatement` declares `unknown[]`
|
|
325
350
|
// directly, so they were already no-ops — and they were the last thing in
|
|
326
351
|
// this file referring to a concrete driver.
|
|
327
|
-
|
|
328
|
-
|
|
352
|
+
//
|
|
353
|
+
// All three map their parameters through `toSqliteParam`, which is what makes
|
|
354
|
+
// `row-values.ts`'s claim true — every bind on this connection, reads and
|
|
355
|
+
// writes alike. It was true only of the query path until 2026-09-10, so a
|
|
356
|
+
// declared `boolean` written through `putMetadata` reached the driver as a
|
|
357
|
+
// JavaScript boolean and threw "Provided value cannot be bound to SQLite
|
|
358
|
+
// parameter". Nothing had exercised it because no metadata column was a
|
|
359
|
+
// boolean until `exif_present`.
|
|
360
|
+
runStmt(sql4, ...params) {
|
|
361
|
+
this.getDatabase().prepare(sql4).run(...params.map(toSqliteParam));
|
|
329
362
|
}
|
|
330
|
-
getRow(
|
|
331
|
-
return this.getDatabase().prepare(
|
|
363
|
+
getRow(sql4, ...params) {
|
|
364
|
+
return this.getDatabase().prepare(sql4).get(...params.map(toSqliteParam));
|
|
332
365
|
}
|
|
333
|
-
allRows(
|
|
334
|
-
return this.getDatabase().prepare(
|
|
366
|
+
allRows(sql4, ...params) {
|
|
367
|
+
return this.getDatabase().prepare(sql4).all(...params.map(toSqliteParam));
|
|
335
368
|
}
|
|
336
369
|
async put(record) {
|
|
337
370
|
const row = recordToRow(record);
|
|
@@ -352,7 +385,7 @@ var SqliteDatabaseAdapter = class {
|
|
|
352
385
|
}
|
|
353
386
|
async delete(id, hlc) {
|
|
354
387
|
const ts = serializeHLC2(hlc);
|
|
355
|
-
const query = compiler.updateTable("shared_records").set({ deleted_at: ts, updated_at: ts, node_id: hlc.nodeId }).where("id", "=", id).compile();
|
|
388
|
+
const query = compiler.updateTable("shared_records").set({ deleted_at: ts, updated_at: ts, node_id: hlc.nodeId, stand_in_slot: null }).where("id", "=", id).compile();
|
|
356
389
|
this.runStmt(query.sql, ...query.parameters);
|
|
357
390
|
}
|
|
358
391
|
async getNodeWatermarks() {
|
|
@@ -364,18 +397,100 @@ var SqliteDatabaseAdapter = class {
|
|
|
364
397
|
}
|
|
365
398
|
return out;
|
|
366
399
|
}
|
|
400
|
+
async querySince(peerWatermarks, limit) {
|
|
401
|
+
const scans = planNodeScans(await this.getNodeWatermarks(), peerWatermarks);
|
|
402
|
+
const { rows, hasMore, truncated } = await collectSince(
|
|
403
|
+
scans,
|
|
404
|
+
limit,
|
|
405
|
+
async (scan, remaining) => {
|
|
406
|
+
const compiled = buildScanSinceForNode(compiler, RECORDS, scan, remaining);
|
|
407
|
+
return this.allRows(compiled.sql, ...compiled.parameters);
|
|
408
|
+
},
|
|
409
|
+
(row) => deserializeHLC2(row["updated_at"])
|
|
410
|
+
);
|
|
411
|
+
return { rows: rows.map(rowToRecord), hasMore, truncated };
|
|
412
|
+
}
|
|
413
|
+
async queryLabelsSince(peerWatermarks, limit) {
|
|
414
|
+
const scans = planNodeScans(await this.getLabelNodeWatermarks(), peerWatermarks);
|
|
415
|
+
const { rows, hasMore, truncated } = await collectSince(
|
|
416
|
+
scans,
|
|
417
|
+
limit,
|
|
418
|
+
async (scan, remaining) => {
|
|
419
|
+
const compiled = buildScanSinceForNode(compiler, LABELS.table, scan, remaining);
|
|
420
|
+
return this.allRows(compiled.sql, ...compiled.parameters);
|
|
421
|
+
},
|
|
422
|
+
(row) => deserializeHLC2(row["updated_at"])
|
|
423
|
+
);
|
|
424
|
+
return { rows: rows.map(rowToLabel2), hasMore, truncated };
|
|
425
|
+
}
|
|
426
|
+
async bucketDigest(prefixLength = DEFAULT_BUCKET_PREFIX_LENGTH) {
|
|
427
|
+
const out = [];
|
|
428
|
+
for (const table of [RECORDS, LABELS.table]) {
|
|
429
|
+
const compiled = buildBucketDigest(compiler, table, prefixLength);
|
|
430
|
+
out.push(
|
|
431
|
+
...toDigestBuckets(
|
|
432
|
+
this.allRows(compiled.sql, ...compiled.parameters)
|
|
433
|
+
)
|
|
434
|
+
);
|
|
435
|
+
}
|
|
436
|
+
return mergeDigestBuckets(out);
|
|
437
|
+
}
|
|
367
438
|
async query(query) {
|
|
368
|
-
const { sql:
|
|
369
|
-
const rows = this.allRows(
|
|
439
|
+
const { sql: sql4, params } = buildSelectQuery(query);
|
|
440
|
+
const rows = this.allRows(sql4, ...params);
|
|
370
441
|
const limit = query.limit;
|
|
371
442
|
const hasMore = limit ? rows.length > limit : false;
|
|
372
443
|
const resultRows = hasMore ? rows.slice(0, limit) : rows;
|
|
373
444
|
return {
|
|
374
445
|
records: resultRows.map(rowToRecord),
|
|
375
|
-
|
|
446
|
+
// Cut from the row itself, ordering values and all — see `nextCursorFrom`.
|
|
447
|
+
// The page carries those values back under reserved aliases precisely so
|
|
448
|
+
// this does not have to recompute what the database ordered on.
|
|
449
|
+
nextCursor: hasMore ? nextCursorFrom(query, resultRows[resultRows.length - 1]) : null,
|
|
376
450
|
hasMore
|
|
377
451
|
};
|
|
378
452
|
}
|
|
453
|
+
async countRecords(query) {
|
|
454
|
+
const { sql: sql4, params } = buildCountQuery(query);
|
|
455
|
+
const row = this.getRow(sql4, ...params);
|
|
456
|
+
return row?.total ?? 0;
|
|
457
|
+
}
|
|
458
|
+
async countRecordsByType(query) {
|
|
459
|
+
const { sql: sql4, params } = buildTypeCountsQuery(query);
|
|
460
|
+
const rows = this.allRows(sql4, ...params);
|
|
461
|
+
return rows.map((row) => ({
|
|
462
|
+
type: row.type,
|
|
463
|
+
count: Number(row.count),
|
|
464
|
+
latestUpdatedAt: row.latest_updated_at
|
|
465
|
+
}));
|
|
466
|
+
}
|
|
467
|
+
/**
|
|
468
|
+
* See `DatabaseAdapter.queryShared`. Compiled by the same builder the
|
|
469
|
+
* app-syncable plane uses, against the table the target names on this engine.
|
|
470
|
+
*/
|
|
471
|
+
async queryShared(target, query, options = {}) {
|
|
472
|
+
const schema = sharedQuerySchema(target);
|
|
473
|
+
const table = sharedQueryTableName(target, "sqlite");
|
|
474
|
+
const build = {
|
|
475
|
+
serverWhere: options.serverWhere,
|
|
476
|
+
excludeSoftDeleted: sharedQueryExcludesSoftDeleted(target),
|
|
477
|
+
...options.softDeleted ? { softDeleted: options.softDeleted } : {}
|
|
478
|
+
};
|
|
479
|
+
const booleans = booleanColumnNames(schema.columns);
|
|
480
|
+
if (query.mode === "aggregate") {
|
|
481
|
+
const compiled2 = buildAppAggregateQuery(compiler, table, query, SQLITE_APP_QUERY_DIALECT, build);
|
|
482
|
+
return collectAggregatePage(query, fromSqliteRows(this.selectCompiled(compiled2), booleans));
|
|
483
|
+
}
|
|
484
|
+
const compiled = buildAppRowQuery(compiler, table, query, build);
|
|
485
|
+
return collectRowPage(query, fromSqliteRows(this.selectCompiled(compiled), booleans));
|
|
486
|
+
}
|
|
487
|
+
/** Bind and run a compiled SELECT, adapting values SQLite cannot bind. */
|
|
488
|
+
selectCompiled(compiled) {
|
|
489
|
+
return this.allRows(
|
|
490
|
+
compiled.sql,
|
|
491
|
+
...compiled.parameters.map(toSqliteParam)
|
|
492
|
+
);
|
|
493
|
+
}
|
|
379
494
|
async batch(operations) {
|
|
380
495
|
this.getDatabase().exec("BEGIN");
|
|
381
496
|
try {
|
|
@@ -410,13 +525,9 @@ var SqliteDatabaseAdapter = class {
|
|
|
410
525
|
throw new TransactionError("Transaction failed", error);
|
|
411
526
|
}
|
|
412
527
|
}
|
|
413
|
-
async putMetadata(
|
|
414
|
-
const table =
|
|
415
|
-
const values =
|
|
416
|
-
for (const [key, value] of Object.entries(row)) {
|
|
417
|
-
if (key === "recordId") continue;
|
|
418
|
-
values[key] = value;
|
|
419
|
-
}
|
|
528
|
+
async putMetadata(recordType, row) {
|
|
529
|
+
const table = sqliteMetadataTableName2(recordType);
|
|
530
|
+
const values = metadataValues(recordType, row);
|
|
420
531
|
const updateColumns = Object.keys(values).filter((c) => c !== "record_id");
|
|
421
532
|
const query = compiler.insertInto(table).values(values).onConflict(
|
|
422
533
|
(oc) => updateColumns.length > 0 ? oc.column("record_id").doUpdateSet(
|
|
@@ -426,29 +537,40 @@ var SqliteDatabaseAdapter = class {
|
|
|
426
537
|
this.runStmt(query.sql, ...query.parameters);
|
|
427
538
|
}
|
|
428
539
|
async getMetadata(typeId, recordId) {
|
|
429
|
-
const table =
|
|
540
|
+
const table = sqliteMetadataTableName2(typeId);
|
|
430
541
|
const query = compiler.selectFrom(table).selectAll().where("record_id", "=", recordId).compile();
|
|
431
542
|
const row = this.getRow(query.sql, ...query.parameters);
|
|
432
543
|
if (!row) return null;
|
|
433
|
-
return columnsToMetadataRow(recordId, row);
|
|
544
|
+
return columnsToMetadataRow(recordId, typeId, row);
|
|
434
545
|
}
|
|
435
546
|
async getMetadataByIds(typeId, recordIds) {
|
|
436
547
|
const result = /* @__PURE__ */ new Map();
|
|
437
548
|
if (recordIds.length === 0) return result;
|
|
438
|
-
const table =
|
|
549
|
+
const table = sqliteMetadataTableName2(typeId);
|
|
439
550
|
const query = compiler.selectFrom(table).selectAll().where("record_id", "in", recordIds).compile();
|
|
440
551
|
const rows = this.allRows(query.sql, ...query.parameters);
|
|
441
552
|
for (const row of rows) {
|
|
442
553
|
const recordId = row["record_id"];
|
|
443
|
-
result.set(recordId, columnsToMetadataRow(recordId, row));
|
|
554
|
+
result.set(recordId, columnsToMetadataRow(recordId, typeId, row));
|
|
444
555
|
}
|
|
445
556
|
return result;
|
|
446
557
|
}
|
|
447
558
|
async deleteMetadata(typeId, recordId) {
|
|
448
|
-
const table =
|
|
559
|
+
const table = sqliteMetadataTableName2(typeId);
|
|
449
560
|
const query = compiler.deleteFrom(table).where("record_id", "=", recordId).compile();
|
|
450
561
|
this.runStmt(query.sql, ...query.parameters);
|
|
451
562
|
}
|
|
563
|
+
async tombstoneMetadata(typeId, recordId, hlc) {
|
|
564
|
+
this.setMetadataTombstone(typeId, recordId, serializeHLC2(hlc));
|
|
565
|
+
}
|
|
566
|
+
async restoreMetadata(typeId, recordId) {
|
|
567
|
+
this.setMetadataTombstone(typeId, recordId, null);
|
|
568
|
+
}
|
|
569
|
+
setMetadataTombstone(typeId, recordId, value) {
|
|
570
|
+
const table = sqliteMetadataTableName2(typeId);
|
|
571
|
+
const query = compiler.updateTable(table).set({ [METADATA_DELETED_AT_COLUMN]: value }).where("record_id", "=", recordId).compile();
|
|
572
|
+
this.runStmt(query.sql, ...query.parameters);
|
|
573
|
+
}
|
|
452
574
|
// ---- Cross-app record labels -------------------------------------------
|
|
453
575
|
//
|
|
454
576
|
// The SQL for all of these is built in `@starkeep/storage-adapter`, shared
|
|
@@ -498,15 +620,22 @@ var SqliteDatabaseAdapter = class {
|
|
|
498
620
|
async getLabelsByRecordIds(recordIds) {
|
|
499
621
|
if (recordIds.length === 0) return /* @__PURE__ */ new Map();
|
|
500
622
|
const query = buildLabelsByRecordIds(compiler, LABELS, recordIds);
|
|
501
|
-
return groupLabelsByRecordId(
|
|
502
|
-
this.allRows(query.sql, ...query.parameters)
|
|
503
|
-
);
|
|
623
|
+
return groupLabelsByRecordId(this.allRows(query.sql, ...query.parameters));
|
|
504
624
|
}
|
|
625
|
+
/**
|
|
626
|
+
* The reverse label read, run through the query grammar — see
|
|
627
|
+
* `label-find.ts`. The grant rides in as `serverWhere` rather than as one of
|
|
628
|
+
* the caller's own predicates, which is the separation every other shared
|
|
629
|
+
* query draws.
|
|
630
|
+
*/
|
|
505
631
|
async findByLabel(query) {
|
|
506
|
-
const
|
|
507
|
-
if (!
|
|
508
|
-
|
|
509
|
-
|
|
632
|
+
const plan = planFindByLabel(query);
|
|
633
|
+
if (!plan) return emptyLabelPage();
|
|
634
|
+
return labelPageFrom(
|
|
635
|
+
await this.queryShared(LABEL_QUERY_TARGET, plan.query, {
|
|
636
|
+
serverWhere: plan.serverWhere
|
|
637
|
+
})
|
|
638
|
+
);
|
|
510
639
|
}
|
|
511
640
|
// ---- Label sync ---------------------------------------------------------
|
|
512
641
|
async putLabel(label) {
|
|
@@ -561,7 +690,11 @@ var SqliteDatabaseAdapter = class {
|
|
|
561
690
|
this.runStmt(compiled.sql, ...compiled.parameters);
|
|
562
691
|
}
|
|
563
692
|
async countRestoringObjects() {
|
|
564
|
-
const compiled = compiler.selectFrom(AVAILABILITY).innerJoin(
|
|
693
|
+
const compiled = compiler.selectFrom(AVAILABILITY).innerJoin(
|
|
694
|
+
"shared_records",
|
|
695
|
+
"shared_records.object_storage_key",
|
|
696
|
+
`${AVAILABILITY}.object_storage_key`
|
|
697
|
+
).select(({ fn }) => [
|
|
565
698
|
fn.count(`${AVAILABILITY}.object_storage_key`).as("object_count"),
|
|
566
699
|
fn.sum("shared_records.size_bytes").as("bytes")
|
|
567
700
|
]).where(`${AVAILABILITY}.state`, "=", kSql.lit("restoring")).compile();
|
|
@@ -575,17 +708,47 @@ var SqliteDatabaseAdapter = class {
|
|
|
575
708
|
const query = buildTombstoneLabelsForRecord(compiler, LABELS, recordId, hlc);
|
|
576
709
|
this.runStmt(query.sql, ...query.parameters);
|
|
577
710
|
}
|
|
711
|
+
async restoreLabelsForRecord(recordId, deletedAt, hlc) {
|
|
712
|
+
const query = buildRestoreLabelsForRecord(compiler, LABELS, recordId, deletedAt, hlc);
|
|
713
|
+
this.runStmt(query.sql, ...query.parameters);
|
|
714
|
+
}
|
|
578
715
|
};
|
|
579
|
-
function columnsToMetadataRow(recordId, columns) {
|
|
716
|
+
function columnsToMetadataRow(recordId, typeId, columns) {
|
|
717
|
+
const category = typeCategory(typeId) ?? typeId;
|
|
718
|
+
const booleans = booleanColumnNames(getCategory(category)?.metadataColumns);
|
|
580
719
|
const row = { recordId };
|
|
581
720
|
for (const [key, value] of Object.entries(columns)) {
|
|
582
|
-
if (key === "record_id") continue;
|
|
583
|
-
row[key] = value;
|
|
721
|
+
if (key === "record_id" || key === METADATA_DELETED_AT_COLUMN) continue;
|
|
722
|
+
row[key] = booleans?.has(key) && typeof value === "number" ? value !== 0 : value;
|
|
584
723
|
}
|
|
585
724
|
return row;
|
|
586
725
|
}
|
|
726
|
+
function metadataValues(recordType, row) {
|
|
727
|
+
if (!isKnownType(recordType)) {
|
|
728
|
+
throw new StorageError(
|
|
729
|
+
`putMetadata needs the record's own type, not "${recordType}": the ${METADATA_DISCRIMINANT_COLUMN} column gates every read of this row`
|
|
730
|
+
);
|
|
731
|
+
}
|
|
732
|
+
const values = {
|
|
733
|
+
record_id: row.recordId,
|
|
734
|
+
[METADATA_DISCRIMINANT_COLUMN]: recordType,
|
|
735
|
+
// Writing metadata asserts the row is live. Without this an app re-reporting
|
|
736
|
+
// a restored record's dimensions would write into a row still carrying its
|
|
737
|
+
// tombstone, and the one read that filters on the stamp would keep answering
|
|
738
|
+
// nothing. Server-owned either way: a wire row carrying it is ignored below.
|
|
739
|
+
[METADATA_DELETED_AT_COLUMN]: null
|
|
740
|
+
};
|
|
741
|
+
for (const [key, value] of Object.entries(row)) {
|
|
742
|
+
if (key === "recordId" || key === METADATA_DISCRIMINANT_COLUMN || key === METADATA_DELETED_AT_COLUMN) {
|
|
743
|
+
continue;
|
|
744
|
+
}
|
|
745
|
+
values[key] = value;
|
|
746
|
+
}
|
|
747
|
+
return values;
|
|
748
|
+
}
|
|
587
749
|
|
|
588
750
|
// src/app-syncable/namespace.ts
|
|
751
|
+
import { parseAppSyncableTables } from "@starkeep/shared-space-api";
|
|
589
752
|
function normalizeAppId(appId) {
|
|
590
753
|
return appId.toLowerCase().replace(/-/g, "_");
|
|
591
754
|
}
|
|
@@ -593,7 +756,7 @@ function appSyncableTableName(appId, tableName) {
|
|
|
593
756
|
return `${normalizeAppId(appId)}_syncable_${tableName}`;
|
|
594
757
|
}
|
|
595
758
|
function rowToNamespace(r) {
|
|
596
|
-
const tables =
|
|
759
|
+
const tables = parseAppSyncableTables(r.app_id, r.tables_json);
|
|
597
760
|
return {
|
|
598
761
|
appId: r.app_id,
|
|
599
762
|
tables,
|
|
@@ -645,11 +808,26 @@ var SqliteAppSyncableNamespaceStore = class {
|
|
|
645
808
|
// src/app-syncable/apply.ts
|
|
646
809
|
import { sql as sql2 } from "kysely";
|
|
647
810
|
import { serializeHLC as serializeHLC3, deserializeHLC as deserializeHLC3 } from "@starkeep/protocol-primitives";
|
|
811
|
+
import {
|
|
812
|
+
buildAppAggregateQuery as buildAppAggregateQuery2,
|
|
813
|
+
buildAppRowQuery as buildAppRowQuery2,
|
|
814
|
+
collectAggregatePage as collectAggregatePage2,
|
|
815
|
+
collectRowPage as collectRowPage2,
|
|
816
|
+
SQLITE_APP_QUERY_DIALECT as SQLITE_APP_QUERY_DIALECT2,
|
|
817
|
+
buildBucketDigest as buildBucketDigest2,
|
|
818
|
+
buildScanSinceForNode as buildScanSinceForNode2,
|
|
819
|
+
collectSince as collectSince2,
|
|
820
|
+
planNodeScans as planNodeScans2,
|
|
821
|
+
requireKeyedWhere,
|
|
822
|
+
rowToWireEntry,
|
|
823
|
+
toDigestBuckets as toDigestBuckets2,
|
|
824
|
+
DEFAULT_BUCKET_PREFIX_LENGTH as DEFAULT_BUCKET_PREFIX_LENGTH2
|
|
825
|
+
} from "@starkeep/storage-adapter";
|
|
648
826
|
function runCompiled(db, compiled) {
|
|
649
|
-
db.prepare(compiled.sql).run(...compiled.parameters);
|
|
827
|
+
db.prepare(compiled.sql).run(...compiled.parameters.map(toSqliteParam));
|
|
650
828
|
}
|
|
651
829
|
function allCompiled(db, compiled) {
|
|
652
|
-
return db.prepare(compiled.sql).all(...compiled.parameters);
|
|
830
|
+
return db.prepare(compiled.sql).all(...compiled.parameters.map(toSqliteParam));
|
|
653
831
|
}
|
|
654
832
|
var SqliteAppSyncableApplier = class {
|
|
655
833
|
constructor(db, namespace) {
|
|
@@ -661,9 +839,7 @@ var SqliteAppSyncableApplier = class {
|
|
|
661
839
|
apply(entry) {
|
|
662
840
|
const ns = this.namespace.get(entry.appId);
|
|
663
841
|
if (!ns) {
|
|
664
|
-
throw new Error(
|
|
665
|
-
`SqliteAppSyncableApplier: app "${entry.appId}" not installed`
|
|
666
|
-
);
|
|
842
|
+
throw new Error(`SqliteAppSyncableApplier: app "${entry.appId}" not installed`);
|
|
667
843
|
}
|
|
668
844
|
const tableInfo = ns.tables.find((t) => t.name === entry.table);
|
|
669
845
|
if (!tableInfo) {
|
|
@@ -676,9 +852,9 @@ var SqliteAppSyncableApplier = class {
|
|
|
676
852
|
if (entry.op === "insert") {
|
|
677
853
|
this.applyInsert(fullName, pkColumns, entry);
|
|
678
854
|
} else if (entry.op === "update") {
|
|
679
|
-
this.applyUpdate(fullName, entry);
|
|
855
|
+
this.applyUpdate(fullName, pkColumns, entry);
|
|
680
856
|
} else {
|
|
681
|
-
this.applyDelete(fullName, entry);
|
|
857
|
+
this.applyDelete(fullName, pkColumns, entry);
|
|
682
858
|
}
|
|
683
859
|
}
|
|
684
860
|
applyInsert(fullName, pkColumns, entry) {
|
|
@@ -702,10 +878,10 @@ var SqliteAppSyncableApplier = class {
|
|
|
702
878
|
).compile()
|
|
703
879
|
);
|
|
704
880
|
}
|
|
705
|
-
applyUpdate(fullName, entry) {
|
|
881
|
+
applyUpdate(fullName, pkColumns, entry) {
|
|
706
882
|
const rawPatch = entry.row ?? {};
|
|
707
883
|
const patch = rawPatch["updated_at"] ? withNodeId(rawPatch, entry) : rawPatch;
|
|
708
|
-
const where = entry
|
|
884
|
+
const where = requireKeyedWhere(entry, "update", pkColumns);
|
|
709
885
|
const patchCols = Object.keys(patch);
|
|
710
886
|
const whereCols = Object.keys(where);
|
|
711
887
|
if (patchCols.length === 0) return;
|
|
@@ -715,36 +891,90 @@ var SqliteAppSyncableApplier = class {
|
|
|
715
891
|
query = query.where(c, "=", where[c]);
|
|
716
892
|
}
|
|
717
893
|
if (incomingUpdatedAt) {
|
|
718
|
-
query = query.where(
|
|
894
|
+
query = query.where(
|
|
895
|
+
(eb) => eb.or([eb("updated_at", "is", null), eb("updated_at", "<", incomingUpdatedAt)])
|
|
896
|
+
);
|
|
719
897
|
}
|
|
720
898
|
runCompiled(this.db, query.compile());
|
|
721
899
|
}
|
|
722
|
-
applyDelete(fullName, entry) {
|
|
723
|
-
const where = entry
|
|
900
|
+
applyDelete(fullName, pkColumns, entry) {
|
|
901
|
+
const where = requireKeyedWhere(entry, "delete", pkColumns);
|
|
724
902
|
const whereCols = Object.keys(where);
|
|
725
903
|
const incomingUpdatedAt = entry.row?.["updated_at"];
|
|
726
904
|
const ts = incomingUpdatedAt ?? serializeHLC3(entry.timestamp);
|
|
905
|
+
const carriesFullRow = entry.row !== void 0 && pkColumns.length > 0 && pkColumns.every((column) => entry.row?.[column] !== void 0);
|
|
906
|
+
if (carriesFullRow) {
|
|
907
|
+
const row = withNodeId({ ...entry.row, updated_at: ts, deleted_at: ts }, entry);
|
|
908
|
+
const updateCols = Object.keys(row).filter((c) => !pkColumns.includes(c));
|
|
909
|
+
runCompiled(
|
|
910
|
+
this.db,
|
|
911
|
+
compiler.insertInto(fullName).values({ ...row }).onConflict(
|
|
912
|
+
(oc) => oc.columns(pkColumns).doUpdateSet(
|
|
913
|
+
(eb) => Object.fromEntries(updateCols.map((c) => [c, eb.ref(`excluded.${c}`)]))
|
|
914
|
+
).where(sql2.ref("excluded.updated_at"), ">", sql2.ref(`${fullName}.updated_at`))
|
|
915
|
+
).compile()
|
|
916
|
+
);
|
|
917
|
+
return;
|
|
918
|
+
}
|
|
727
919
|
let query = compiler.updateTable(fullName).set({ deleted_at: ts, updated_at: ts, node_id: nodeIdOf(ts, entry) });
|
|
728
920
|
for (const c of whereCols) {
|
|
729
921
|
query = query.where(c, "=", where[c]);
|
|
730
922
|
}
|
|
731
|
-
query = query.where(
|
|
732
|
-
(eb) => eb.or([eb("updated_at", "is", null), eb("updated_at", "<", ts)])
|
|
733
|
-
);
|
|
923
|
+
query = query.where((eb) => eb.or([eb("updated_at", "is", null), eb("updated_at", "<", ts)]));
|
|
734
924
|
runCompiled(this.db, query.compile());
|
|
735
925
|
}
|
|
736
|
-
/**
|
|
926
|
+
/**
|
|
927
|
+
* Whether this table exists in *this* database.
|
|
928
|
+
*
|
|
929
|
+
* Asked before the digest rather than inferred from a failed read, because
|
|
930
|
+
* those are two different answers and only one of them is `[]`. An app that
|
|
931
|
+
* is not installed here genuinely holds nothing; a table that is locked,
|
|
932
|
+
* corrupt or unreadable holds an unknown number of rows, and reporting zero
|
|
933
|
+
* for it is the loudest possible false alarm — every bucket in it reads as a
|
|
934
|
+
* hole, in whichever direction the comparison runs.
|
|
935
|
+
*
|
|
936
|
+
* `sqlite_master` is the cheapest question that separates them, and asking it
|
|
937
|
+
* cannot itself fail for a reason worth swallowing: a database whose schema
|
|
938
|
+
* table will not read is not one whose app tables are merely absent.
|
|
939
|
+
*/
|
|
940
|
+
tableExists(fullName) {
|
|
941
|
+
const row = this.db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?`).all(fullName);
|
|
942
|
+
return row.length > 0;
|
|
943
|
+
}
|
|
944
|
+
/**
|
|
945
|
+
* See `ScanCapableApplier.bucketDigest`. Missing table → `[]`; a table that
|
|
946
|
+
* exists and will not read **throws**, so the caller's `supported: false`
|
|
947
|
+
* path can do its job.
|
|
948
|
+
*
|
|
949
|
+
* This used to be `try { … } catch { return [] }`, which is the same mistake
|
|
950
|
+
* `scanSince` made: `[]` is the wire value for "this table holds nothing", not
|
|
951
|
+
* for "I could not count it". It made both engine-side "could not count ⇒
|
|
952
|
+
* supported: false" guards unreachable on every production path, because no
|
|
953
|
+
* real applier ever threw.
|
|
954
|
+
*/
|
|
955
|
+
async bucketDigest(appId, table, prefixLength = DEFAULT_BUCKET_PREFIX_LENGTH2) {
|
|
956
|
+
const fullName = appSyncableTableName(appId, table);
|
|
957
|
+
if (!this.tableExists(fullName)) return [];
|
|
958
|
+
const compiled = buildBucketDigest2(compiler, fullName, prefixLength);
|
|
959
|
+
return toDigestBuckets2(allCompiled(this.db, compiled));
|
|
960
|
+
}
|
|
961
|
+
/**
|
|
962
|
+
* See `ScanCapableApplier.getNodeWatermarks`. Missing table → `{}`; a table
|
|
963
|
+
* that exists and will not read **throws**.
|
|
964
|
+
*
|
|
965
|
+
* The distinction matters more here than the `{}` return suggests, because
|
|
966
|
+
* `scanSince` plans its authors from this map: an unreadable table that
|
|
967
|
+
* answered `{}` produced no scans, and the scan then reported "nothing owed,
|
|
968
|
+
* every author complete" — the exact silent-loss shape `scanSince`'s own catch
|
|
969
|
+
* was written to prevent, reached one frame earlier.
|
|
970
|
+
*/
|
|
737
971
|
async getNodeWatermarks(appId, table) {
|
|
738
972
|
const fullName = appSyncableTableName(appId, table);
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
);
|
|
745
|
-
} catch {
|
|
746
|
-
return {};
|
|
747
|
-
}
|
|
973
|
+
if (!this.tableExists(fullName)) return {};
|
|
974
|
+
const rows = allCompiled(
|
|
975
|
+
this.db,
|
|
976
|
+
compiler.selectFrom(fullName).select(({ fn }) => ["node_id", fn.max("updated_at").as("max_updated_at")]).groupBy("node_id").compile()
|
|
977
|
+
);
|
|
748
978
|
const out = {};
|
|
749
979
|
for (const row of rows) {
|
|
750
980
|
out[row.node_id] = deserializeHLC3(row.max_updated_at);
|
|
@@ -752,41 +982,74 @@ var SqliteAppSyncableApplier = class {
|
|
|
752
982
|
return out;
|
|
753
983
|
}
|
|
754
984
|
/**
|
|
755
|
-
* Pull-side synthesis:
|
|
756
|
-
*
|
|
757
|
-
*
|
|
758
|
-
*
|
|
759
|
-
* tiebreaker column is needed.
|
|
985
|
+
* Pull-side synthesis: rows the peer hasn't seen, per author, seeking the
|
|
986
|
+
* `(node_id, updated_at)` index rather than reading the table and filtering.
|
|
987
|
+
* See `ScanCapableApplier.scanSince` for the contract and
|
|
988
|
+
* `storage-adapter/src/database/since-queries.ts` for why it is a loop.
|
|
760
989
|
*/
|
|
761
|
-
async scanSince(appId, table,
|
|
990
|
+
async scanSince(appId, table, peerWatermarks, limit) {
|
|
762
991
|
const fullName = appSyncableTableName(appId, table);
|
|
763
|
-
const
|
|
764
|
-
const
|
|
765
|
-
let rows;
|
|
992
|
+
const scans = planNodeScans2(await this.getNodeWatermarks(appId, table), peerWatermarks);
|
|
993
|
+
const pkColumns = this.namespace.get(appId)?.tables.find((t) => t.name === table)?.pkColumns ?? [];
|
|
766
994
|
try {
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
995
|
+
const { rows, hasMore, truncated } = await collectSince2(
|
|
996
|
+
scans,
|
|
997
|
+
limit,
|
|
998
|
+
async (scan, remaining) => {
|
|
999
|
+
const compiled = buildScanSinceForNode2(compiler, fullName, scan, remaining);
|
|
1000
|
+
return allCompiled(this.db, compiled);
|
|
1001
|
+
},
|
|
1002
|
+
(row) => deserializeHLC3(row["updated_at"])
|
|
1003
|
+
);
|
|
1004
|
+
return {
|
|
1005
|
+
// Converted before the row becomes a wire entry: what goes on the wire
|
|
1006
|
+
// is what the app sees, and a peer's applier binds a boolean it can.
|
|
1007
|
+
rows: fromSqliteRows(rows, this.booleanColumnsOf(appId, table)).map((row) => rowToWireEntry(appId, table, row, pkColumns, deserializeHLC3)).filter((entry) => entry !== null),
|
|
1008
|
+
hasMore,
|
|
1009
|
+
truncated
|
|
1010
|
+
};
|
|
1011
|
+
} catch (err) {
|
|
1012
|
+
console.warn(
|
|
1013
|
+
`[app-syncable] scanSince failed for ${appId}.${table}: ${err.message}`
|
|
1014
|
+
);
|
|
1015
|
+
const truncated = {};
|
|
1016
|
+
for (const scan of scans) truncated[scan.nodeId] = null;
|
|
1017
|
+
return { rows: [], hasMore: true, truncated };
|
|
774
1018
|
}
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
1019
|
+
}
|
|
1020
|
+
/**
|
|
1021
|
+
* Run a parsed query. The read half of the app-data plane.
|
|
1022
|
+
*
|
|
1023
|
+
* Compiled by the shared builder rather than here, so the SQL this emits and
|
|
1024
|
+
* the SQL the DSQL applier emits are one piece of code — the two hand-written
|
|
1025
|
+
* read grammars this replaces had already drifted on their limit defaults
|
|
1026
|
+
* alone.
|
|
1027
|
+
*/
|
|
1028
|
+
async runQuery(appId, table, query, options = {}) {
|
|
783
1029
|
const fullName = appSyncableTableName(appId, table);
|
|
784
|
-
const
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
1030
|
+
const booleanColumns = this.booleanColumnsOf(appId, table);
|
|
1031
|
+
if (query.mode === "aggregate") {
|
|
1032
|
+
const compiled2 = buildAppAggregateQuery2(
|
|
1033
|
+
compiler,
|
|
1034
|
+
fullName,
|
|
1035
|
+
query,
|
|
1036
|
+
SQLITE_APP_QUERY_DIALECT2,
|
|
1037
|
+
options
|
|
1038
|
+
);
|
|
1039
|
+
return collectAggregatePage2(query, fromSqliteRows(this.selectRows(compiled2), booleanColumns));
|
|
788
1040
|
}
|
|
789
|
-
|
|
1041
|
+
const compiled = buildAppRowQuery2(compiler, fullName, query, options);
|
|
1042
|
+
return collectRowPage2(query, fromSqliteRows(this.selectRows(compiled), booleanColumns));
|
|
1043
|
+
}
|
|
1044
|
+
/** The names of a declared table's `boolean` columns, or null when it has none. */
|
|
1045
|
+
booleanColumnsOf(appId, table) {
|
|
1046
|
+
return booleanColumnNames(
|
|
1047
|
+
this.namespace.get(appId)?.tables.find((t) => t.name === table)?.columns
|
|
1048
|
+
);
|
|
1049
|
+
}
|
|
1050
|
+
/** Bind and run a compiled SELECT, adapting values SQLite cannot bind. */
|
|
1051
|
+
selectRows(compiled) {
|
|
1052
|
+
return this.db.prepare(compiled.sql).all(...compiled.parameters.map(toSqliteParam));
|
|
790
1053
|
}
|
|
791
1054
|
};
|
|
792
1055
|
function withNodeId(row, entry) {
|
|
@@ -801,23 +1064,138 @@ function nodeIdOf(updatedAt, entry) {
|
|
|
801
1064
|
}
|
|
802
1065
|
return entry.timestamp.nodeId;
|
|
803
1066
|
}
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
1067
|
+
|
|
1068
|
+
// src/library-owner.ts
|
|
1069
|
+
import { sql as sql3 } from "kysely";
|
|
1070
|
+
import { LOCAL_USER_SEGMENT, USER_SEGMENT_PATTERN } from "@starkeep/protocol-primitives";
|
|
1071
|
+
var OWNER_TABLE = "library_owner";
|
|
1072
|
+
function ensureOwnerTable(db) {
|
|
1073
|
+
db.exec(
|
|
1074
|
+
compiler.schema.createTable(OWNER_TABLE).ifNotExists().addColumn("id", "integer", (c) => c.primaryKey()).addColumn("user_id", "text", (c) => c.notNull()).compile().sql
|
|
1075
|
+
);
|
|
1076
|
+
}
|
|
1077
|
+
function boundUserId(db) {
|
|
1078
|
+
ensureOwnerTable(db);
|
|
1079
|
+
const row = db.prepare(
|
|
1080
|
+
compiler.selectFrom(OWNER_TABLE).select("user_id").where("id", "=", sql3.lit(0)).compile().sql
|
|
1081
|
+
).get();
|
|
1082
|
+
return row?.user_id ?? null;
|
|
1083
|
+
}
|
|
1084
|
+
function libraryUserSegment(db) {
|
|
1085
|
+
return boundUserId(db) ?? LOCAL_USER_SEGMENT;
|
|
1086
|
+
}
|
|
1087
|
+
function libraryKeyColumns(db) {
|
|
1088
|
+
return coreKeyColumns(db).filter(({ table }) => tableExists(db, table));
|
|
1089
|
+
}
|
|
1090
|
+
function coreKeyColumns(db) {
|
|
1091
|
+
const columns = [
|
|
1092
|
+
{ table: "shared_records", column: "object_storage_key", primaryKey: false },
|
|
1093
|
+
{ table: "shared_object_availability", column: "object_storage_key", primaryKey: true },
|
|
1094
|
+
{ table: "resident_keys", column: "object_storage_key", primaryKey: true },
|
|
1095
|
+
{ table: "resident_keys", column: "record_id", primaryKey: false }
|
|
1096
|
+
];
|
|
1097
|
+
const tables = db.prepare(
|
|
1098
|
+
compiler.selectFrom("sqlite_master").select("name").where("type", "=", sql3.lit("table")).compile().sql
|
|
1099
|
+
).all();
|
|
1100
|
+
for (const { name } of tables) {
|
|
1101
|
+
if (name.endsWith("__starkeep_sync_records")) {
|
|
1102
|
+
columns.push(
|
|
1103
|
+
{ table: name, column: "id", primaryKey: true },
|
|
1104
|
+
{ table: name, column: "object_storage_key", primaryKey: false }
|
|
1105
|
+
);
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
return columns;
|
|
1109
|
+
}
|
|
1110
|
+
function tableExists(db, table) {
|
|
1111
|
+
return db.prepare(
|
|
1112
|
+
compiler.selectFrom("sqlite_master").select("name").where("type", "=", sql3.lit("table")).where("name", "=", sql3.lit(table)).compile().sql
|
|
1113
|
+
).get() !== void 0;
|
|
1114
|
+
}
|
|
1115
|
+
function namespaces(column, userId) {
|
|
1116
|
+
return [
|
|
1117
|
+
{
|
|
1118
|
+
local: sql3.lit(`shared/${LOCAL_USER_SEGMENT}/%`),
|
|
1119
|
+
// Keeps the slash after `local`, so `shared/local/x` becomes `shared/<userId>/x`.
|
|
1120
|
+
rebound: sql3`'shared/' || ${sql3.lit(userId)} || substr(${sql3.ref(column)}, ${sql3.lit(`shared/${LOCAL_USER_SEGMENT}`.length + 1)})`
|
|
1121
|
+
},
|
|
1122
|
+
// apps/<appId>/local/syncable/... — the app id is the segment before `local`.
|
|
1123
|
+
{
|
|
1124
|
+
local: sql3.lit(`apps/%/${LOCAL_USER_SEGMENT}/syncable/%`),
|
|
1125
|
+
rebound: sql3`replace(${sql3.ref(column)}, ${sql3.lit(`/${LOCAL_USER_SEGMENT}/syncable/`)}, ${sql3.lit(`/${userId}/syncable/`)})`
|
|
1126
|
+
}
|
|
1127
|
+
];
|
|
1128
|
+
}
|
|
1129
|
+
function rebindColumn(db, { table, column, primaryKey }, userId) {
|
|
1130
|
+
let changed = 0;
|
|
1131
|
+
for (const { local, rebound } of namespaces(column, userId)) {
|
|
1132
|
+
if (primaryKey) {
|
|
1133
|
+
const duplicates = compiler.deleteFrom(table).where(sql3.ref(column), "like", local).where(
|
|
1134
|
+
sql3`${rebound} in (${compiler.selectFrom(table).select(sql3.ref(column).as("k"))})`
|
|
1135
|
+
).compile();
|
|
1136
|
+
db.prepare(duplicates.sql).run(...duplicates.parameters);
|
|
1137
|
+
}
|
|
1138
|
+
const update = compiler.updateTable(table).set({ [column]: rebound }).where(sql3.ref(column), "like", local).compile();
|
|
1139
|
+
const result = db.prepare(update.sql).run(...update.parameters);
|
|
1140
|
+
changed += Number(result.changes ?? 0);
|
|
1141
|
+
}
|
|
1142
|
+
return changed;
|
|
1143
|
+
}
|
|
1144
|
+
var LibraryBoundElsewhereError = class extends Error {
|
|
1145
|
+
constructor(boundTo, cloudSays) {
|
|
1146
|
+
super(
|
|
1147
|
+
`This library belongs to user ${boundTo}, but the cloud says this device syncs for user ${cloudSays}. Sync stays off so the two libraries do not mix.`
|
|
1148
|
+
);
|
|
1149
|
+
this.boundTo = boundTo;
|
|
1150
|
+
this.cloudSays = cloudSays;
|
|
1151
|
+
this.name = "LibraryBoundElsewhereError";
|
|
1152
|
+
}
|
|
1153
|
+
boundTo;
|
|
1154
|
+
cloudSays;
|
|
1155
|
+
};
|
|
1156
|
+
function checkLibraryBindable(db, userId) {
|
|
1157
|
+
if (!USER_SEGMENT_PATTERN.test(userId) || userId === LOCAL_USER_SEGMENT) {
|
|
1158
|
+
throw new Error(`cannot bind the library to ${JSON.stringify(userId)}`);
|
|
1159
|
+
}
|
|
1160
|
+
const current = boundUserId(db);
|
|
1161
|
+
if (current !== null && current !== userId) throw new LibraryBoundElsewhereError(current, userId);
|
|
1162
|
+
}
|
|
1163
|
+
function commitLibraryBinding(db, userId, extraKeyColumns = []) {
|
|
1164
|
+
checkLibraryBindable(db, userId);
|
|
1165
|
+
const columns = [...coreKeyColumns(db), ...extraKeyColumns].filter(
|
|
1166
|
+
({ table }) => tableExists(db, table)
|
|
1167
|
+
);
|
|
1168
|
+
let rewritten = 0;
|
|
1169
|
+
db.exec("BEGIN");
|
|
1170
|
+
try {
|
|
1171
|
+
for (const column of columns) rewritten += rebindColumn(db, column, userId);
|
|
1172
|
+
db.prepare(
|
|
1173
|
+
compiler.insertInto(OWNER_TABLE).values({ id: sql3.lit(0), user_id: sql3.raw("?") }).onConflict(
|
|
1174
|
+
(oc) => oc.column("id").doUpdateSet((eb) => ({ user_id: eb.ref("excluded.user_id") }))
|
|
1175
|
+
).compile().sql
|
|
1176
|
+
).run(userId);
|
|
1177
|
+
db.exec("COMMIT");
|
|
1178
|
+
} catch (err) {
|
|
1179
|
+
db.exec("ROLLBACK");
|
|
1180
|
+
throw err;
|
|
810
1181
|
}
|
|
811
|
-
return
|
|
1182
|
+
return rewritten;
|
|
812
1183
|
}
|
|
813
1184
|
export {
|
|
1185
|
+
LibraryBoundElsewhereError,
|
|
814
1186
|
SqliteAppSyncableApplier,
|
|
815
1187
|
SqliteAppSyncableNamespaceStore,
|
|
816
1188
|
SqliteDatabaseAdapter,
|
|
817
1189
|
appSyncableTableName,
|
|
1190
|
+
applyConnectionPragmas,
|
|
1191
|
+
boundUserId,
|
|
1192
|
+
checkLibraryBindable,
|
|
1193
|
+
commitLibraryBinding,
|
|
818
1194
|
deleteAppSyncableNamespace,
|
|
819
1195
|
getAppSyncableNamespace,
|
|
820
1196
|
initializeLocalSchema,
|
|
1197
|
+
libraryKeyColumns,
|
|
1198
|
+
libraryUserSegment,
|
|
821
1199
|
listAppSyncableNamespaces,
|
|
822
1200
|
compiler as sqliteCompiler,
|
|
823
1201
|
upsertAppSyncableNamespace
|