@opendatalabs/personal-server-ts-server 1.34.2 → 1.35.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api-auth.d.ts +1 -0
- package/dist/api-auth.d.ts.map +1 -1
- package/dist/api-auth.js.map +1 -1
- package/dist/api.d.ts +3 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +3 -0
- package/dist/api.js.map +1 -1
- package/dist/app.d.ts +14 -2
- package/dist/app.d.ts.map +1 -1
- package/dist/app.js +64 -8
- package/dist/app.js.map +1 -1
- package/dist/bootstrap.d.ts +19 -0
- package/dist/bootstrap.d.ts.map +1 -1
- package/dist/bootstrap.js +87 -4
- package/dist/bootstrap.js.map +1 -1
- package/dist/client.d.ts +1 -1
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +7 -0
- package/dist/client.js.map +1 -1
- package/dist/enclave-main.d.ts.map +1 -1
- package/dist/enclave-main.js +0 -2
- package/dist/enclave-main.js.map +1 -1
- package/dist/mcp/durable-state.d.ts +2 -1
- package/dist/mcp/durable-state.d.ts.map +1 -1
- package/dist/mcp/durable-state.js +35 -1
- package/dist/mcp/durable-state.js.map +1 -1
- package/dist/pdpp/bootstrap.d.ts.map +1 -1
- package/dist/pdpp/bootstrap.js +1 -0
- package/dist/pdpp/bootstrap.js.map +1 -1
- package/dist/pdpp/records-bootstrap.d.ts +47 -1
- package/dist/pdpp/records-bootstrap.d.ts.map +1 -1
- package/dist/pdpp/records-bootstrap.js +78 -2
- package/dist/pdpp/records-bootstrap.js.map +1 -1
- package/dist/pdpp/replica-authorization.d.ts +35 -0
- package/dist/pdpp/replica-authorization.d.ts.map +1 -0
- package/dist/pdpp/replica-authorization.js +234 -0
- package/dist/pdpp/replica-authorization.js.map +1 -0
- package/dist/pending-blob-deletions.d.ts +1 -0
- package/dist/pending-blob-deletions.d.ts.map +1 -1
- package/dist/pending-blob-deletions.js +8 -4
- package/dist/pending-blob-deletions.js.map +1 -1
- package/dist/routes/data.d.ts.map +1 -1
- package/dist/routes/data.js +5 -0
- package/dist/routes/data.js.map +1 -1
- package/dist/routes/mcp.d.ts +18 -2
- package/dist/routes/mcp.d.ts.map +1 -1
- package/dist/routes/mcp.js +111 -11
- package/dist/routes/mcp.js.map +1 -1
- package/dist/routes/oura.d.ts +6 -0
- package/dist/routes/oura.d.ts.map +1 -0
- package/dist/routes/oura.js +272 -0
- package/dist/routes/oura.js.map +1 -0
- package/dist/routes/pdpp-auth-query-keys.d.ts +7 -0
- package/dist/routes/pdpp-auth-query-keys.d.ts.map +1 -0
- package/dist/routes/pdpp-auth-query-keys.js +25 -0
- package/dist/routes/pdpp-auth-query-keys.js.map +1 -0
- package/dist/routes/pdpp-auth.d.ts +3 -1
- package/dist/routes/pdpp-auth.d.ts.map +1 -1
- package/dist/routes/pdpp-auth.js +126 -45
- package/dist/routes/pdpp-auth.js.map +1 -1
- package/dist/routes/pdpp-blobs.d.ts +13 -1
- package/dist/routes/pdpp-blobs.d.ts.map +1 -1
- package/dist/routes/pdpp-blobs.js +100 -65
- package/dist/routes/pdpp-blobs.js.map +1 -1
- package/dist/routes/pdpp-connections.d.ts.map +1 -1
- package/dist/routes/pdpp-connections.js +24 -1
- package/dist/routes/pdpp-connections.js.map +1 -1
- package/dist/routes/pdpp-instance-bindings.d.ts +3 -0
- package/dist/routes/pdpp-instance-bindings.d.ts.map +1 -1
- package/dist/routes/pdpp-instance-bindings.js +63 -0
- package/dist/routes/pdpp-instance-bindings.js.map +1 -1
- package/dist/routes/pdpp-record-query.d.ts +19 -0
- package/dist/routes/pdpp-record-query.d.ts.map +1 -0
- package/dist/routes/pdpp-record-query.js +51 -0
- package/dist/routes/pdpp-record-query.js.map +1 -0
- package/dist/routes/pdpp-records.d.ts +12 -1
- package/dist/routes/pdpp-records.d.ts.map +1 -1
- package/dist/routes/pdpp-records.js +120 -233
- package/dist/routes/pdpp-records.js.map +1 -1
- package/dist/routes/pdpp-replica-quarantine-fixture.d.ts +65 -0
- package/dist/routes/pdpp-replica-quarantine-fixture.d.ts.map +1 -0
- package/dist/routes/pdpp-replica-quarantine-fixture.js +389 -0
- package/dist/routes/pdpp-replica-quarantine-fixture.js.map +1 -0
- package/dist/storage/claude-method-transition.d.ts +12 -0
- package/dist/storage/claude-method-transition.d.ts.map +1 -0
- package/dist/storage/claude-method-transition.js +139 -0
- package/dist/storage/claude-method-transition.js.map +1 -0
- package/dist/storage/import-operations.d.ts +16 -0
- package/dist/storage/import-operations.d.ts.map +1 -1
- package/dist/storage/import-operations.js +54 -1
- package/dist/storage/import-operations.js.map +1 -1
- package/dist/storage/index-schema.d.ts +1 -1
- package/dist/storage/index-schema.d.ts.map +1 -1
- package/dist/storage/index-schema.js +22 -5
- package/dist/storage/index-schema.js.map +1 -1
- package/dist/storage/legacy-scope-authority.d.ts.map +1 -1
- package/dist/storage/legacy-scope-authority.js +83 -11
- package/dist/storage/legacy-scope-authority.js.map +1 -1
- package/dist/storage/node-data-storage.d.ts.map +1 -1
- package/dist/storage/node-data-storage.js +58 -2
- package/dist/storage/node-data-storage.js.map +1 -1
- package/dist/storage/pdpp-binary-chunks.d.ts +120 -0
- package/dist/storage/pdpp-binary-chunks.d.ts.map +1 -0
- package/dist/storage/pdpp-binary-chunks.js +480 -0
- package/dist/storage/pdpp-binary-chunks.js.map +1 -0
- package/dist/storage/pdpp-records-sqlite-store.d.ts +96 -15
- package/dist/storage/pdpp-records-sqlite-store.d.ts.map +1 -1
- package/dist/storage/pdpp-records-sqlite-store.js +616 -211
- package/dist/storage/pdpp-records-sqlite-store.js.map +1 -1
- package/dist/storage/pdpp-replica-canonical-port.d.ts +202 -0
- package/dist/storage/pdpp-replica-canonical-port.d.ts.map +1 -0
- package/dist/storage/pdpp-replica-canonical-port.js +451 -0
- package/dist/storage/pdpp-replica-canonical-port.js.map +1 -0
- package/dist/storage/pdpp-replica-coordinator-state.d.ts +16 -0
- package/dist/storage/pdpp-replica-coordinator-state.d.ts.map +1 -0
- package/dist/storage/pdpp-replica-coordinator-state.js +25 -0
- package/dist/storage/pdpp-replica-coordinator-state.js.map +1 -0
- package/dist/storage/pdpp-replica-copies.d.ts +81 -0
- package/dist/storage/pdpp-replica-copies.d.ts.map +1 -0
- package/dist/storage/pdpp-replica-copies.js +59 -0
- package/dist/storage/pdpp-replica-copies.js.map +1 -0
- package/dist/storage/pdpp-replica-reconciler.d.ts +96 -0
- package/dist/storage/pdpp-replica-reconciler.d.ts.map +1 -0
- package/dist/storage/pdpp-replica-reconciler.js +710 -0
- package/dist/storage/pdpp-replica-reconciler.js.map +1 -0
- package/dist/storage/pdpp-replica-sql.d.ts +13 -0
- package/dist/storage/pdpp-replica-sql.d.ts.map +1 -0
- package/dist/storage/pdpp-replica-sql.js +43 -0
- package/dist/storage/pdpp-replica-sql.js.map +1 -0
- package/dist/storage/pdpp-replica-staging.d.ts +36 -0
- package/dist/storage/pdpp-replica-staging.d.ts.map +1 -0
- package/dist/storage/pdpp-replica-staging.js +202 -0
- package/dist/storage/pdpp-replica-staging.js.map +1 -0
- package/dist/storage/pdpp-replica-visibility-view.d.ts +64 -0
- package/dist/storage/pdpp-replica-visibility-view.d.ts.map +1 -0
- package/dist/storage/pdpp-replica-visibility-view.js +1397 -0
- package/dist/storage/pdpp-replica-visibility-view.js.map +1 -0
- package/dist/storage/pdpp-replica-writer-identity.d.ts +64 -0
- package/dist/storage/pdpp-replica-writer-identity.d.ts.map +1 -0
- package/dist/storage/pdpp-replica-writer-identity.js +81 -0
- package/dist/storage/pdpp-replica-writer-identity.js.map +1 -0
- package/dist/storage/pdpp-replication-door.d.ts +60 -0
- package/dist/storage/pdpp-replication-door.d.ts.map +1 -0
- package/dist/storage/pdpp-replication-door.js +203 -0
- package/dist/storage/pdpp-replication-door.js.map +1 -0
- package/dist/storage/pdpp-replication-host.d.ts +67 -0
- package/dist/storage/pdpp-replication-host.d.ts.map +1 -0
- package/dist/storage/pdpp-replication-host.js +159 -0
- package/dist/storage/pdpp-replication-host.js.map +1 -0
- package/dist/storage/pdpp-replication-memory.fixture.d.ts +2 -0
- package/dist/storage/pdpp-replication-memory.fixture.d.ts.map +1 -0
- package/dist/storage/pdpp-replication-memory.fixture.js +267 -0
- package/dist/storage/pdpp-replication-memory.fixture.js.map +1 -0
- package/dist/storage/pdpp-replication-preparation.d.ts +3 -0
- package/dist/storage/pdpp-replication-preparation.d.ts.map +1 -0
- package/dist/storage/pdpp-replication-preparation.js +3 -0
- package/dist/storage/pdpp-replication-preparation.js.map +1 -0
- package/dist/storage/pdpp-replication-promotion-memory.fixture.d.ts +2 -0
- package/dist/storage/pdpp-replication-promotion-memory.fixture.d.ts.map +1 -0
- package/dist/storage/pdpp-replication-promotion-memory.fixture.js +65 -0
- package/dist/storage/pdpp-replication-promotion-memory.fixture.js.map +1 -0
- package/dist/storage/pdpp-replication-receiver.d.ts +230 -0
- package/dist/storage/pdpp-replication-receiver.d.ts.map +1 -0
- package/dist/storage/pdpp-replication-receiver.js +950 -0
- package/dist/storage/pdpp-replication-receiver.js.map +1 -0
- package/dist/storage/pdpp-replication-sqlite.d.ts +95 -0
- package/dist/storage/pdpp-replication-sqlite.d.ts.map +1 -0
- package/dist/storage/pdpp-replication-sqlite.js +615 -0
- package/dist/storage/pdpp-replication-sqlite.js.map +1 -0
- package/dist/storage/pdpp-replication-transport.d.ts +39 -0
- package/dist/storage/pdpp-replication-transport.d.ts.map +1 -0
- package/dist/storage/pdpp-replication-transport.js +196 -0
- package/dist/storage/pdpp-replication-transport.js.map +1 -0
- package/dist/storage/schema-versions.d.ts +5 -0
- package/dist/storage/schema-versions.d.ts.map +1 -0
- package/dist/storage/schema-versions.js +26 -0
- package/dist/storage/schema-versions.js.map +1 -0
- package/dist/storage/strava-method-transition.d.ts +18 -0
- package/dist/storage/strava-method-transition.d.ts.map +1 -0
- package/dist/storage/strava-method-transition.js +209 -0
- package/dist/storage/strava-method-transition.js.map +1 -0
- package/dist/ui/ps-lite-debug.js +154947 -124896
- package/package.json +3 -3
|
@@ -1,32 +1,15 @@
|
|
|
1
|
+
import { assertSupportedSchemaVersions } from "./schema-versions.js";
|
|
2
|
+
import { binaryLength, readBinary, verifyBinary, } from "./pdpp-binary-chunks.js";
|
|
3
|
+
import { parseTime, timeSortKey, } from "@opendatalabs/personal-server-ts-core/pdpp";
|
|
4
|
+
import { selectSyncChange, decodeListCursor, decodeSyncPageCursor, listSnapshotPage, compareRecordIdentity, LIST_ORDER_COLUMNS, } from "@opendatalabs/personal-server-ts-core/storage/pdpp-records";
|
|
1
5
|
import { createHash } from "node:crypto";
|
|
2
6
|
import { randomUUID } from "node:crypto";
|
|
3
7
|
import { encodeRecordKey, planIngest, RecordKeyError, summarizeIngest, decodeCursor, encodeCursor, InvalidCursorSyntaxError, CursorExpiredError, InvalidCursorError, BlobConflictError, } from "@opendatalabs/personal-server-ts-core/storage/pdpp-records";
|
|
4
8
|
import { isCompletionTime, liveSnapshotDigest, parseCompletionRequest, StreamCompletionError, } from "./pdpp-stream-completion.js";
|
|
5
9
|
export { StreamCompletionError, } from "./pdpp-stream-completion.js";
|
|
6
|
-
|
|
7
|
-
/** RFC 3339 `date-time`, with calendar-valid fields. No clock check. */
|
|
10
|
+
/** Compatibility export; the portable temporal policy owns RFC3339 validity. */
|
|
8
11
|
export function isRfc3339DateTime(value) {
|
|
9
|
-
|
|
10
|
-
return false;
|
|
11
|
-
const match = RFC3339_DATE_TIME.exec(value);
|
|
12
|
-
if (!match)
|
|
13
|
-
return false;
|
|
14
|
-
const [year, month, day, hour, minute, second] = match
|
|
15
|
-
.slice(1, 7)
|
|
16
|
-
.map(Number);
|
|
17
|
-
const offsetHour = Number(match[8] ?? 0);
|
|
18
|
-
const offsetMinute = Number(match[9] ?? 0);
|
|
19
|
-
const daysInMonth = new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
20
|
-
return (month >= 1 &&
|
|
21
|
-
month <= 12 &&
|
|
22
|
-
day >= 1 &&
|
|
23
|
-
day <= daysInMonth &&
|
|
24
|
-
hour <= 23 &&
|
|
25
|
-
minute <= 59 &&
|
|
26
|
-
second <= 59 &&
|
|
27
|
-
offsetHour <= 23 &&
|
|
28
|
-
offsetMinute <= 59 &&
|
|
29
|
-
!Number.isNaN(Date.parse(value)));
|
|
12
|
+
return parseTime(value, "date-time") !== null;
|
|
30
13
|
}
|
|
31
14
|
export class PdppBindingError extends Error {
|
|
32
15
|
reason;
|
|
@@ -221,8 +204,54 @@ CREATE TABLE IF NOT EXISTS pdpp_record_arrival_clock (
|
|
|
221
204
|
value INTEGER NOT NULL
|
|
222
205
|
);
|
|
223
206
|
INSERT OR IGNORE INTO pdpp_record_arrival_clock (id, value) VALUES (1, 0);
|
|
207
|
+
`,
|
|
208
|
+
// v12: ordered history traversal and one-key as-of lookup. The horizon
|
|
209
|
+
// index alone scans an instance's full history for every outer row.
|
|
210
|
+
`
|
|
211
|
+
CREATE INDEX IF NOT EXISTS idx_pdpp_record_changes_snapshot_order
|
|
212
|
+
ON pdpp_record_changes (stream, emitted_at, record_key, instance, written_at);
|
|
213
|
+
CREATE INDEX IF NOT EXISTS idx_pdpp_record_changes_snapshot_latest
|
|
214
|
+
ON pdpp_record_changes (instance, stream, record_key, written_at DESC, version DESC);
|
|
215
|
+
`,
|
|
216
|
+
// v13: an `emitted_key` that orders emitted time by instant rather than by
|
|
217
|
+
// timestamp text, for list order, list seeks and the latest stream time.
|
|
218
|
+
// `migrateToV13` applies it.
|
|
219
|
+
"",
|
|
220
|
+
// v14: one recoverable export-to-live transition per Claude connection.
|
|
221
|
+
`CREATE TABLE IF NOT EXISTS pdpp_claude_transition (
|
|
222
|
+
instance TEXT PRIMARY KEY,
|
|
223
|
+
archive_instance TEXT NOT NULL UNIQUE,
|
|
224
|
+
generation INTEGER NOT NULL,
|
|
225
|
+
snapshots_json TEXT NOT NULL,
|
|
226
|
+
conversations_covered INTEGER NOT NULL DEFAULT 0
|
|
227
|
+
);`,
|
|
228
|
+
// v15: expression index for the retained-history blob reference check in
|
|
229
|
+
// `deleteUnclaimedBlobs`. Without it each candidate blob scanned every
|
|
230
|
+
// history row at every writable open. `json_valid` guards the extraction:
|
|
231
|
+
// `json_extract` raises on malformed JSON, which would make this index
|
|
232
|
+
// (and every later insert of such a row) fail. The outer CAST gives the
|
|
233
|
+
// expression TEXT affinity, which the TEXT `blob_id` comparison needs before
|
|
234
|
+
// the planner will search it; a CASE alone does not provide that. Additive
|
|
235
|
+
// only; the expression must stay identical to `noRetainedHistoryReferenceSql`.
|
|
236
|
+
`
|
|
237
|
+
CREATE INDEX IF NOT EXISTS idx_pdpp_record_changes_blob_ref
|
|
238
|
+
ON pdpp_record_changes (CAST(CASE WHEN json_valid(data) THEN json_extract(data, '$.blob_ref.blob_id') END AS TEXT));
|
|
239
|
+
`,
|
|
240
|
+
// v16: converge main v15 (history index) with Desktop v15 (Strava).
|
|
241
|
+
// Either v15 database may lack the other addition; preserve both by presence.
|
|
242
|
+
`CREATE TABLE IF NOT EXISTS pdpp_strava_transition (
|
|
243
|
+
instance TEXT PRIMARY KEY,
|
|
244
|
+
archive_instance TEXT NOT NULL UNIQUE,
|
|
245
|
+
generation INTEGER NOT NULL,
|
|
246
|
+
snapshots_json TEXT NOT NULL,
|
|
247
|
+
activities_covered INTEGER NOT NULL DEFAULT 0
|
|
248
|
+
);
|
|
249
|
+
CREATE INDEX IF NOT EXISTS idx_pdpp_record_changes_blob_ref
|
|
250
|
+
ON pdpp_record_changes (CAST(CASE WHEN json_valid(data) THEN json_extract(data, '$.blob_ref.blob_id') END AS TEXT));
|
|
224
251
|
`,
|
|
225
252
|
];
|
|
253
|
+
/** The records namespace in the shared index.db; preflight before bootstrap effects. */
|
|
254
|
+
export const PDPP_RECORD_SCHEMA_VERSION = MIGRATIONS.length;
|
|
226
255
|
function addCompletionProofColumns(db) {
|
|
227
256
|
// `table_xinfo`, unlike `table_info`, also lists generated columns.
|
|
228
257
|
const existing = new Map(db.prepare("PRAGMA table_xinfo(pdpp_stream_completion)").all().map((column) => [column.name, column]));
|
|
@@ -240,6 +269,27 @@ function addCompletionProofColumns(db) {
|
|
|
240
269
|
}
|
|
241
270
|
}
|
|
242
271
|
}
|
|
272
|
+
/** `emitted_key` is `timeSortKey(emitted_at)`, written by SQL on every insert. */
|
|
273
|
+
function registerTimeKey(db) {
|
|
274
|
+
db.function("pdpp_time_key", { deterministic: true }, (value) => timeSortKey(String(value)));
|
|
275
|
+
}
|
|
276
|
+
function migrateToV13(db) {
|
|
277
|
+
for (const table of ["pdpp_records", "pdpp_record_changes"]) {
|
|
278
|
+
const columns = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
279
|
+
if (!columns.some((c) => c.name === "emitted_key")) {
|
|
280
|
+
db.exec(`ALTER TABLE ${table} ADD COLUMN emitted_key TEXT`);
|
|
281
|
+
}
|
|
282
|
+
db.exec(`UPDATE ${table} SET emitted_key = pdpp_time_key(emitted_at) WHERE emitted_key IS NULL`);
|
|
283
|
+
}
|
|
284
|
+
db.exec(`
|
|
285
|
+
DROP INDEX IF EXISTS idx_pdpp_record_changes_snapshot_order;
|
|
286
|
+
CREATE INDEX IF NOT EXISTS idx_pdpp_record_changes_snapshot_key_order
|
|
287
|
+
ON pdpp_record_changes (stream, emitted_key, record_key, instance, written_at);
|
|
288
|
+
CREATE INDEX IF NOT EXISTS idx_pdpp_records_live_instance_stream_emitted_key
|
|
289
|
+
ON pdpp_records (instance, stream, emitted_key, emitted_at)
|
|
290
|
+
WHERE deleted = 0;
|
|
291
|
+
`);
|
|
292
|
+
}
|
|
243
293
|
function migrate(db) {
|
|
244
294
|
db.exec(`
|
|
245
295
|
CREATE TABLE IF NOT EXISTS pdpp_schema_version (
|
|
@@ -259,6 +309,8 @@ function migrate(db) {
|
|
|
259
309
|
if (v === 4) {
|
|
260
310
|
db.prepare("INSERT OR IGNORE INTO pdpp_store_metadata (id, epoch) VALUES (1, ?)").run(randomUUID());
|
|
261
311
|
}
|
|
312
|
+
if (v === 12)
|
|
313
|
+
migrateToV13(db);
|
|
262
314
|
}
|
|
263
315
|
db.prepare("UPDATE pdpp_schema_version SET version = ? WHERE id = 1").run(MIGRATIONS.length);
|
|
264
316
|
});
|
|
@@ -274,28 +326,71 @@ function migrate(db) {
|
|
|
274
326
|
}
|
|
275
327
|
function sweepStaleBlobClaims(db) {
|
|
276
328
|
db.transaction(() => {
|
|
329
|
+
// A method-transition archive is retained history, not a registered
|
|
330
|
+
// instance: it has no binding row, so only the transition ledger vouches
|
|
331
|
+
// for its claims. Erasing the connection drops the ledger row and claims.
|
|
277
332
|
db.exec(`
|
|
278
333
|
DELETE FROM pdpp_blob_claims
|
|
279
334
|
WHERE NOT EXISTS (
|
|
280
335
|
SELECT 1 FROM pdpp_instance_binding b
|
|
281
336
|
WHERE b.instance = pdpp_blob_claims.instance
|
|
282
337
|
AND b.generation = pdpp_blob_claims.generation
|
|
338
|
+
)
|
|
339
|
+
AND NOT EXISTS (
|
|
340
|
+
SELECT 1 FROM pdpp_claude_transition t
|
|
341
|
+
WHERE t.archive_instance = pdpp_blob_claims.instance
|
|
342
|
+
)
|
|
343
|
+
AND NOT EXISTS (
|
|
344
|
+
SELECT 1 FROM pdpp_strava_transition t
|
|
345
|
+
WHERE t.archive_instance = pdpp_blob_claims.instance
|
|
283
346
|
);
|
|
284
347
|
`);
|
|
285
|
-
|
|
348
|
+
collectStartupUnclaimedBlobs(db);
|
|
286
349
|
})();
|
|
287
350
|
}
|
|
351
|
+
/**
|
|
352
|
+
* Automatic (startup) blob collection. A history row whose JSON cannot be
|
|
353
|
+
* parsed may reference any blob, so collection is skipped while one exists.
|
|
354
|
+
* Explicit owner erasure calls `deleteUnclaimedBlobs` directly and is not
|
|
355
|
+
* gated here.
|
|
356
|
+
*/
|
|
357
|
+
function collectStartupUnclaimedBlobs(db) {
|
|
358
|
+
const { found } = db
|
|
359
|
+
.prepare("SELECT EXISTS (SELECT 1 FROM pdpp_record_changes WHERE NOT json_valid(data)) AS found")
|
|
360
|
+
.get();
|
|
361
|
+
if (found === 1) {
|
|
362
|
+
const { count } = db
|
|
363
|
+
.prepare("SELECT COUNT(*) AS count FROM pdpp_record_changes WHERE NOT json_valid(data)")
|
|
364
|
+
.get();
|
|
365
|
+
console.warn(`[pdpp-records] skipped startup blob collection: ${count} history row(s) hold invalid JSON, so their blob references cannot be read`);
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
deleteUnclaimedBlobs(db);
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* `NOT EXISTS` predicate: no retained history row references `blobId`.
|
|
372
|
+
* The expression must match the v15 index `idx_pdpp_record_changes_blob_ref`
|
|
373
|
+
* exactly, so the planner searches by blob id rather than scanning history.
|
|
374
|
+
*/
|
|
375
|
+
export function noRetainedHistoryReferenceSql(blobId) {
|
|
376
|
+
return `NOT EXISTS (SELECT 1 FROM pdpp_record_changes WHERE CAST(CASE WHEN json_valid(data) THEN json_extract(data, '$.blob_ref.blob_id') END AS TEXT) = ${blobId})`;
|
|
377
|
+
}
|
|
288
378
|
function deleteUnclaimedBlobs(db) {
|
|
379
|
+
// Claims reconstructed by v4 only cover current records. Retained history
|
|
380
|
+
// independently owns its bytes, even after overwrite or tombstone. Explicit
|
|
381
|
+
// owner erasure removes that history before calling this collector.
|
|
289
382
|
db.exec(`
|
|
290
383
|
DELETE FROM pdpp_blob_bytes
|
|
291
384
|
WHERE blob_id IN (
|
|
292
385
|
SELECT blob_id FROM pdpp_blobs b
|
|
293
386
|
WHERE NOT EXISTS (SELECT 1 FROM pdpp_blob_claims c WHERE c.blob_id = b.blob_id)
|
|
294
387
|
AND NOT EXISTS (SELECT 1 FROM pdpp_records r WHERE r.blob_id = b.blob_id AND r.deleted = 0)
|
|
388
|
+
AND ${noRetainedHistoryReferenceSql("b.blob_id")}
|
|
295
389
|
);
|
|
296
390
|
DELETE FROM pdpp_blobs
|
|
297
391
|
WHERE NOT EXISTS (SELECT 1 FROM pdpp_blob_claims c WHERE c.blob_id = pdpp_blobs.blob_id)
|
|
298
|
-
AND NOT EXISTS (SELECT 1 FROM pdpp_records r WHERE r.blob_id = pdpp_blobs.blob_id AND r.deleted = 0)
|
|
392
|
+
AND NOT EXISTS (SELECT 1 FROM pdpp_records r WHERE r.blob_id = pdpp_blobs.blob_id AND r.deleted = 0)
|
|
393
|
+
AND ${noRetainedHistoryReferenceSql("pdpp_blobs.blob_id")};
|
|
299
394
|
`);
|
|
300
395
|
}
|
|
301
396
|
function toBinding(row) {
|
|
@@ -370,23 +465,11 @@ function toRow(db) {
|
|
|
370
465
|
deleted: false,
|
|
371
466
|
};
|
|
372
467
|
}
|
|
373
|
-
function
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
if (field in data)
|
|
379
|
-
projected[field] = data[field];
|
|
380
|
-
}
|
|
381
|
-
return projected;
|
|
382
|
-
}
|
|
383
|
-
/**
|
|
384
|
-
* SQLite-backed PdppRecordStore for desktop persistence. Requires
|
|
385
|
-
* `better-sqlite3` (already a `packages/server` dependency in this repo).
|
|
386
|
-
* A readonly database must already have the current schema; no migration,
|
|
387
|
-
* journal-mode change or stale-blob sweep runs. SQLite rejects write methods.
|
|
388
|
-
*/
|
|
389
|
-
export function createSqliteRecordStore(db) {
|
|
468
|
+
export function createSqliteRecordStore(db, options = {}) {
|
|
469
|
+
assertSupportedSchemaVersions(db, PDPP_RECORD_SCHEMA_VERSION);
|
|
470
|
+
// Insert statements call it, so even a readonly handle must register it
|
|
471
|
+
// before they are prepared.
|
|
472
|
+
registerTimeKey(db);
|
|
390
473
|
if (db.readonly) {
|
|
391
474
|
const current = db
|
|
392
475
|
.prepare("SELECT version FROM pdpp_schema_version WHERE id = 1")
|
|
@@ -400,25 +483,56 @@ export function createSqliteRecordStore(db) {
|
|
|
400
483
|
migrate(db);
|
|
401
484
|
sweepStaleBlobClaims(db);
|
|
402
485
|
}
|
|
486
|
+
const replica = options.replica?.(db);
|
|
487
|
+
/**
|
|
488
|
+
* Every connection-scoped write enters through this quarantine boundary.
|
|
489
|
+
* Without replication a top-level call begins the transaction main began:
|
|
490
|
+
* deferred, so a mutation that writes nothing takes no write lock.
|
|
491
|
+
* With replication the transaction is immediate and starts from derived
|
|
492
|
+
* state that matches canonical rows.
|
|
493
|
+
*/
|
|
494
|
+
const connectionMutation = (instances, write, begin = "deferred") => {
|
|
495
|
+
const run = () => {
|
|
496
|
+
const ids = typeof instances === "function" ? instances() : instances;
|
|
497
|
+
for (const instance of new Set(typeof ids === "string" ? [ids] : ids))
|
|
498
|
+
replica?.assertConnectionWritable(instance);
|
|
499
|
+
return write();
|
|
500
|
+
};
|
|
501
|
+
// Inside an outer transaction better-sqlite3 runs this as a SAVEPOINT, so
|
|
502
|
+
// a mutator that throws undoes its own writes even if the caller catches
|
|
503
|
+
// the error and commits. That is what main's per-mutator
|
|
504
|
+
// `db.transaction` gave every one of these.
|
|
505
|
+
if (replica)
|
|
506
|
+
return replica.exclusive(run);
|
|
507
|
+
const atomic = db.transaction(run);
|
|
508
|
+
return begin === "immediate" ? atomic.immediate() : atomic();
|
|
509
|
+
};
|
|
510
|
+
/** The one mutation main opened with `BEGIN IMMEDIATE`: a stream completion. */
|
|
511
|
+
const immediateConnectionMutation = (instance, write) => connectionMutation(instance, write, "immediate");
|
|
512
|
+
/** Replica quarantine filter for read SQL; empty without replication. */
|
|
513
|
+
const unquarantined = (alias) => replica ? ` AND ${replica.unquarantinedRecordSql(alias)}` : "";
|
|
514
|
+
/** Without replication a read needs no snapshot beyond its write horizon. */
|
|
515
|
+
const inReadSnapshot = (read) => replica ? db.transaction(read).deferred() : read();
|
|
403
516
|
const nextWriteSeq = db.prepare("UPDATE pdpp_write_clock SET value = value + 1 WHERE id = 1 RETURNING value");
|
|
404
517
|
const readWriteClockStmt = db.prepare("SELECT value FROM pdpp_write_clock WHERE id = 1");
|
|
405
518
|
const readWriteClock = () => readWriteClockStmt.get().value;
|
|
406
519
|
const getCurrentStmt = db.prepare("SELECT * FROM pdpp_records WHERE instance = ? AND stream = ? AND record_key = ?");
|
|
407
520
|
const upsertCurrentStmt = db.prepare(`
|
|
408
|
-
INSERT INTO pdpp_records (instance, stream, record_key, data, version, emitted_at, deleted, deleted_at, blob_id)
|
|
409
|
-
VALUES (@instance, @stream, @record_key, @data, @version, @emitted_at, @deleted, @deleted_at, @blob_id)
|
|
521
|
+
INSERT INTO pdpp_records (instance, stream, record_key, data, version, emitted_at, emitted_key, deleted, deleted_at, blob_id)
|
|
522
|
+
VALUES (@instance, @stream, @record_key, @data, @version, @emitted_at, pdpp_time_key(@emitted_at), @deleted, @deleted_at, @blob_id)
|
|
410
523
|
ON CONFLICT (instance, stream, record_key) DO UPDATE SET
|
|
411
524
|
data = excluded.data,
|
|
412
525
|
version = excluded.version,
|
|
413
526
|
emitted_at = excluded.emitted_at,
|
|
527
|
+
emitted_key = excluded.emitted_key,
|
|
414
528
|
deleted = excluded.deleted,
|
|
415
529
|
deleted_at = excluded.deleted_at,
|
|
416
530
|
blob_id = excluded.blob_id
|
|
417
531
|
`);
|
|
418
532
|
const insertHistoryStmt = db.prepare(`
|
|
419
533
|
INSERT INTO pdpp_record_changes
|
|
420
|
-
(instance, stream, record_key, version, data, emitted_at, deleted, deleted_at, written_at)
|
|
421
|
-
VALUES (@instance, @stream, @record_key, @version, @data, @emitted_at, @deleted, @deleted_at, @written_at)
|
|
534
|
+
(instance, stream, record_key, version, data, emitted_at, emitted_key, deleted, deleted_at, written_at)
|
|
535
|
+
VALUES (@instance, @stream, @record_key, @version, @data, @emitted_at, pdpp_time_key(@emitted_at), @deleted, @deleted_at, @written_at)
|
|
422
536
|
`);
|
|
423
537
|
const putBlobStmt = db.prepare(`
|
|
424
538
|
INSERT INTO pdpp_blobs (blob_id, mime_type, size_bytes, sha256)
|
|
@@ -427,10 +541,14 @@ export function createSqliteRecordStore(db) {
|
|
|
427
541
|
mime_type = excluded.mime_type, size_bytes = excluded.size_bytes, sha256 = excluded.sha256
|
|
428
542
|
`);
|
|
429
543
|
const getBlobStmt = db.prepare("SELECT * FROM pdpp_blobs WHERE blob_id = ?");
|
|
430
|
-
const findBlobRefsStmt = db.prepare(
|
|
544
|
+
const findBlobRefsStmt = db.prepare(`SELECT r.instance, r.stream, r.record_key FROM pdpp_records r WHERE r.blob_id = ? AND r.deleted = 0${unquarantined("r")}`);
|
|
431
545
|
const insertBlobBytesStmt = db.prepare("INSERT INTO pdpp_blob_bytes (blob_id, bytes) VALUES (@blob_id, @bytes)");
|
|
432
546
|
const getBlobBytesStmt = db.prepare("SELECT bytes FROM pdpp_blob_bytes WHERE blob_id = ?");
|
|
433
547
|
const dropCompletionStmt = db.prepare("DELETE FROM pdpp_stream_completion WHERE instance = ? AND stream = ?");
|
|
548
|
+
function dropCompletion(instance, stream) {
|
|
549
|
+
if (dropCompletionStmt.run(instance, stream).changes > 0)
|
|
550
|
+
replica?.captureInvalidation(instance, stream);
|
|
551
|
+
}
|
|
434
552
|
const noteArrivalStmt = db.prepare(`
|
|
435
553
|
INSERT INTO pdpp_record_arrival (instance, stream, record_key, generation, arrival)
|
|
436
554
|
VALUES (@instance, @stream, @record_key, @generation, (
|
|
@@ -456,7 +574,7 @@ export function createSqliteRecordStore(db) {
|
|
|
456
574
|
const row = getCurrentStmt.get(instance, stream, recordKey);
|
|
457
575
|
return row?.version ?? 0;
|
|
458
576
|
}
|
|
459
|
-
function ingestBatch(envelopes, streamSemantics, primaryKeyFields, binding, validateData) {
|
|
577
|
+
function ingestBatch(envelopes, streamSemantics, primaryKeyFields, binding, validateData, declarationFor) {
|
|
460
578
|
const results = [];
|
|
461
579
|
// The whole batch is one SQLite transaction: a thrown error rolls back
|
|
462
580
|
// every write from this call, never leaving partial version/
|
|
@@ -482,7 +600,9 @@ export function createSqliteRecordStore(db) {
|
|
|
482
600
|
checkBinding(binding.method, generation, envelopes);
|
|
483
601
|
}
|
|
484
602
|
envelopes.forEach((envelope, index) => {
|
|
485
|
-
const
|
|
603
|
+
const semantics = streamSemantics(envelope.stream, envelope.instance);
|
|
604
|
+
const primaryKey = primaryKeyFields(envelope.stream, envelope.instance);
|
|
605
|
+
const plan = planIngest(index, envelope, semantics, primaryKey, (recordKey) => {
|
|
486
606
|
const row = getCurrentStmt.get(envelope.instance, envelope.stream, recordKey);
|
|
487
607
|
return row
|
|
488
608
|
? {
|
|
@@ -515,6 +635,7 @@ export function createSqliteRecordStore(db) {
|
|
|
515
635
|
return;
|
|
516
636
|
}
|
|
517
637
|
writeVersion(envelope.instance, envelope.stream, recordKey, data, envelope.emitted_at);
|
|
638
|
+
replica?.recordWritten(envelope.instance, envelope.stream, recordKey, semantics, primaryKey, declarationFor?.(envelope.stream, envelope.instance));
|
|
518
639
|
if (data === null) {
|
|
519
640
|
forgetArrivalStmt.run(envelope.instance, envelope.stream, recordKey);
|
|
520
641
|
}
|
|
@@ -522,8 +643,9 @@ export function createSqliteRecordStore(db) {
|
|
|
522
643
|
noteArrival(envelope.instance, envelope.stream, recordKey, generation);
|
|
523
644
|
}
|
|
524
645
|
});
|
|
646
|
+
replica?.settleLocalWrites();
|
|
525
647
|
});
|
|
526
|
-
runBatch();
|
|
648
|
+
connectionMutation(envelopes.map((envelope) => envelope.instance), () => runBatch());
|
|
527
649
|
return summarizeIngest(results);
|
|
528
650
|
}
|
|
529
651
|
function blobClaimedByCurrentGeneration(blobId, instance) {
|
|
@@ -533,12 +655,13 @@ export function createSqliteRecordStore(db) {
|
|
|
533
655
|
}
|
|
534
656
|
/** Writes the next version of a key: `data === null` is a tombstone. */
|
|
535
657
|
function writeVersion(instance, stream, recordKey, data, emittedAt) {
|
|
658
|
+
replica?.assertKeyWritable(instance, stream, recordKey);
|
|
536
659
|
const deleted = data === null;
|
|
537
660
|
const version = latestVersion(instance, stream, recordKey) + 1;
|
|
538
661
|
const writtenAt = nextWriteSeq.get();
|
|
539
662
|
// The stream no longer holds what a replace delivered. `replaceStream`
|
|
540
663
|
// writes its own receipt after its writes.
|
|
541
|
-
|
|
664
|
+
dropCompletion(instance, stream);
|
|
542
665
|
const dataJson = deleted ? null : JSON.stringify(data);
|
|
543
666
|
const deletedAt = deleted ? emittedAt : null;
|
|
544
667
|
upsertCurrentStmt.run({
|
|
@@ -584,8 +707,9 @@ export function createSqliteRecordStore(db) {
|
|
|
584
707
|
input.sourceCollectedAt !== undefined)) {
|
|
585
708
|
throw new TypeError("invalid_completion_policy");
|
|
586
709
|
}
|
|
587
|
-
return
|
|
710
|
+
return connectionMutation(input.instance, () => {
|
|
588
711
|
checkInstanceBinding(input.instance, input.method, input.generation);
|
|
712
|
+
replica?.assertStreamWritable(input.instance, input.stream, input.generation);
|
|
589
713
|
const seen = new Set();
|
|
590
714
|
/** Live keys of the snapshot, in the order it delivered them. */
|
|
591
715
|
const snapshotOrder = [];
|
|
@@ -641,12 +765,14 @@ export function createSqliteRecordStore(db) {
|
|
|
641
765
|
: false;
|
|
642
766
|
for (const write of writes) {
|
|
643
767
|
writeVersion(input.instance, input.stream, write.recordKey, write.data, write.emittedAt);
|
|
768
|
+
replica?.recordWritten(input.instance, input.stream, write.recordKey, "mutable_state", input.primaryKey, input.declaration);
|
|
644
769
|
}
|
|
645
770
|
const missing = db
|
|
646
771
|
.prepare("SELECT record_key FROM pdpp_records WHERE instance = ? AND stream = ? AND deleted = 0")
|
|
647
772
|
.all(input.instance, input.stream).filter(({ record_key }) => !seen.has(record_key));
|
|
648
773
|
for (const { record_key } of missing) {
|
|
649
774
|
writeVersion(input.instance, input.stream, record_key, null, input.emittedAt);
|
|
775
|
+
replica?.recordWritten(input.instance, input.stream, record_key, "mutable_state", input.primaryKey, input.declaration);
|
|
650
776
|
}
|
|
651
777
|
// Desktop rebuilt a fully refreshed stream in emit order, unchanged
|
|
652
778
|
// keys included.
|
|
@@ -655,7 +781,7 @@ export function createSqliteRecordStore(db) {
|
|
|
655
781
|
noteArrival(input.instance, input.stream, recordKey, input.generation);
|
|
656
782
|
}
|
|
657
783
|
if (!preserved)
|
|
658
|
-
|
|
784
|
+
dropCompletion(input.instance, input.stream);
|
|
659
785
|
if (input.sourceCollectedAt !== undefined) {
|
|
660
786
|
db.prepare(`INSERT INTO pdpp_stream_completion
|
|
661
787
|
(instance, stream, method, generation, source_collected_at, write_seq, live_count)
|
|
@@ -663,8 +789,11 @@ export function createSqliteRecordStore(db) {
|
|
|
663
789
|
.prepare("SELECT COUNT(*) AS n FROM pdpp_records WHERE instance = ? AND stream = ? AND deleted = 0")
|
|
664
790
|
.get(input.instance, input.stream).n);
|
|
665
791
|
}
|
|
792
|
+
if (input.sourceCollectedAt !== undefined)
|
|
793
|
+
replica?.captureCompletion(input.instance, input.stream);
|
|
794
|
+
replica?.settleLocalWrites();
|
|
666
795
|
return { ...summary, applied: true, deleted: missing.length };
|
|
667
|
-
})
|
|
796
|
+
});
|
|
668
797
|
}
|
|
669
798
|
// Under `preserve_if_unchanged`: true when this replace writes nothing to
|
|
670
799
|
// the stream's rows, no live key is missing from the snapshot, and the
|
|
@@ -756,7 +885,7 @@ export function createSqliteRecordStore(db) {
|
|
|
756
885
|
};
|
|
757
886
|
}
|
|
758
887
|
function resetInstanceBinding(input) {
|
|
759
|
-
return
|
|
888
|
+
return connectionMutation(input.instance, () => {
|
|
760
889
|
const current = ensureBinding(db, input.instance);
|
|
761
890
|
if (current.deleted_at !== null) {
|
|
762
891
|
throw new PdppBindingError("connection_deleted");
|
|
@@ -782,11 +911,213 @@ export function createSqliteRecordStore(db) {
|
|
|
782
911
|
eraseInstanceData(input.instance);
|
|
783
912
|
const resetClock = nextWriteSeq.get().value;
|
|
784
913
|
db.prepare("UPDATE pdpp_instance_binding SET method = ?, generation = ?, reset_clock = ? WHERE instance = ?").run(input.nextMethod, input.expectedGeneration + 1, resetClock, input.instance);
|
|
914
|
+
replica?.captureLifecycle(input.instance, "connection-reset");
|
|
915
|
+
replica?.notifyVisibility();
|
|
785
916
|
return {
|
|
786
917
|
binding: toBinding(ensureBinding(db, input.instance)),
|
|
787
918
|
alreadyReset: false,
|
|
788
919
|
};
|
|
789
|
-
})
|
|
920
|
+
});
|
|
921
|
+
}
|
|
922
|
+
function transitionWriteClock() {
|
|
923
|
+
return db.prepare("SELECT value FROM pdpp_write_clock WHERE id = 1").get().value;
|
|
924
|
+
}
|
|
925
|
+
function claudeTransition(instance) {
|
|
926
|
+
const row = db
|
|
927
|
+
.prepare("SELECT * FROM pdpp_claude_transition WHERE instance = ?")
|
|
928
|
+
.get(instance);
|
|
929
|
+
return row
|
|
930
|
+
? {
|
|
931
|
+
instance: row.instance,
|
|
932
|
+
archiveInstance: row.archive_instance,
|
|
933
|
+
conversationsCovered: row.conversations_covered === 1,
|
|
934
|
+
generation: row.generation,
|
|
935
|
+
snapshots: JSON.parse(row.snapshots_json),
|
|
936
|
+
}
|
|
937
|
+
: null;
|
|
938
|
+
}
|
|
939
|
+
function listClaudeTransitions() {
|
|
940
|
+
return db
|
|
941
|
+
.prepare("SELECT instance FROM pdpp_claude_transition ORDER BY instance")
|
|
942
|
+
.all().map((row) => claudeTransition(row.instance));
|
|
943
|
+
}
|
|
944
|
+
function confirmClaudeTransition(instance, generation) {
|
|
945
|
+
connectionMutation(instance, () => db
|
|
946
|
+
.prepare("UPDATE pdpp_claude_transition SET conversations_covered = 1 WHERE instance = ? AND generation = ?")
|
|
947
|
+
.run(instance, generation));
|
|
948
|
+
}
|
|
949
|
+
function clearClaudeTransitionScope(scope) {
|
|
950
|
+
if (scope !== "claude.conversations" && scope !== "claude.projects")
|
|
951
|
+
return;
|
|
952
|
+
connectionMutation(() => listClaudeTransitions().map((transition) => transition.instance), () => {
|
|
953
|
+
for (const transition of listClaudeTransitions()) {
|
|
954
|
+
delete transition.snapshots[scope];
|
|
955
|
+
db.prepare("UPDATE pdpp_claude_transition SET snapshots_json = ? WHERE instance = ?").run(JSON.stringify(transition.snapshots), transition.instance);
|
|
956
|
+
const streams = scope === "claude.conversations"
|
|
957
|
+
? ["conversations", "messages"]
|
|
958
|
+
: ["projects", "project_documents"];
|
|
959
|
+
if (Object.keys(transition.snapshots).length === 0)
|
|
960
|
+
streams.push("account_profile");
|
|
961
|
+
for (const table of [
|
|
962
|
+
"pdpp_records",
|
|
963
|
+
"pdpp_record_changes",
|
|
964
|
+
"pdpp_stream_completion",
|
|
965
|
+
"pdpp_record_arrival",
|
|
966
|
+
]) {
|
|
967
|
+
for (const stream of streams)
|
|
968
|
+
db.prepare(`DELETE FROM ${table} WHERE instance = ? AND stream = ?`).run(transition.archiveInstance, stream);
|
|
969
|
+
}
|
|
970
|
+
db.prepare(`DELETE FROM pdpp_blob_claims WHERE instance = ?
|
|
971
|
+
AND NOT EXISTS (SELECT 1 FROM pdpp_records r WHERE r.instance = pdpp_blob_claims.instance AND r.blob_id = pdpp_blob_claims.blob_id)
|
|
972
|
+
AND NOT EXISTS (SELECT 1 FROM pdpp_record_changes c WHERE c.instance = pdpp_blob_claims.instance AND json_extract(c.data, '$.blob_ref.blob_id') = pdpp_blob_claims.blob_id)`).run(transition.archiveInstance);
|
|
973
|
+
}
|
|
974
|
+
deleteUnclaimedBlobs(db);
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
const STRAVA_SOURCE = "https://registry.pdpp.dev/sources/strava";
|
|
978
|
+
const STRAVA_EXPORT_METHOD = "strava";
|
|
979
|
+
const STRAVA_BROWSER_METHOD = "strava-browser";
|
|
980
|
+
const STRAVA_SCOPE_STREAMS = {
|
|
981
|
+
"strava.activities": "activities",
|
|
982
|
+
"strava.coverage_diagnostics": "coverage_diagnostics",
|
|
983
|
+
};
|
|
984
|
+
/** Tables whose rows move to the archive instance on a method transition. */
|
|
985
|
+
const ARCHIVED_TABLES = [
|
|
986
|
+
"pdpp_records",
|
|
987
|
+
"pdpp_record_changes",
|
|
988
|
+
"pdpp_blob_claims",
|
|
989
|
+
"pdpp_stream_completion",
|
|
990
|
+
"pdpp_record_arrival",
|
|
991
|
+
];
|
|
992
|
+
function stravaTransition(instance) {
|
|
993
|
+
const row = db
|
|
994
|
+
.prepare("SELECT * FROM pdpp_strava_transition WHERE instance = ?")
|
|
995
|
+
.get(instance);
|
|
996
|
+
if (!row)
|
|
997
|
+
return null;
|
|
998
|
+
return {
|
|
999
|
+
instance: row.instance,
|
|
1000
|
+
archiveInstance: row.archive_instance,
|
|
1001
|
+
activitiesCovered: row.activities_covered === 1,
|
|
1002
|
+
generation: row.generation,
|
|
1003
|
+
snapshots: JSON.parse(row.snapshots_json),
|
|
1004
|
+
};
|
|
1005
|
+
}
|
|
1006
|
+
function listStravaTransitions() {
|
|
1007
|
+
return db
|
|
1008
|
+
.prepare("SELECT instance FROM pdpp_strava_transition ORDER BY instance")
|
|
1009
|
+
.all().map((row) => stravaTransition(row.instance));
|
|
1010
|
+
}
|
|
1011
|
+
function confirmStravaTransition(instance, generation) {
|
|
1012
|
+
connectionMutation(instance, () => db
|
|
1013
|
+
.prepare("UPDATE pdpp_strava_transition SET activities_covered = 1 WHERE instance = ? AND generation = ?")
|
|
1014
|
+
.run(instance, generation));
|
|
1015
|
+
}
|
|
1016
|
+
/** A takeover reset drops the saved body and its archived rows. */
|
|
1017
|
+
function clearStravaTransitionScope(scope) {
|
|
1018
|
+
if (!(scope in STRAVA_SCOPE_STREAMS))
|
|
1019
|
+
return;
|
|
1020
|
+
const stream = STRAVA_SCOPE_STREAMS[scope];
|
|
1021
|
+
connectionMutation(() => listStravaTransitions().map((transition) => transition.instance), () => {
|
|
1022
|
+
for (const transition of listStravaTransitions()) {
|
|
1023
|
+
delete transition.snapshots[scope];
|
|
1024
|
+
db.prepare("UPDATE pdpp_strava_transition SET snapshots_json = ? WHERE instance = ?").run(JSON.stringify(transition.snapshots), transition.instance);
|
|
1025
|
+
for (const table of ARCHIVED_TABLES) {
|
|
1026
|
+
if (table === "pdpp_blob_claims")
|
|
1027
|
+
continue;
|
|
1028
|
+
db.prepare(`DELETE FROM ${table} WHERE instance = ? AND stream = ?`).run(transition.archiveInstance, stream);
|
|
1029
|
+
}
|
|
1030
|
+
// Unreadable retained history may still reference any archived blob.
|
|
1031
|
+
db.prepare(`DELETE FROM pdpp_blob_claims WHERE instance = ?
|
|
1032
|
+
AND NOT EXISTS (SELECT 1 FROM pdpp_records r WHERE r.instance = pdpp_blob_claims.instance AND r.blob_id = pdpp_blob_claims.blob_id)
|
|
1033
|
+
AND NOT EXISTS (
|
|
1034
|
+
SELECT 1 FROM pdpp_record_changes c
|
|
1035
|
+
WHERE c.instance = pdpp_blob_claims.instance
|
|
1036
|
+
AND CASE WHEN json_valid(c.data)
|
|
1037
|
+
THEN json_extract(c.data, '$.blob_ref.blob_id') = pdpp_blob_claims.blob_id
|
|
1038
|
+
ELSE c.data IS NOT NULL END
|
|
1039
|
+
)`).run(transition.archiveInstance);
|
|
1040
|
+
}
|
|
1041
|
+
// A scope reset must retain blobs that unreadable history may reference.
|
|
1042
|
+
collectStartupUnclaimedBlobs(db);
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
/**
|
|
1046
|
+
* Owner-authorized, one-way: archive the export rows and bind the browser
|
|
1047
|
+
* method at the next generation in one transaction. Replaying the same
|
|
1048
|
+
* intent after success returns the current binding and erases nothing.
|
|
1049
|
+
*/
|
|
1050
|
+
function transitionStravaToBrowser(input) {
|
|
1051
|
+
return connectionMutation(input.instance, () => {
|
|
1052
|
+
replica?.assertMethodTransitionAllowed();
|
|
1053
|
+
const current = getInstanceBinding(input.instance);
|
|
1054
|
+
if (current.deletedAt !== null)
|
|
1055
|
+
throw new PdppBindingError("connection_deleted");
|
|
1056
|
+
const previous = stravaTransition(input.instance);
|
|
1057
|
+
if (previous &&
|
|
1058
|
+
previous.generation === input.expectedGeneration + 1 &&
|
|
1059
|
+
current.method === STRAVA_BROWSER_METHOD &&
|
|
1060
|
+
current.generation === previous.generation)
|
|
1061
|
+
return current;
|
|
1062
|
+
if (current.method !== STRAVA_EXPORT_METHOD ||
|
|
1063
|
+
current.sourceId !== STRAVA_SOURCE)
|
|
1064
|
+
throw new PdppBindingError("connection_conflict");
|
|
1065
|
+
if (current.generation !== input.expectedGeneration ||
|
|
1066
|
+
transitionWriteClock() !== input.expectedWriteClock)
|
|
1067
|
+
throw new PdppBindingError("binding_generation_mismatch");
|
|
1068
|
+
if (previous || listConnections(current.sourceId).length !== 1)
|
|
1069
|
+
throw new PdppBindingError("connection_conflict");
|
|
1070
|
+
const archive = `retired-strava:${randomUUID()}`;
|
|
1071
|
+
const generation = current.generation + 1;
|
|
1072
|
+
db.prepare("INSERT INTO pdpp_strava_transition (instance, archive_instance, generation, snapshots_json) VALUES (?, ?, ?, ?)").run(input.instance, archive, generation, JSON.stringify(input.snapshots));
|
|
1073
|
+
// The archive is not a registered connection: it cannot accept writes.
|
|
1074
|
+
for (const table of ARCHIVED_TABLES) {
|
|
1075
|
+
db.prepare(`UPDATE ${table} SET instance = ? WHERE instance = ?`).run(archive, input.instance);
|
|
1076
|
+
}
|
|
1077
|
+
const clock = nextWriteSeq.get().value;
|
|
1078
|
+
db.prepare("UPDATE pdpp_instance_binding SET method = ?, generation = ?, reset_clock = ? WHERE instance = ?").run(STRAVA_BROWSER_METHOD, generation, clock, input.instance);
|
|
1079
|
+
return getInstanceBinding(input.instance);
|
|
1080
|
+
});
|
|
1081
|
+
}
|
|
1082
|
+
function transitionClaudeToLive(input) {
|
|
1083
|
+
return connectionMutation(input.instance, () => {
|
|
1084
|
+
replica?.assertMethodTransitionAllowed();
|
|
1085
|
+
const current = getInstanceBinding(input.instance);
|
|
1086
|
+
if (current.deletedAt !== null)
|
|
1087
|
+
throw new PdppBindingError("connection_deleted");
|
|
1088
|
+
const previous = claudeTransition(input.instance);
|
|
1089
|
+
// Replaying the same intent must not erase a successful live import.
|
|
1090
|
+
if (previous &&
|
|
1091
|
+
previous.generation === input.expectedGeneration + 1 &&
|
|
1092
|
+
current.method === "anthropic-live" &&
|
|
1093
|
+
current.generation === previous.generation)
|
|
1094
|
+
return current;
|
|
1095
|
+
if (current.method !== "anthropic" ||
|
|
1096
|
+
current.sourceId !== "https://registry.pdpp.dev/sources/claude")
|
|
1097
|
+
throw new PdppBindingError("connection_conflict");
|
|
1098
|
+
if (current.generation !== input.expectedGeneration ||
|
|
1099
|
+
transitionWriteClock() !== input.expectedWriteClock)
|
|
1100
|
+
throw new PdppBindingError("binding_generation_mismatch");
|
|
1101
|
+
if (previous || listConnections(current.sourceId).length !== 1)
|
|
1102
|
+
throw new PdppBindingError("connection_conflict");
|
|
1103
|
+
const archive = `retired-claude:${randomUUID()}`;
|
|
1104
|
+
const generation = current.generation + 1;
|
|
1105
|
+
db.prepare("INSERT INTO pdpp_claude_transition (instance, archive_instance, generation, snapshots_json) VALUES (?, ?, ?, ?)").run(input.instance, archive, generation, JSON.stringify(input.snapshots));
|
|
1106
|
+
// An archive is deliberately not a registered connection and cannot mint
|
|
1107
|
+
// owner tokens or accept writes. Blob claims keep the old bytes alive.
|
|
1108
|
+
for (const table of [
|
|
1109
|
+
"pdpp_records",
|
|
1110
|
+
"pdpp_record_changes",
|
|
1111
|
+
"pdpp_blob_claims",
|
|
1112
|
+
"pdpp_stream_completion",
|
|
1113
|
+
"pdpp_record_arrival",
|
|
1114
|
+
]) {
|
|
1115
|
+
db.prepare(`UPDATE ${table} SET instance = ? WHERE instance = ?`).run(archive, input.instance);
|
|
1116
|
+
}
|
|
1117
|
+
const clock = nextWriteSeq.get().value;
|
|
1118
|
+
db.prepare("UPDATE pdpp_instance_binding SET method = 'anthropic-live', generation = ?, reset_clock = ? WHERE instance = ?").run(generation, clock, input.instance);
|
|
1119
|
+
return getInstanceBinding(input.instance);
|
|
1120
|
+
});
|
|
790
1121
|
}
|
|
791
1122
|
function listConnections(sourceId) {
|
|
792
1123
|
const rows = db
|
|
@@ -795,7 +1126,7 @@ export function createSqliteRecordStore(db) {
|
|
|
795
1126
|
return rows.map(toBinding);
|
|
796
1127
|
}
|
|
797
1128
|
function registerConnection(input) {
|
|
798
|
-
return
|
|
1129
|
+
return connectionMutation(input.instance, () => {
|
|
799
1130
|
const current = ensureBinding(db, input.instance);
|
|
800
1131
|
if (current.deleted_at !== null) {
|
|
801
1132
|
throw new PdppBindingError("connection_deleted");
|
|
@@ -809,11 +1140,14 @@ export function createSqliteRecordStore(db) {
|
|
|
809
1140
|
throw new PdppBindingError("connection_conflict");
|
|
810
1141
|
}
|
|
811
1142
|
db.prepare("UPDATE pdpp_instance_binding SET source_id = ?, method = ?, label = ? WHERE instance = ?").run(input.sourceId, input.method, input.label, input.instance);
|
|
1143
|
+
if (current.source_id === null)
|
|
1144
|
+
replica?.captureLifecycle(input.instance, "connection-register");
|
|
1145
|
+
replica?.notifyVisibility();
|
|
812
1146
|
return getInstanceBinding(input.instance);
|
|
813
|
-
})
|
|
1147
|
+
});
|
|
814
1148
|
}
|
|
815
1149
|
function deleteConnection(instance) {
|
|
816
|
-
return
|
|
1150
|
+
return connectionMutation(instance, () => {
|
|
817
1151
|
const current = ensureBinding(db, instance);
|
|
818
1152
|
if (current.deleted_at !== null)
|
|
819
1153
|
return toBinding(current);
|
|
@@ -821,10 +1155,22 @@ export function createSqliteRecordStore(db) {
|
|
|
821
1155
|
const resetClock = nextWriteSeq.get().value;
|
|
822
1156
|
const deletedAt = new Date().toISOString();
|
|
823
1157
|
db.prepare("UPDATE pdpp_instance_binding SET deleted_at = ?, generation = generation + 1, reset_clock = ? WHERE instance = ?").run(deletedAt, resetClock, instance);
|
|
1158
|
+
replica?.captureLifecycle(instance, "connection-delete");
|
|
1159
|
+
replica?.notifyVisibility();
|
|
824
1160
|
return getInstanceBinding(instance);
|
|
825
|
-
})
|
|
1161
|
+
});
|
|
826
1162
|
}
|
|
827
1163
|
function eraseInstanceData(instance) {
|
|
1164
|
+
const transition = claudeTransition(instance);
|
|
1165
|
+
if (transition) {
|
|
1166
|
+
db.prepare("DELETE FROM pdpp_claude_transition WHERE instance = ?").run(instance);
|
|
1167
|
+
eraseInstanceData(transition.archiveInstance);
|
|
1168
|
+
}
|
|
1169
|
+
const strava = stravaTransition(instance);
|
|
1170
|
+
if (strava) {
|
|
1171
|
+
db.prepare("DELETE FROM pdpp_strava_transition WHERE instance = ?").run(instance);
|
|
1172
|
+
eraseInstanceData(strava.archiveInstance);
|
|
1173
|
+
}
|
|
828
1174
|
db.prepare("DELETE FROM pdpp_records WHERE instance = ?").run(instance);
|
|
829
1175
|
db.prepare("DELETE FROM pdpp_record_changes WHERE instance = ?").run(instance);
|
|
830
1176
|
db.prepare("DELETE FROM pdpp_blob_claims WHERE instance = ?").run(instance);
|
|
@@ -875,8 +1221,7 @@ export function createSqliteRecordStore(db) {
|
|
|
875
1221
|
}
|
|
876
1222
|
function completeStreams(rawRequest, declarations) {
|
|
877
1223
|
const request = parseCompletionRequest(rawRequest);
|
|
878
|
-
return
|
|
879
|
-
.transaction(() => {
|
|
1224
|
+
return immediateConnectionMutation(request.instance, () => {
|
|
880
1225
|
const binding = db
|
|
881
1226
|
.prepare("SELECT * FROM pdpp_instance_binding WHERE instance = ?")
|
|
882
1227
|
.get(request.instance);
|
|
@@ -900,6 +1245,7 @@ export function createSqliteRecordStore(db) {
|
|
|
900
1245
|
// Validate every stream before the first receipt is written.
|
|
901
1246
|
const writes = [];
|
|
902
1247
|
for (const proof of request.streams) {
|
|
1248
|
+
replica?.assertStreamWritable(request.instance, proof.stream, request.generation);
|
|
903
1249
|
const primaryKey = declarations(request.sourceId, proof.stream);
|
|
904
1250
|
if (primaryKey === undefined) {
|
|
905
1251
|
throw new StreamCompletionError("unknown_stream");
|
|
@@ -969,6 +1315,10 @@ export function createSqliteRecordStore(db) {
|
|
|
969
1315
|
WHERE instance = ? AND stream = ?`).run(request.sourceCollectedAt, clock, proof.snapshotSha256, request.collectionTimeBasis, request.instance, proof.stream);
|
|
970
1316
|
}
|
|
971
1317
|
}
|
|
1318
|
+
for (const write of writes) {
|
|
1319
|
+
if (write.kind !== "keep")
|
|
1320
|
+
replica?.captureCompletion(request.instance, write.stream);
|
|
1321
|
+
}
|
|
972
1322
|
return {
|
|
973
1323
|
receipts: request.streams.map((proof) => {
|
|
974
1324
|
const row = db
|
|
@@ -987,8 +1337,7 @@ export function createSqliteRecordStore(db) {
|
|
|
987
1337
|
};
|
|
988
1338
|
}),
|
|
989
1339
|
};
|
|
990
|
-
})
|
|
991
|
-
.immediate();
|
|
1340
|
+
});
|
|
992
1341
|
}
|
|
993
1342
|
function listArrivalOrder(instance, stream) {
|
|
994
1343
|
const rows = db
|
|
@@ -1001,7 +1350,11 @@ export function createSqliteRecordStore(db) {
|
|
|
1001
1350
|
return rows.map((row) => row.record_key);
|
|
1002
1351
|
}
|
|
1003
1352
|
function getRecord(instance, stream, recordKey) {
|
|
1004
|
-
const row =
|
|
1353
|
+
const row = (replica
|
|
1354
|
+
? db
|
|
1355
|
+
.prepare(`SELECT r.* FROM pdpp_records r WHERE r.instance=? AND r.stream=? AND r.record_key=?${unquarantined("r")}`)
|
|
1356
|
+
.get(instance, stream, recordKey)
|
|
1357
|
+
: getCurrentStmt.get(instance, stream, recordKey));
|
|
1005
1358
|
if (!row || row.deleted)
|
|
1006
1359
|
return undefined;
|
|
1007
1360
|
return toRow(row);
|
|
@@ -1036,75 +1389,88 @@ export function createSqliteRecordStore(db) {
|
|
|
1036
1389
|
throw new CursorExpiredError();
|
|
1037
1390
|
}
|
|
1038
1391
|
}
|
|
1392
|
+
/** Latest version of each complete identity at one fixed write horizon. */
|
|
1393
|
+
function* snapshotRows(stream, instanceIds, horizon, options = {}) {
|
|
1394
|
+
const dir = options.order === "desc" ? "DESC" : "ASC";
|
|
1395
|
+
const comparison = options.order === "desc" ? "<" : ">";
|
|
1396
|
+
let sql = `SELECT c.* FROM pdpp_record_changes c INDEXED BY idx_pdpp_record_changes_snapshot_key_order
|
|
1397
|
+
WHERE c.stream = ? AND c.instance IN (${instanceIds.map(() => "?").join(",")}) AND c.written_at <= ?
|
|
1398
|
+
AND c.version = (
|
|
1399
|
+
SELECT c2.version FROM pdpp_record_changes c2 INDEXED BY idx_pdpp_record_changes_snapshot_latest
|
|
1400
|
+
WHERE c2.instance = c.instance AND c2.stream = c.stream AND c2.record_key = c.record_key AND c2.written_at <= ?
|
|
1401
|
+
ORDER BY c2.written_at DESC, c2.version DESC LIMIT 1
|
|
1402
|
+
)`;
|
|
1403
|
+
const params = [stream, ...instanceIds, horizon, horizon];
|
|
1404
|
+
if (options.liveOnly)
|
|
1405
|
+
sql += " AND c.deleted = 0";
|
|
1406
|
+
sql += unquarantined("c");
|
|
1407
|
+
if (options.after) {
|
|
1408
|
+
sql += ` AND (c.emitted_key, c.record_key, c.instance) ${comparison} (?, ?, ?)`;
|
|
1409
|
+
params.push(timeSortKey(options.after.val), options.after.key, options.after.instance);
|
|
1410
|
+
}
|
|
1411
|
+
sql += ` ORDER BY ${LIST_ORDER_COLUMNS.map((column) => `c.${column} ${dir}`).join(", ")}`;
|
|
1412
|
+
for (const row of db.prepare(sql).iterate(...params))
|
|
1413
|
+
yield row;
|
|
1414
|
+
}
|
|
1039
1415
|
function listRecords(stream, options) {
|
|
1416
|
+
// With replication, one read snapshot: a fence that commits meanwhile
|
|
1417
|
+
// cannot mix into the page or label its cursor with the post-fence epoch.
|
|
1418
|
+
return inReadSnapshot(() => listRecordsAtHorizon(stream, options));
|
|
1419
|
+
}
|
|
1420
|
+
function listRecordsAtHorizon(stream, options) {
|
|
1040
1421
|
let startAfter = null;
|
|
1041
1422
|
let horizon;
|
|
1042
1423
|
if (options.cursor) {
|
|
1043
|
-
const payload =
|
|
1044
|
-
if (payload.kind !== "list")
|
|
1045
|
-
throw new InvalidCursorError();
|
|
1424
|
+
const payload = decodeListCursor(options.cursor);
|
|
1046
1425
|
assertCursorScope(payload, stream);
|
|
1047
1426
|
if (payload.order !== options.order)
|
|
1048
1427
|
throw new InvalidCursorError();
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1428
|
+
horizon = parseHorizon(payload.horizon);
|
|
1429
|
+
replica?.assertCursorIntegrity(payload, options.instanceIds, options.fields);
|
|
1430
|
+
assertNotResetSince(options.instanceIds, [horizon]);
|
|
1431
|
+
startAfter = {
|
|
1432
|
+
val: payload.sortValue ?? "",
|
|
1433
|
+
key: payload.recordKey,
|
|
1434
|
+
instance: payload.instance,
|
|
1435
|
+
};
|
|
1055
1436
|
}
|
|
1056
1437
|
else {
|
|
1057
1438
|
horizon = readWriteClock();
|
|
1058
1439
|
}
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
sql += ` AND (emitted_at ${cmp} ? OR (emitted_at = ? AND record_key ${cmp} ?))`;
|
|
1069
|
-
params.push(startAfter.val, startAfter.val, startAfter.key);
|
|
1440
|
+
// The generator holds at most one raw row beyond the visible client page.
|
|
1441
|
+
function* rows() {
|
|
1442
|
+
for (const row of snapshotRows(stream, options.instanceIds, horizon, {
|
|
1443
|
+
order: options.order,
|
|
1444
|
+
after: startAfter,
|
|
1445
|
+
liveOnly: true,
|
|
1446
|
+
})) {
|
|
1447
|
+
yield toRow(row);
|
|
1448
|
+
}
|
|
1070
1449
|
}
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
data: page.map((row) => {
|
|
1079
|
-
const stored = toRow(row);
|
|
1080
|
-
return { ...stored, data: projectFields(stored.data, options.fields) };
|
|
1081
|
-
}),
|
|
1082
|
-
hasMore,
|
|
1083
|
-
nextCursor: hasMore && last
|
|
1084
|
-
? encodeCursor({
|
|
1085
|
-
kind: "list",
|
|
1086
|
-
stream,
|
|
1087
|
-
epoch: cursorEpoch(),
|
|
1088
|
-
order: options.order,
|
|
1089
|
-
sortValue: last.emitted_at,
|
|
1090
|
-
recordKey: last.record_key,
|
|
1091
|
-
horizon: String(horizon),
|
|
1092
|
-
})
|
|
1093
|
-
: undefined,
|
|
1450
|
+
const hasLaterWrites = horizon < readWriteClock();
|
|
1451
|
+
const erasure = db.prepare(`SELECT 1 FROM pdpp_record_changes INDEXED BY idx_pdpp_record_changes_snapshot_latest
|
|
1452
|
+
WHERE instance = ? AND stream = ? AND record_key = ? AND written_at > ? AND deleted = 1 LIMIT 1`);
|
|
1453
|
+
return listSnapshotPage(rows(), options, {
|
|
1454
|
+
stream,
|
|
1455
|
+
epoch: cursorEpoch(),
|
|
1456
|
+
...replica?.cursorBinding(options.instanceIds, options.fields),
|
|
1094
1457
|
horizon: String(horizon),
|
|
1095
|
-
}
|
|
1458
|
+
}, (row) => hasLaterWrites &&
|
|
1459
|
+
erasure.get(row.instance, row.stream, row.recordKey, horizon) !==
|
|
1460
|
+
undefined);
|
|
1096
1461
|
}
|
|
1097
1462
|
function deleteRecord(instance, stream, recordKey, deletedAt, streamSemantics) {
|
|
1098
1463
|
if (streamSemantics === "append_only")
|
|
1099
1464
|
return false;
|
|
1100
1465
|
let result = false;
|
|
1101
|
-
const run =
|
|
1466
|
+
const run = () => connectionMutation(instance, () => {
|
|
1467
|
+
replica?.assertKeyWritable(instance, stream, recordKey);
|
|
1102
1468
|
const existing = getCurrentStmt.get(instance, stream, recordKey);
|
|
1103
1469
|
if (!existing || existing.deleted)
|
|
1104
1470
|
return;
|
|
1105
1471
|
const version = existing.version + 1;
|
|
1106
1472
|
const writtenAt = nextWriteSeq.get();
|
|
1107
|
-
|
|
1473
|
+
dropCompletion(instance, stream);
|
|
1108
1474
|
upsertCurrentStmt.run({
|
|
1109
1475
|
instance,
|
|
1110
1476
|
stream,
|
|
@@ -1127,20 +1493,31 @@ export function createSqliteRecordStore(db) {
|
|
|
1127
1493
|
deleted_at: deletedAt,
|
|
1128
1494
|
written_at: writtenAt.value,
|
|
1129
1495
|
});
|
|
1496
|
+
replica?.recordWritten(instance, stream, recordKey, streamSemantics);
|
|
1130
1497
|
forgetArrivalStmt.run(instance, stream, recordKey);
|
|
1498
|
+
replica?.settleLocalWrites();
|
|
1131
1499
|
result = true;
|
|
1132
1500
|
});
|
|
1133
1501
|
run();
|
|
1134
1502
|
return result;
|
|
1135
1503
|
}
|
|
1136
|
-
function changesSince(stream, options) {
|
|
1504
|
+
function changesSince(stream, options, visibleInstanceIds = options.instanceIds) {
|
|
1505
|
+
if (!replica)
|
|
1506
|
+
return changesSinceInSnapshot(stream, options, visibleInstanceIds, undefined);
|
|
1507
|
+
// A new session allocates its horizon first, in its own write. The rest
|
|
1508
|
+
// reads one snapshot, so a concurrent fence is wholly before or after it.
|
|
1509
|
+
const fresh = options.cursor ? undefined : allocateHorizon();
|
|
1510
|
+
return inReadSnapshot(() => changesSinceInSnapshot(stream, options, visibleInstanceIds, fresh));
|
|
1511
|
+
}
|
|
1512
|
+
function allocateHorizon() {
|
|
1513
|
+
return nextWriteSeq.get().value;
|
|
1514
|
+
}
|
|
1515
|
+
function changesSinceInSnapshot(stream, options, visibleInstanceIds, freshHorizon) {
|
|
1137
1516
|
let horizon;
|
|
1138
1517
|
let sinceHorizon;
|
|
1139
1518
|
let offset = 0;
|
|
1140
1519
|
if (options.cursor) {
|
|
1141
|
-
const payload =
|
|
1142
|
-
if (payload.kind !== "changes_since")
|
|
1143
|
-
throw new InvalidCursorError();
|
|
1520
|
+
const payload = decodeSyncPageCursor(options.cursor);
|
|
1144
1521
|
assertCursorScope(payload, stream);
|
|
1145
1522
|
horizon = parseHorizon(payload.horizon);
|
|
1146
1523
|
sinceHorizon =
|
|
@@ -1148,6 +1525,7 @@ export function createSqliteRecordStore(db) {
|
|
|
1148
1525
|
? parseHorizon(payload.sinceHorizon)
|
|
1149
1526
|
: null;
|
|
1150
1527
|
offset = parseOffset(payload.offset);
|
|
1528
|
+
replica?.assertCursorIntegrity(payload, options.instanceIds, options.fields);
|
|
1151
1529
|
}
|
|
1152
1530
|
else if (options.changesSince) {
|
|
1153
1531
|
let payload;
|
|
@@ -1163,29 +1541,21 @@ export function createSqliteRecordStore(db) {
|
|
|
1163
1541
|
throw new InvalidCursorError();
|
|
1164
1542
|
assertCursorScope(payload, stream);
|
|
1165
1543
|
sinceHorizon = parseHorizon(payload.horizon);
|
|
1166
|
-
|
|
1544
|
+
replica?.assertCursorIntegrity(payload, options.instanceIds, undefined);
|
|
1545
|
+
horizon = freshHorizon ?? allocateHorizon();
|
|
1167
1546
|
}
|
|
1168
1547
|
else {
|
|
1169
1548
|
sinceHorizon = null;
|
|
1170
|
-
horizon =
|
|
1549
|
+
horizon = freshHorizon ?? allocateHorizon();
|
|
1171
1550
|
}
|
|
1172
1551
|
assertNotResetSince(options.instanceIds, [
|
|
1173
1552
|
horizon,
|
|
1174
1553
|
sinceHorizon ?? horizon,
|
|
1175
1554
|
]);
|
|
1176
|
-
const
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
SELECT c.* FROM pdpp_record_changes c
|
|
1181
|
-
WHERE c.stream = ? AND c.instance IN (${placeholders}) AND c.written_at <= ?
|
|
1182
|
-
AND c.version = (
|
|
1183
|
-
SELECT MAX(c2.version) FROM pdpp_record_changes c2
|
|
1184
|
-
WHERE c2.stream = c.stream AND c2.instance = c.instance
|
|
1185
|
-
AND c2.record_key = c.record_key AND c2.written_at <= ?
|
|
1186
|
-
)
|
|
1187
|
-
`)
|
|
1188
|
-
.all(stream, ...options.instanceIds, horizon, horizon);
|
|
1555
|
+
const visibleIds = [...new Set(visibleInstanceIds)].filter((id) => options.instanceIds.includes(id));
|
|
1556
|
+
const latestRows = visibleIds.length
|
|
1557
|
+
? snapshotRows(stream, visibleIds, horizon)
|
|
1558
|
+
: [];
|
|
1189
1559
|
const changed = [];
|
|
1190
1560
|
for (const entry of latestRows) {
|
|
1191
1561
|
if (sinceHorizon !== null) {
|
|
@@ -1197,55 +1567,30 @@ export function createSqliteRecordStore(db) {
|
|
|
1197
1567
|
if (!wroteSincePrior)
|
|
1198
1568
|
continue;
|
|
1199
1569
|
}
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
deleted: true,
|
|
1209
|
-
});
|
|
1210
|
-
continue;
|
|
1211
|
-
}
|
|
1212
|
-
const currentData = entry.data ? JSON.parse(entry.data) : {};
|
|
1213
|
-
const projectedNow = projectFields(currentData, options.fields);
|
|
1214
|
-
if (sinceHorizon !== null && options.fields) {
|
|
1215
|
-
const priorRow = db
|
|
1216
|
-
.prepare(`SELECT * FROM pdpp_record_changes
|
|
1217
|
-
WHERE stream = ? AND instance = ? AND record_key = ? AND written_at <= ?
|
|
1218
|
-
ORDER BY version DESC LIMIT 1`)
|
|
1570
|
+
const previous = sinceHorizon === null
|
|
1571
|
+
? undefined
|
|
1572
|
+
: db
|
|
1573
|
+
.prepare(`
|
|
1574
|
+
SELECT * FROM pdpp_record_changes
|
|
1575
|
+
WHERE stream = ? AND instance = ? AND record_key = ? AND written_at <= ?
|
|
1576
|
+
ORDER BY version DESC LIMIT 1
|
|
1577
|
+
`)
|
|
1219
1578
|
.get(entry.stream, entry.instance, entry.record_key, sinceHorizon);
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
if (JSON.stringify(priorProjected) === JSON.stringify(projectedNow)) {
|
|
1224
|
-
continue;
|
|
1225
|
-
}
|
|
1226
|
-
}
|
|
1227
|
-
}
|
|
1228
|
-
// Return the RAW (unprojected) data here, not `projectedNow`.
|
|
1229
|
-
// `projectedNow`/`priorProjected` above exist only to decide
|
|
1230
|
-
// eligibility (did the grant-authorized projection change) without
|
|
1231
|
-
// leaking a hidden-field change. A caller enforcing time_constraint
|
|
1232
|
-
// against `data` needs the real field value; projecting here would
|
|
1233
|
-
// silently break that filter for any grant whose fields don't happen
|
|
1234
|
-
// to include the time_constraint field. Response field projection is
|
|
1235
|
-
// the caller's job, applied after any filtering on real values.
|
|
1236
|
-
changed.push({
|
|
1237
|
-
instance: entry.instance,
|
|
1238
|
-
stream: entry.stream,
|
|
1239
|
-
recordKey: entry.record_key,
|
|
1240
|
-
data: currentData,
|
|
1241
|
-
version: entry.version,
|
|
1242
|
-
emittedAt: entry.emitted_at,
|
|
1243
|
-
deleted: false,
|
|
1244
|
-
});
|
|
1579
|
+
const selected = selectSyncChange(toRow(entry), previous ? toRow(previous) : undefined, options);
|
|
1580
|
+
if (selected)
|
|
1581
|
+
changed.push(selected);
|
|
1245
1582
|
}
|
|
1246
|
-
changed.sort(
|
|
1583
|
+
changed.sort(compareRecordIdentity);
|
|
1247
1584
|
const page = changed.slice(offset, offset + options.limit);
|
|
1248
1585
|
const hasMore = offset + options.limit < changed.length;
|
|
1586
|
+
if (options.expireOnErasure && horizon < readWriteClock()) {
|
|
1587
|
+
const erasure = db.prepare(`SELECT 1 FROM pdpp_record_changes INDEXED BY idx_pdpp_record_changes_snapshot_latest
|
|
1588
|
+
WHERE instance = ? AND stream = ? AND record_key = ? AND written_at > ? AND deleted = 1 LIMIT 1`);
|
|
1589
|
+
if (page.some((row) => !row.deleted &&
|
|
1590
|
+
erasure.get(row.instance, stream, row.recordKey, horizon) !==
|
|
1591
|
+
undefined))
|
|
1592
|
+
throw new CursorExpiredError();
|
|
1593
|
+
}
|
|
1249
1594
|
return {
|
|
1250
1595
|
data: page,
|
|
1251
1596
|
hasMore,
|
|
@@ -1254,9 +1599,11 @@ export function createSqliteRecordStore(db) {
|
|
|
1254
1599
|
kind: "changes_since",
|
|
1255
1600
|
stream,
|
|
1256
1601
|
epoch: cursorEpoch(),
|
|
1602
|
+
...replica?.cursorBinding(options.instanceIds, options.fields),
|
|
1257
1603
|
horizon: String(horizon),
|
|
1258
1604
|
sinceHorizon: sinceHorizon !== null ? String(sinceHorizon) : null,
|
|
1259
1605
|
offset: offset + options.limit,
|
|
1606
|
+
orderKey: "code-point",
|
|
1260
1607
|
})
|
|
1261
1608
|
: undefined,
|
|
1262
1609
|
nextChangesSince: hasMore
|
|
@@ -1265,6 +1612,7 @@ export function createSqliteRecordStore(db) {
|
|
|
1265
1612
|
kind: "changes_since",
|
|
1266
1613
|
stream,
|
|
1267
1614
|
epoch: cursorEpoch(),
|
|
1615
|
+
...replica?.cursorBinding(options.instanceIds, undefined),
|
|
1268
1616
|
horizon: String(horizon),
|
|
1269
1617
|
sinceHorizon: null,
|
|
1270
1618
|
offset: 0,
|
|
@@ -1274,8 +1622,10 @@ export function createSqliteRecordStore(db) {
|
|
|
1274
1622
|
function listStreams(instanceIds) {
|
|
1275
1623
|
const placeholders = instanceIds.map(() => "?").join(",");
|
|
1276
1624
|
const rows = db
|
|
1277
|
-
.prepare(
|
|
1278
|
-
|
|
1625
|
+
.prepare(
|
|
1626
|
+
// SQLite takes a bare column from the row that holds the single MAX.
|
|
1627
|
+
`SELECT stream, COUNT(*) as count, emitted_at as last_updated, MAX(emitted_key)
|
|
1628
|
+
FROM pdpp_records r WHERE deleted = 0 AND instance IN (${placeholders})${unquarantined("r")}
|
|
1279
1629
|
GROUP BY stream`)
|
|
1280
1630
|
.all(...instanceIds);
|
|
1281
1631
|
return rows.map((r) => ({
|
|
@@ -1295,15 +1645,38 @@ export function createSqliteRecordStore(db) {
|
|
|
1295
1645
|
sha256: row.sha256,
|
|
1296
1646
|
};
|
|
1297
1647
|
}
|
|
1648
|
+
/**
|
|
1649
|
+
* Without replication the store writes main's `pdpp_blob_bytes` format. It
|
|
1650
|
+
* also reads a blob over 1 MiB that replication stored only as verified
|
|
1651
|
+
* canonical chunks, so that blob stays readable before a flag-off boot
|
|
1652
|
+
* writes main's row for it.
|
|
1653
|
+
*/
|
|
1654
|
+
const chunkStore = () => db
|
|
1655
|
+
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'pdpp_binary_manifests'")
|
|
1656
|
+
.get() !== undefined;
|
|
1657
|
+
const blobBytes = replica?.blobBytes ?? {
|
|
1658
|
+
has: (blobId) => getBlobBytesStmt.get(blobId) !== undefined ||
|
|
1659
|
+
(chunkStore() && binaryLength(db, "canonical", blobId) !== undefined),
|
|
1660
|
+
readVerified(meta) {
|
|
1661
|
+
const row = getBlobBytesStmt.get(meta.blobId);
|
|
1662
|
+
if (!row)
|
|
1663
|
+
return chunkStore() &&
|
|
1664
|
+
verifyBinary(db, "canonical", meta.blobId, `sha256:${meta.sha256}`, meta.sizeBytes)
|
|
1665
|
+
? readBinary(db, "canonical", meta.blobId)
|
|
1666
|
+
: undefined;
|
|
1667
|
+
const stored = new Uint8Array(row.bytes);
|
|
1668
|
+
if (stored.byteLength !== meta.sizeBytes)
|
|
1669
|
+
return undefined;
|
|
1670
|
+
return createHash("sha256").update(stored).digest("hex") === meta.sha256
|
|
1671
|
+
? stored
|
|
1672
|
+
: undefined;
|
|
1673
|
+
},
|
|
1674
|
+
clearMalformed: () => { },
|
|
1675
|
+
write: (blobId, bytes) => insertBlobBytesStmt.run({ blob_id: blobId, bytes: Buffer.from(bytes) }),
|
|
1676
|
+
};
|
|
1298
1677
|
/** True only when a stored bytes row actually hash/size-matches `meta`. */
|
|
1299
1678
|
function storedBytesVerify(meta) {
|
|
1300
|
-
|
|
1301
|
-
if (!row)
|
|
1302
|
-
return false;
|
|
1303
|
-
const stored = new Uint8Array(row.bytes);
|
|
1304
|
-
if (stored.byteLength !== meta.sizeBytes)
|
|
1305
|
-
return false;
|
|
1306
|
-
return createHash("sha256").update(stored).digest("hex") === meta.sha256;
|
|
1679
|
+
return blobBytes.readVerified(meta) !== undefined;
|
|
1307
1680
|
}
|
|
1308
1681
|
function storeBlobBytes(bytes, mimeType) {
|
|
1309
1682
|
const blobId = blobIdForBytes(bytes);
|
|
@@ -1320,6 +1693,7 @@ export function createSqliteRecordStore(db) {
|
|
|
1320
1693
|
// row" and race to insert, or one could complete a metadata-only row
|
|
1321
1694
|
// while another concurrently overwrites it.
|
|
1322
1695
|
const run = db.transaction(() => {
|
|
1696
|
+
blobBytes.clearMalformed(blobId);
|
|
1323
1697
|
const existing = getBlobMeta(blobId);
|
|
1324
1698
|
if (existing) {
|
|
1325
1699
|
if (existing.mimeType !== mimeType ||
|
|
@@ -1327,15 +1701,11 @@ export function createSqliteRecordStore(db) {
|
|
|
1327
1701
|
existing.sizeBytes !== bytes.byteLength) {
|
|
1328
1702
|
throw new BlobConflictError(blobId, existing);
|
|
1329
1703
|
}
|
|
1330
|
-
|
|
1331
|
-
if (!bytesRow) {
|
|
1704
|
+
if (!blobBytes.has(blobId)) {
|
|
1332
1705
|
// Metadata-only row (legacy/test fixture via putBlobMeta, or a
|
|
1333
1706
|
// prior write that never got this far): complete it with real
|
|
1334
1707
|
// bytes now instead of reporting a false "already stored" no-op.
|
|
1335
|
-
|
|
1336
|
-
blob_id: existing.blobId,
|
|
1337
|
-
bytes: Buffer.from(bytes),
|
|
1338
|
-
});
|
|
1708
|
+
blobBytes.write(blobId, bytes);
|
|
1339
1709
|
return existing;
|
|
1340
1710
|
}
|
|
1341
1711
|
if (!storedBytesVerify(existing)) {
|
|
@@ -1354,16 +1724,13 @@ export function createSqliteRecordStore(db) {
|
|
|
1354
1724
|
size_bytes: candidate.sizeBytes,
|
|
1355
1725
|
sha256: candidate.sha256,
|
|
1356
1726
|
});
|
|
1357
|
-
|
|
1358
|
-
blob_id: candidate.blobId,
|
|
1359
|
-
bytes: Buffer.from(bytes),
|
|
1360
|
-
});
|
|
1727
|
+
blobBytes.write(blobId, bytes);
|
|
1361
1728
|
return candidate;
|
|
1362
1729
|
});
|
|
1363
1730
|
return run();
|
|
1364
1731
|
}
|
|
1365
1732
|
function storeBlobBytesForInstance(input) {
|
|
1366
|
-
return
|
|
1733
|
+
return connectionMutation(input.instance, () => {
|
|
1367
1734
|
const binding = ensureBinding(db, input.instance);
|
|
1368
1735
|
if (binding.deleted_at !== null) {
|
|
1369
1736
|
throw new PdppBindingError("connection_deleted");
|
|
@@ -1381,41 +1748,79 @@ export function createSqliteRecordStore(db) {
|
|
|
1381
1748
|
db.prepare(`INSERT INTO pdpp_blob_claims (blob_id, instance, generation)
|
|
1382
1749
|
VALUES (?, ?, ?)
|
|
1383
1750
|
ON CONFLICT (blob_id, instance) DO UPDATE SET generation = excluded.generation`).run(meta.blobId, input.instance, input.generation);
|
|
1751
|
+
replica?.captureBlob(input.instance, meta.blobId);
|
|
1752
|
+
replica?.notifyVisibility();
|
|
1384
1753
|
return meta;
|
|
1385
|
-
})
|
|
1754
|
+
});
|
|
1386
1755
|
}
|
|
1387
1756
|
function getBlobBytes(blobId) {
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
return undefined;
|
|
1394
|
-
const bytes = new Uint8Array(row.bytes);
|
|
1395
|
-
if (bytes.byteLength !== meta.sizeBytes)
|
|
1396
|
-
return undefined;
|
|
1397
|
-
const actualHash = createHash("sha256").update(bytes).digest("hex");
|
|
1398
|
-
if (actualHash !== meta.sha256)
|
|
1399
|
-
return undefined;
|
|
1400
|
-
return bytes;
|
|
1757
|
+
// Verification and assembly share a snapshot.
|
|
1758
|
+
return inReadSnapshot(() => {
|
|
1759
|
+
const meta = getBlobMeta(blobId);
|
|
1760
|
+
return meta ? blobBytes.readVerified(meta) : undefined;
|
|
1761
|
+
});
|
|
1401
1762
|
}
|
|
1402
1763
|
return {
|
|
1764
|
+
transitionClaudeToLive,
|
|
1765
|
+
claudeTransition,
|
|
1766
|
+
listClaudeTransitions,
|
|
1767
|
+
clearClaudeTransitionScope,
|
|
1768
|
+
confirmClaudeTransition,
|
|
1769
|
+
transitionStravaToBrowser,
|
|
1770
|
+
stravaTransition,
|
|
1771
|
+
listStravaTransitions,
|
|
1772
|
+
clearStravaTransitionScope,
|
|
1773
|
+
confirmStravaTransition,
|
|
1774
|
+
transitionWriteClock,
|
|
1403
1775
|
listConnections,
|
|
1404
1776
|
registerConnection,
|
|
1405
1777
|
deleteConnection,
|
|
1406
1778
|
ingestBatch,
|
|
1407
1779
|
getInstanceBinding,
|
|
1408
|
-
|
|
1780
|
+
listRetainedRecordData: (instance, stream) => {
|
|
1781
|
+
const prior = db.prepare(`SELECT data FROM pdpp_record_changes
|
|
1782
|
+
WHERE instance = ? AND stream = ? AND record_key = ? AND deleted = 0
|
|
1783
|
+
ORDER BY version DESC LIMIT 1`);
|
|
1784
|
+
const rows = db
|
|
1785
|
+
.prepare(`SELECT record_key, data, emitted_at FROM pdpp_records
|
|
1786
|
+
WHERE instance = ? AND stream = ? ORDER BY record_key`)
|
|
1787
|
+
.all(instance, stream);
|
|
1788
|
+
return rows.map((row) => {
|
|
1789
|
+
const data = row.data ??
|
|
1790
|
+
prior.get(instance, stream, row.record_key)?.data;
|
|
1791
|
+
if (!data)
|
|
1792
|
+
throw new Error("Retained record history unavailable");
|
|
1793
|
+
return {
|
|
1794
|
+
data: JSON.parse(data),
|
|
1795
|
+
emittedAt: row.emitted_at,
|
|
1796
|
+
};
|
|
1797
|
+
});
|
|
1798
|
+
},
|
|
1799
|
+
readSnapshot: (read) => replica ? replica.snapshot(read) : db.transaction(read).deferred(),
|
|
1800
|
+
writeTransaction: (write) => replica ? replica.exclusive(write) : db.transaction(write).immediate(),
|
|
1409
1801
|
resetInstanceBinding,
|
|
1410
1802
|
storeBlobBytesForInstance,
|
|
1411
1803
|
replaceStream,
|
|
1804
|
+
replaceStreams: (inputs) => {
|
|
1805
|
+
const replaceAll = () => {
|
|
1806
|
+
for (const input of inputs) {
|
|
1807
|
+
if (!replaceStream(input).applied)
|
|
1808
|
+
throw new Error("canonical_batch_rejected");
|
|
1809
|
+
}
|
|
1810
|
+
};
|
|
1811
|
+
if (replica)
|
|
1812
|
+
replica.exclusive(replaceAll);
|
|
1813
|
+
else
|
|
1814
|
+
db.transaction(replaceAll)();
|
|
1815
|
+
},
|
|
1412
1816
|
listStreamCompletions,
|
|
1413
1817
|
completeStreams,
|
|
1414
1818
|
listArrivalOrder,
|
|
1415
1819
|
getRecord,
|
|
1416
1820
|
listRecords,
|
|
1417
1821
|
deleteRecord,
|
|
1418
|
-
changesSince,
|
|
1822
|
+
changesSince: (stream, options) => changesSince(stream, options),
|
|
1823
|
+
changesSinceForInstances: changesSince,
|
|
1419
1824
|
listStreams,
|
|
1420
1825
|
putBlobMeta: (meta) => putBlobStmt.run({
|
|
1421
1826
|
blob_id: meta.blobId,
|